fix: 400-600% CPU — move all monitor.lock() calls to spawn_blocking

The new get_top_processes endpoint called refresh_processes() (reads
/proc for every process) directly on the tokio async runtime while
holding the monitor Mutex. Combined with the 2-second background
monitoring loop also locking the same mutex on the async runtime,
this starved tokio worker threads and pinned multiple CPU cores.

Wrapped ALL monitor.lock().collect() and top_processes() calls in
tokio::task::spawn_blocking across api/mod.rs and main.rs so they
run on the blocking thread pool instead of starving the async executor.

Also fixes:
- Plugin backends not starting on reboot (start_all_backends was
  never called — wired into startup and reload)
- Mobile menu broken after update (CSS used static cache buster
  while JS used Date.now(), stale CSS lacked mobile sidebar fixes)

v16.11.3

Author: Paul C

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wolfstack"
-version = "16.11.2"
+version = "16.11.3"
 edition = "2024"
 authors = ["Wolf Software Systems Ltd"]
 description = "Server management platform for the Wolf software suite"

src/api/mod.rs

@@ -744,7 +744,10 @@ pub async fn disable_2fa(req: HttpRequest, state: web::Data<AppState>, path: web
 /// GET /api/metrics — current system metrics
 pub async fn get_metrics(req: HttpRequest, state: web::Data<AppState>) -> HttpResponse {
     if let Err(resp) = require_auth(&req, &state) { return resp; }
-    let metrics = state.monitor.lock().unwrap().collect();
+    let st = state.into_inner();
+    let metrics = tokio::task::spawn_blocking(move || {
+        st.monitor.lock().unwrap().collect()
+    }).await.unwrap();
     HttpResponse::Ok().json(metrics)
 }
 
@@ -758,7 +761,10 @@ pub async fn get_metrics_history(req: HttpRequest, state: web::Data<AppState>) -
 /// GET /api/metrics/processes — top CPU and memory consuming processes
 pub async fn get_top_processes(req: HttpRequest, state: web::Data<AppState>) -> HttpResponse {
     if let Err(resp) = require_auth(&req, &state) { return resp; }
-    let (top_cpu, top_mem) = state.monitor.lock().unwrap().top_processes(10);
+    let st = state.into_inner();
+    let (top_cpu, top_mem) = tokio::task::spawn_blocking(move || {
+        st.monitor.lock().unwrap().top_processes(10)
+    }).await.unwrap_or_default();
     HttpResponse::Ok().json(serde_json::json!({
         "top_cpu": top_cpu,
         "top_mem": top_mem,
@@ -2765,15 +2771,20 @@ pub async fn agent_status(req: HttpRequest, state: web::Data<AppState>) -> HttpR
     }
 
     // Fallback: first request before cache is populated (only happens once at startup)
-    let metrics = state.monitor.lock().unwrap().collect();
-    let components = installer::get_all_status();
+    let st = state.clone().into_inner();
+    let (metrics, components, docker_count, lxc_count, vm_count, has_docker, has_lxc, has_kvm) =
+        tokio::task::spawn_blocking(move || {
+            let m = st.monitor.lock().unwrap().collect();
+            let c = installer::get_all_status();
+            let dc = containers::docker_list_all().len() as u32;
+            let lc = containers::lxc_list_all().len() as u32;
+            let vc = st.vms.lock().unwrap().list_vms().len() as u32;
+            let hd = containers::docker_status().installed;
+            let hl = containers::lxc_status().installed;
+            let hk = containers::kvm_installed();
+            (m, c, dc, lc, vc, hd, hl, hk)
+        }).await.unwrap();
     let hostname = metrics.hostname.clone();
-    let docker_count = containers::docker_list_all().len() as u32;
-    let lxc_count = containers::lxc_list_all().len() as u32;
-    let vm_count = state.vms.lock().unwrap().list_vms().len() as u32;
-    let has_docker = containers::docker_status().installed;
-    let has_lxc = containers::lxc_status().installed;
-    let has_kvm = containers::kvm_installed();
     let public_ip = state.cluster.get_node(&state.cluster.self_id).and_then(|n| n.public_ip);
     let msg = AgentMessage::StatusReport {
         node_id: state.cluster.self_id.clone(),
@@ -12279,7 +12290,10 @@ pub async fn scan_issues(
 ) -> HttpResponse {
     if let Err(resp) = require_auth(&req, &state) { return resp; }
 
-    let metrics = state.monitor.lock().unwrap().collect();
+    let st = state.clone().into_inner();
+    let metrics = tokio::task::spawn_blocking(move || {
+        st.monitor.lock().unwrap().collect()
+    }).await.unwrap();
     let issues = collect_issues(&metrics);
 
     HttpResponse::Ok().json(serde_json::json!({

src/main.rs

@@ -253,6 +253,7 @@ async fn main() -> std::io::Result<()> {
         containers::lxc_autostart_all();
         networking::apply_all_wireguard_bridges();
         kubernetes::apply_all_wolfnet_routes();
+        plugins::start_all_backends();
     });
 
     // Check if TLS will be available (so the frontend knows the correct protocol for URLs)
@@ -331,27 +332,27 @@ async fn main() -> std::io::Result<()> {
         tokio::spawn(async move {
             loop {
                 tokio::time::sleep(Duration::from_secs(2)).await;
-                let (metrics, components) = {
-                    let mut monitor = state_clone.monitor.lock().unwrap();
-                    let m = monitor.collect();
-                    let c = installer::get_all_status_cached();
-                    (m, c)
-                };
+                // Run all blocking sysinfo/subprocess work off the async runtime
+                let sc = state_clone.clone();
+                let (metrics, components, docker_count, lxc_count, vm_count, has_docker, has_lxc, has_kvm) =
+                    tokio::task::spawn_blocking(move || {
+                        let mut monitor = sc.monitor.lock().unwrap();
+                        let m = monitor.collect();
+                        drop(monitor);  // release mutex before spawning subprocesses
+                        let c = installer::get_all_status_cached();
+                        let dc = containers::docker_count();
+                        let lc = containers::lxc_count();
+                        let vc = sc.vms.lock().unwrap().list_vms().len() as u32;
+                        let hd = containers::has_docker_cached();
+                        let hl = containers::has_lxc_cached();
+                        let hk = containers::has_kvm_cached();
+                        (m, c, dc, lc, vc, hd, hl, hk)
+                    }).await.unwrap();
                 // Record historical snapshot
                 {
                     let mut history = state_clone.metrics_history.lock().unwrap();
                     history.push(&metrics);
                 }
-                // Use lightweight counts (1 subprocess each) instead of full listing
-                // (which spawns 3+ subprocesses per container for docker inspect)
-                let docker_count = containers::docker_count();
-                let lxc_count = containers::lxc_count();
-                let vm_count = state_clone.vms.lock().unwrap().list_vms().len() as u32;
-                // Use cached runtime detection (TTL 120s) instead of spawning
-                // 'which', 'docker info', etc. on every 2-second cycle
-                let has_docker = containers::has_docker_cached();
-                let has_lxc = containers::has_lxc_cached();
-                let has_kvm = containers::has_kvm_cached();
 
                 // Cache the agent status report for instant polling responses
                 let self_id = cluster_clone.self_id.clone();
@@ -508,7 +509,10 @@ async fn main() -> std::io::Result<()> {
                         let mut all_issues: Vec<(String, String, api::Issue)> = Vec::new();
 
                         // Local node
-                        let metrics = scan_state.monitor.lock().unwrap().collect();
+                        let ss = scan_state.clone();
+                        let metrics = tokio::task::spawn_blocking(move || {
+                            ss.monitor.lock().unwrap().collect()
+                        }).await.unwrap();
                         let local_hostname = metrics.hostname.clone();
                         let local_cluster = {
                             let nodes = scan_cluster.get_all_nodes();
@@ -1103,24 +1107,26 @@ a{color:#dc2626;text-decoration:none;}a:hover{text-decoration:underline;}
 
                 if is_configured {
                     // Collect local metrics (sync — release mutex before any .await)
+                    let ai_sc = ai_state.clone();
                     let (hostname, cpu_pct, mem_used_gb, mem_total_gb, disk_used_gb, disk_total_gb,
-                         docker_count, lxc_count, vm_count, uptime_secs) = {
-                        let mut monitor = ai_state.monitor.lock().unwrap();
-                        let m = monitor.collect();
-                        let docker_count = containers::docker_count();
-                        let lxc_count = containers::lxc_count();
-                        let vm_count = ai_state.vms.lock().unwrap().list_vms().len() as u32;
-
-                        let mem_used = m.memory_used_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
-                        let mem_total = m.memory_total_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
-                        let root_disk = m.disks.iter().find(|d| d.mount_point == "/").or_else(|| m.disks.first());
-                        let disk_used = root_disk.map(|d| d.used_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).unwrap_or(0.0);
-                        let disk_total = root_disk.map(|d| d.total_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).unwrap_or(0.0);
-
-                        (m.hostname.clone(), m.cpu_usage_percent, mem_used, mem_total,
-                         disk_used, disk_total, docker_count, lxc_count, vm_count, m.uptime_secs)
-                    };
-                    // MutexGuard is now dropped — safe to .await below
+                         docker_count, lxc_count, vm_count, uptime_secs) =
+                        tokio::task::spawn_blocking(move || {
+                            let mut monitor = ai_sc.monitor.lock().unwrap();
+                            let m = monitor.collect();
+                            drop(monitor);
+                            let docker_count = containers::docker_count();
+                            let lxc_count = containers::lxc_count();
+                            let vm_count = ai_sc.vms.lock().unwrap().list_vms().len() as u32;
+
+                            let mem_used = m.memory_used_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
+                            let mem_total = m.memory_total_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
+                            let root_disk = m.disks.iter().find(|d| d.mount_point == "/").or_else(|| m.disks.first());
+                            let disk_used = root_disk.map(|d| d.used_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).unwrap_or(0.0);
+                            let disk_total = root_disk.map(|d| d.total_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).unwrap_or(0.0);
+
+                            (m.hostname.clone(), m.cpu_usage_percent, mem_used, mem_total,
+                             disk_used, disk_total, docker_count, lxc_count, vm_count, m.uptime_secs)
+                        }).await.unwrap();
 
                     // Collect per-guest CPU stats from Proxmox nodes in the cluster
                     let pve_nodes: Vec<_> = ai_state.cluster.get_all_nodes().into_iter()

src/plugins/mod.rs

@@ -179,6 +179,8 @@ fn load_manifest(path: &std::path::Path) -> Result<PluginManifest, String> {
 pub fn reload() {
     let mut plugins = PLUGINS.write().unwrap();
     *plugins = scan_plugins();
+    drop(plugins);
+    start_all_backends();
 }
 
 /// Get all loaded plugins
@@ -264,8 +266,22 @@ pub fn set_enabled(id: &str, enabled: bool) -> Result<String, String> {
     Ok(format!("Plugin '{}' {}", id, if enabled { "enabled" } else { "disabled" }))
 }
 
+/// Start all enabled plugin backends (called on startup and after reload)
+pub fn start_all_backends() {
+    let plugins = PLUGINS.read().unwrap().clone();
+    for plugin in &plugins {
+        if plugin.status == "active" && plugin.manifest.has_backend {
+            if let Err(e) = start_backend(&plugin.manifest.id) {
+                tracing::warn!("Failed to start plugin '{}' backend: {}", plugin.manifest.id, e);
+            } else if plugin.manifest.api_port.is_some() {
+                tracing::info!("Started plugin '{}' backend on port {}",
+                    plugin.manifest.id, plugin.manifest.api_port.unwrap());
+            }
+        }
+    }
+}
+
 /// Start a plugin's backend binary (if it has one)
-#[allow(dead_code)]
 pub fn start_backend(id: &str) -> Result<(), String> {
     let plugin = get(id).ok_or_else(|| format!("Plugin '{}' not found", id))?;
     if !plugin.manifest.has_backend { return Ok(()); }

web/index.html

@@ -20,8 +20,9 @@
     <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"
         integrity="sha256-20nQCchB9co0qIjJZRGuk2/Z9VM+kNiyxNV1lvTlZBo=" crossorigin=""></script>
     <script src="/js/vendor/three.min.js?v=15.10.7a"></script>
-    <link rel="stylesheet" href="/css/style.css?v=16.11.0">
+    <link rel="stylesheet" id="main-css" href="/css/style.css?v=16.11.0">
     <link rel="stylesheet" href="/css/xterm.min.css?v=16.11.0">
+    <script>document.getElementById('main-css').href='/css/style.css?v='+Date.now();</script>
     <script src="/js/vendor/xterm.min.js?v=15.10.7a"></script>
     <script src="/js/vendor/xterm-addon-fit.min.js?v=15.10.7a"></script>
     <style>