v24.55.2: cluster-join secret reconciliation + alert-toggle enforcement + backup folder fixes

Cluster join (closes "added a node but its containers don't show"):
- The admin+token-authenticated join-handshake now mints a single-use,
  TLS-only, 120s bootstrap grant; the master presents it on the secret push,
  so a node holding its OWN secret can adopt the fleet secret right after
  joining. Without it the push failed inter-node auth, the node stayed
  divergent, and its resources never appeared. Grant checked ONLY when normal
  cluster-auth fails (existing pushes byte-identical), single-use, length-
  capped, audited; never emitted as null. (Two security reviews; findings fixed.)

Alerts (wabil — honor the toggles, end-to-end):
- Settings→Alerts event toggles are now ENFORCED: predictive::threshold and
  predictive::container_memory honour alert_cpu / alert_memory / alert_disk /
  alert_containers (default-true → existing installs unchanged; unchecking now
  actually suppresses and existing Inbox findings auto-resolve).
- The alerts save handler was silently dropping alert_containers +
  container_memory_threshold, so unchecking "Container memory alert" reverted to
  on after Save — now persisted.

Backups (wabil):
- System-folder backup filenames now include a short hash of the FULL source
  path, so two folders that differ only by case ("temp" vs "Temp") — or share a
  basename under different parents — no longer collide and overwrite each other
  on a case-insensitive backup destination (SMB/exFAT/APFS).
- PBS file-level that can't apply to a target (Proxmox-LXC block-storage rootfs,
  VMs, config) now records an explicit reason in the backup's log + details
  instead of silently falling back to a tarball.

Golden Rule: all new option fields default to prior behavior; the join grant is
additive (existing secret pushes unchanged). 1311 tests pass.

Author: Paul C

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "wolfstack"
-version = "24.55.1"
+version = "24.55.2"
 edition = "2024"
 authors = ["Wolf Software Systems Ltd"]
 description = "Server management platform for the Wolf software suite"

Cargo.toml

@@ -243,6 +243,12 @@ pub struct AppState {
     /// In-memory, 15-minute TTL, single-use. The static per-install
     /// `join_token` above still works alongside these for backward compat.
     pub one_time_join_tokens: Arc<crate::cluster_join::OneTimeTokens>,
+    /// Single-use bootstrap grants minted by THIS node's join-handshake on
+    /// success. The joining master presents one back on /api/cluster/secret/
+    /// receive, letting a node that currently holds a DIFFERENT secret adopt
+    /// the fleet secret right after an admin-authenticated join (closes the
+    /// "added a node but its containers don't show" gap). Consumed on use.
+    pub bootstrap_grants: Arc<crate::cluster_join::OneTimeTokens>,
     pub pbs_restore_progress: std::sync::Mutex<PbsRestoreProgress>,
     pub ai_agent: Arc<crate::ai::AiAgent>,
     /// Pre-built agent status response, updated every 2s by background task
@@ -3327,12 +3333,26 @@ pub async fn cluster_join_handshake(
     // via `take`, and was reinserted on every failure path above — so a
     // success here means it is correctly single-use with no consume race.
 
-    HttpResponse::Ok().json(serde_json::json!({
+    // Mint a single-use bootstrap grant. The master presents it on the
+    // immediately-following /api/cluster/secret/receive push so the fleet
+    // secret can be adopted even though we currently hold a DIFFERENT secret
+    // (otherwise that push fails inter-node auth and we stay divergent — our
+    // resources then never appear to the cluster). Best-effort: if minting
+    // fails (CSPRNG), the master falls back to the legacy secret/default auth.
+    let bootstrap_grant = state.bootstrap_grants.mint().ok();
+
+    let mut resp_body = serde_json::json!({
         "valid": true,
         "hostname": hostname::get().map(|h| h.to_string_lossy().to_string()).unwrap_or_default(),
         "cluster_fingerprint": crate::cluster_join::cluster_fingerprint(),
         "cluster_name": our_cluster,
-    }))
+    });
+    // Only include the grant when minting succeeded — never emit a null field
+    // (which would needlessly reveal a CSPRNG failure).
+    if let Some(g) = bootstrap_grant {
+        resp_body["bootstrap_grant"] = serde_json::json!(g);
+    }
+    HttpResponse::Ok().json(resp_body)
 }
 
 /// Append a cluster-join / secret-change entry to BOTH the persistent
@@ -3728,7 +3748,44 @@ fn has_no_other_peers(state: &AppState) -> bool {
 /// configured: a peerless node is fresh (case 1); a node with peers
 /// is active (case 2).
 pub async fn cluster_secret_receive(req: HttpRequest, state: web::Data<AppState>, body: web::Json<serde_json::Value>) -> HttpResponse {
-    if let Err(resp) = require_cluster_auth(&req, &state) { return resp; }
+    // Auth: the normal inter-node secret, OR a single-use bootstrap grant this
+    // node minted in its own join-handshake moments ago. The grant path lets a
+    // node that currently holds a DIFFERENT secret adopt the fleet secret right
+    // after an admin-authenticated join (without it, this push fails inter-node
+    // auth, the node stays divergent, and its resources never appear to the
+    // cluster). The grant is checked ONLY when the normal auth fails — so an
+    // existing valid push is byte-identical and never consumes a grant — and it
+    // is single-use (consumed here).
+    let mut authed_via_grant = false;
+    match require_cluster_auth(&req, &state) {
+        Ok(_) => {}
+        Err(resp) => {
+            let grant_ok = body.get("bootstrap_grant")
+                .and_then(|v| v.as_str())
+                // Real grants are 64 hex chars; cap the input so an attacker
+                // can't feed enormous strings into the constant-time scan.
+                .filter(|g| g.len() <= 128)
+                .map(|g| state.bootstrap_grants.take(g).is_some())
+                .unwrap_or(false);
+            if !grant_ok { return resp; }
+            authed_via_grant = true;
+        }
+    }
+    if authed_via_grant {
+        // A secret adopted via the grant path = a freshly-joined node taking
+        // the fleet secret. Audit it (with source IP) so grant-path use is
+        // visible in the trail, distinct from a normal cluster-auth push.
+        let peer = req.peer_addr()
+            .map(|a| a.ip().to_canonical().to_string())
+            .unwrap_or_default();
+        cluster_join_audit(
+            &state, "cluster-node", "warning",
+            "Cluster secret adopted via bootstrap grant",
+            &format!("New cluster secret accepted via a single-use bootstrap grant from {}",
+                     if peer.is_empty() { "unknown source" } else { &peer }),
+            &peer,
+        );
+    }
     let secret = match body.get("secret").and_then(|v| v.as_str()) {
         Some(s) if !s.is_empty() => s,
         _ => return HttpResponse::BadRequest().json(serde_json::json!({ "error": "Missing secret" })),
@@ -3980,6 +4037,7 @@ pub async fn add_node(req: HttpRequest, state: web::Data<AppState>, body: web::J
     let mut last_error = String::new();
     let mut verified = false;
     let mut cluster_fingerprint: Option<String> = None;
+    let mut bootstrap_grant: Option<String> = None;
     for url in &urls {
         match client.post(url)
             .timeout(std::time::Duration::from_secs(8))
@@ -3997,6 +4055,10 @@ pub async fn add_node(req: HttpRequest, state: web::Data<AppState>, body: web::J
                     if data.get("valid").and_then(|v| v.as_bool()) == Some(true) {
                         cluster_fingerprint = data.get("cluster_fingerprint")
                             .and_then(|v| v.as_str()).map(|s| s.to_string());
+                        // Grant authorising the secret push below (lets a node
+                        // with its own secret adopt the fleet secret).
+                        bootstrap_grant = data.get("bootstrap_grant")
+                            .and_then(|v| v.as_str()).map(|s| s.to_string());
                         verified = true;
                         break;
                     } else {
@@ -4142,12 +4204,25 @@ pub async fn add_node(req: HttpRequest, state: web::Data<AppState>, body: web::J
         let addr = body.address.clone();
         let secret = active_secret;
         let our_secret = state.cluster_secret.clone();
+        // Single-use grant from the handshake — lets the receiver adopt the
+        // fleet secret even if it currently holds its own (the X-WolfStack-
+        // Secret attempts below would otherwise be rejected, leaving it
+        // divergent and its resources invisible to the cluster).
+        let grant = bootstrap_grant.clone();
         tokio::spawn(async move {
             let client = &*API_HTTP_CLIENT;
             let urls = build_node_urls(&addr, port, "/api/cluster/secret/receive");
             for auth_value in [our_secret.as_str(), crate::auth::default_cluster_secret()] {
                 let mut pushed = false;
                 for url in &urls {
+                    // Only attach the bootstrap grant on the TLS (https) leg —
+                    // never on the plaintext-http fallback. Capturing the grant
+                    // then requires breaking TLS, at which point the secret in
+                    // the same body is already exposed, so the grant adds no
+                    // marginal on-wire risk. (build_node_urls puts https first;
+                    // the http legs are the WolfNet-overlay and last-resort
+                    // plaintext fallbacks.)
+                    let leg_grant = if url.starts_with("https://") { grant.clone() } else { None };
                     // `bootstrap: true` tells the receiving node "this is a
                     // first-time join; safe to self-restart so the new
                     // secret takes effect in-memory immediately". Old-code
@@ -4159,6 +4234,7 @@ pub async fn add_node(req: HttpRequest, state: web::Data<AppState>, body: web::J
                         .json(&serde_json::json!({
                             "secret": secret,
                             "bootstrap": true,
+                            "bootstrap_grant": leg_grant,
                         }))
                         .send()
                         .await
@@ -26608,6 +26684,13 @@ pub async fn alerts_config_save(req: HttpRequest, state: web::Data<AppState>, bo
     if let Some(b) = v.get("alert_cpu").and_then(|v| v.as_bool()) { config.alert_cpu = b; }
     if let Some(b) = v.get("alert_memory").and_then(|v| v.as_bool()) { config.alert_memory = b; }
     if let Some(b) = v.get("alert_disk").and_then(|v| v.as_bool()) { config.alert_disk = b; }
+    // Container monitoring toggle + threshold — these were MISSING here, so
+    // unchecking "Container memory alert" never persisted and reverted to on
+    // after Save (wabil 2026-06-21).
+    if let Some(b) = v.get("alert_containers").and_then(|v| v.as_bool()) { config.alert_containers = b; }
+    if let Some(t) = v.get("container_memory_threshold").and_then(|v| v.as_f64()) {
+        config.container_memory_threshold = t as f32;
+    }
     if let Some(i) = v.get("check_interval_secs").and_then(|v| v.as_u64()) {
         // Clamp to sensible range: 30 seconds to 1 hour
         config.check_interval_secs = i.max(30).min(3600);

src/api/mod.rs

@@ -364,6 +364,39 @@ impl BackupStorage {
 mod tests {
     use super::*;
 
+    #[test]
+    fn path_discriminator_distinguishes_case_and_parent() {
+        // The whole point: case-only-differing paths must get DIFFERENT
+        // discriminators so they don't collide on a case-insensitive dest.
+        assert_ne!(short_path_discriminator("/data/temp"),
+                   short_path_discriminator("/data/Temp"));
+        // Same basename under different parents must differ too.
+        assert_ne!(short_path_discriminator("/a/x"),
+                   short_path_discriminator("/b/x"));
+        // Stable + hex (8 lowercase hex chars).
+        let d = short_path_discriminator("/data/temp");
+        assert_eq!(d, short_path_discriminator("/data/temp"));
+        assert_eq!(d.len(), 8);
+        assert!(d.chars().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()));
+    }
+
+    #[test]
+    fn pbs_file_level_skip_note_only_for_inapplicable_pbs() {
+        let mut pbs = BackupStorage { storage_type: StorageType::Pbs, pbs_file_level: true, ..Default::default() };
+        let vm = BackupTarget { target_type: BackupTargetType::Vm, ..Default::default() };
+        let docker = BackupTarget { target_type: BackupTargetType::Docker, ..Default::default() };
+        // VM can't do file-level → a note.
+        assert!(pbs_file_level_skip_note(&vm, &pbs).is_some());
+        // Docker WILL use pxar → no note.
+        assert!(pbs_file_level_skip_note(&docker, &pbs).is_none());
+        // file-level off → never a note, even for VM.
+        pbs.pbs_file_level = false;
+        assert!(pbs_file_level_skip_note(&vm, &pbs).is_none());
+        // non-PBS storage → never a note.
+        let local = BackupStorage { storage_type: StorageType::Local, pbs_file_level: true, ..Default::default() };
+        assert!(pbs_file_level_skip_note(&vm, &local).is_none());
+    }
+
     fn wd(path: &str, sub: &str) -> BackupStorage {
         BackupStorage {
             storage_type: StorageType::Wolfdisk,
@@ -1288,6 +1321,25 @@ fn sanitize_archive_name(s: &str) -> String {
         .collect()
 }
 
+/// Short, case-insensitive-safe discriminator derived from the FULL source
+/// path: the first 8 hex chars of SHA-256(path). Two paths that differ only by
+/// case (or share a basename under different parents) get DIFFERENT
+/// discriminators, so their backup filenames never collide — even on a
+/// case-insensitive destination filesystem. Hex is itself case-insensitive,
+/// so the discriminator can't re-introduce a case collision.
+fn short_path_discriminator(path: &str) -> String {
+    use sha2::{Digest, Sha256};
+    let mut hasher = Sha256::new();
+    hasher.update(path.as_bytes());
+    let digest = hasher.finalize();
+    let mut out = String::with_capacity(8);
+    use std::fmt::Write;
+    for b in digest.iter().take(4) {
+        let _ = write!(out, "{:02x}", b);
+    }
+    out
+}
+
 /// tar.gz a directory's contents (NOT the directory itself) into the
 /// given archive path. Returns the resulting archive size in bytes.
 fn tar_dir_to_gz(src_dir: &str, archive: &Path) -> Result<u64, String> {
@@ -1996,7 +2048,15 @@ pub fn backup_system_path(label: &str, path: &str, exclude_mounts: &[String]) ->
     let safe_label = sanitize_archive_name(if label.trim().is_empty() {
         Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or("folder")
     } else { label.trim() });
-    let filename = format!("systempath-{}-{}.tar.gz", safe_label, timestamp);
+    // Disambiguator derived from the FULL source path. Without it, two folders
+    // whose names differ only by case ("temp" vs "Temp") — or that share a
+    // basename under different parents ("/a/x" vs "/b/x") — collide on a
+    // case-insensitive backup destination (SMB / exFAT / APFS), so one
+    // overwrites the other (wabil 2026-06-21). A short hex hash of the exact
+    // path is case-insensitive-safe and unique per source, while the readable
+    // label is preserved.
+    let path_disc = short_path_discriminator(path);
+    let filename = format!("systempath-{}-{}-{}.tar.gz", safe_label, path_disc, timestamp);
     let tar_path = staging.join(&filename);
 
     let src = path.trim_end_matches('/');
@@ -2242,7 +2302,11 @@ fn create_backup_entry(target: BackupTarget, storage: &BackupStorage) -> BackupE
     let id = Uuid::new_v4().to_string();
     let now = Utc::now().to_rfc3339();
     let hostname = local_hostname();
-    let comments = backup_comments(&target);
+    let mut comments = backup_comments(&target);
+    // Make a file-level→tarball fallback (e.g. Proxmox LXC → vzdump) visible.
+    if let Some(note) = pbs_file_level_skip_note(&target, storage) {
+        comments = format!("{} | {}", comments, note);
+    }
     let cluster = local_cluster_name();
 
     // PBS file-level (pxar) path — see create_backup_with_log for the rationale.
@@ -2970,6 +3034,27 @@ fn pbs_file_level_applies(target: &BackupTarget) -> bool {
     }
 }
 
+/// When PBS file-level is ENABLED but a target can't use it, return a plain-
+/// English reason. Makes the tarball/vzdump fallback visibly intentional in the
+/// backup's details + live log instead of looking like "file-level is broken"
+/// (wabil 2026-06-21: "file level not implemented … lxc is tar.zst"). Returns
+/// None when file-level isn't requested or the target WILL use pxar.
+fn pbs_file_level_skip_note(target: &BackupTarget, storage: &BackupStorage) -> Option<String> {
+    if storage.storage_type != StorageType::Pbs || !storage.pbs_file_level || pbs_file_level_applies(target) {
+        return None;
+    }
+    let why = match target.target_type {
+        BackupTargetType::Lxc =>
+            "Proxmox LXC rootfs is on block storage (ZFS/LVM) — pxar file-level isn't possible, using vzdump image",
+        BackupTargetType::Vm =>
+            "VMs back up as a disk image, not per-file — pxar file-level doesn't apply",
+        BackupTargetType::Config =>
+            "config backups are a single archive — pxar file-level doesn't apply",
+        _ => "pxar file-level isn't available for this target — using image/tarball backup",
+    };
+    Some(format!("PBS file-level not applicable: {}", why))
+}
+
 /// Build the pxar source pairs for a file-level PBS backup of `target`.
 /// Returns (backup_type, backup_id, pairs). For Docker the container's
 /// filesystem is materialised into a staging tree via `docker export`;
@@ -5086,7 +5171,13 @@ pub fn create_backup_with_log(
         let _ = log.send(format!("[{}/{}] Starting {} backup: {}",
             i + 1, total, type_name, display_name));
 
-        let comments = backup_comments_with_cluster(t, &cluster);
+        let mut comments = backup_comments_with_cluster(t, &cluster);
+        // Surface why file-level fell back (e.g. Proxmox LXC → vzdump) so it's
+        // not mistaken for a broken feature.
+        if let Some(note) = pbs_file_level_skip_note(t, &storage) {
+            let _ = log.send(format!("  {}", note));
+            comments = format!("{} | {}", comments, note);
+        }
         let hostname = local_hostname();
 
         // PBS file-level (pxar) path — upload the workload's content directory

src/backup/mod.rs

@@ -64,21 +64,37 @@ struct PendingToken {
     expires: Instant,
 }
 
-/// In-memory store of pending one-time join tokens. Kept in-memory only:
-/// these are deliberately ephemeral (15-minute TTL) and a process restart
-/// invalidating them is the safe failure mode — an operator simply mints a
-/// fresh one. Keyed by the token string itself for O(1) verify+consume.
-#[derive(Default)]
+/// In-memory store of single-use, TTL-bounded tokens. Kept in-memory only:
+/// these are deliberately ephemeral and a process restart invalidating them is
+/// the safe failure mode — an operator simply mints a fresh one. Keyed by the
+/// token string itself for O(1) verify+consume. The TTL is per-store: join
+/// tokens use 15 min (`new()`); bootstrap grants use a much shorter window
+/// (`with_ttl`) since they're consumed seconds after minting.
 pub struct OneTimeTokens {
     inner: Mutex<HashMap<String, PendingToken>>,
+    ttl: Duration,
+}
+
+impl Default for OneTimeTokens {
+    fn default() -> Self {
+        Self::with_ttl(ONE_TIME_TTL)
+    }
 }
 
 impl OneTimeTokens {
     pub fn new() -> Self {
         Self::default()
     }
 
-    /// Mint a new one-time token, store it with a 15-minute expiry, and
+    /// Store with a custom TTL (e.g. a short window for bootstrap grants).
+    pub fn with_ttl(ttl: Duration) -> Self {
+        Self {
+            inner: Mutex::new(HashMap::new()),
+            ttl,
+        }
+    }
+
+    /// Mint a new one-time token, store it with this store's TTL, and
     /// return the plaintext (shown to the operator ONCE). Also prunes any
     /// already-expired entries so the map can't grow without bound.
     /// Returns `Err` if the system CSPRNG is unavailable — we refuse to
@@ -94,7 +110,7 @@ impl OneTimeTokens {
         guard.insert(
             token.clone(),
             PendingToken {
-                expires: Instant::now() + ONE_TIME_TTL,
+                expires: Instant::now() + self.ttl,
             },
         );
         Ok(token)

src/cluster_join.rs

@@ -696,6 +696,11 @@ async fn main() -> std::io::Result<()> {
             cluster_secret: cluster_secret.clone(),
             join_token: api::load_join_token(),
             one_time_join_tokens: Arc::new(cluster_join::OneTimeTokens::new()),
+            // Bootstrap grants are consumed seconds after minting (handshake →
+            // immediate secret push), so a tight 120s TTL minimises the window.
+            bootstrap_grants: Arc::new(cluster_join::OneTimeTokens::with_ttl(
+                std::time::Duration::from_secs(120),
+            )),
             pbs_restore_progress: Mutex::new(Default::default()),
             ai_agent: ai_agent.clone(),
             cached_status: cached_status.clone(),

src/main.rs

@@ -130,6 +130,13 @@ pub fn analyze(
     acks: &AckStore,
     proposals: &crate::predictive::proposal::ProposalStore,
 ) -> Vec<Proposal> {
+    // Respect the Settings → Alerts "Container memory alert" toggle. Default
+    // true (existing behaviour unchanged); when off, emit nothing — existing
+    // findings auto-resolve since covered_scopes still reports their scopes
+    // (wabil 2026-06-21).
+    if !crate::alerting::AlertConfig::load().alert_containers {
+        return Vec::new();
+    }
     let mut out = Vec::new();
     for fact in current {
         let scope = ProposalScope {