test(kms): validate Vault circuit recovery

* fix(scanner): isolate corrupt cycle state

* fix(scanner): preserve newer state during recovery reset

* fix(scanner): fence recovery reset state

* fix(scanner): reject terminal recovery epochs

* fix(scanner): reject trailing cycle state bytes

* fix(scanner): reject terminal leadership epochs

* fix(scanner): retain recovery wake notifications

* fix(scanner): recover from oversized markers

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

crates/scanner/src/data_usage_define.rs

@@ -125,6 +125,34 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
     }
 }
 
+/// Read only the object revision without materializing its body.
+pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
+    match store
+        .get_object_reader(
+            RUSTFS_META_BUCKET,
+            path,
+            None,
+            HeaderMap::new(),
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(reader) => reader
+            .object_info
+            .etag
+            .filter(|etag| !etag.is_empty())
+            .map(DataUsageCacheRevision::Etag)
+            .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
+        Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
+            Ok(DataUsageCacheRevision::Missing)
+        }
+        Err(err) => Err(err),
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct DataUsageCacheRevisions {
     main: DataUsageCacheRevision,
@@ -146,6 +174,11 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
 pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
 
+/// Durable companion object for a cycle-state object which cannot be decoded.
+/// The primary object is deliberately never replaced or deleted by recovery.
+pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock<String> =
+    LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str()));
+
 pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
 

crates/scanner/src/data_usage_define/persistence.rs

@@ -74,7 +74,7 @@ impl DataUsageCache {
         let loaded = Self::load_cache(store.clone(), name).await?;
         let backup = match loaded.backup_revision {
             Some(revision) => Some(revision),
-            None => match Self::revision_for_path(store, &backup_path).await {
+            None => match read_config_revision(store, &backup_path).await {
                 Ok(revision) => Some(revision),
                 Err(err) => {
                     counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
@@ -336,33 +336,6 @@ impl DataUsageCache {
         }
     }
 
-    async fn revision_for_path<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
-        match store
-            .get_object_reader(
-                RUSTFS_META_BUCKET,
-                path,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    no_lock: true,
-                    ..Default::default()
-                },
-            )
-            .await
-        {
-            Ok(reader) => reader
-                .object_info
-                .etag
-                .filter(|etag| !etag.is_empty())
-                .map(DataUsageCacheRevision::Etag)
-                .ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))),
-            Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
-                Ok(DataUsageCacheRevision::Missing)
-            }
-            Err(err) => Err(err),
-        }
-    }
-
     pub(super) fn cache_save_timeout() -> Duration {
         crate::runtime_config::scanner_cache_save_timeout()
     }

crates/scanner/src/lib.rs

@@ -75,7 +75,10 @@ pub use remote_scanner::{
 };
 pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
 pub use rustfs_common::last_minute;
-pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
+pub use scanner::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
+    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
+};
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
     record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,

crates/scanner/src/scanner.rs

@@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock};
 
 use crate::data_usage_define::{
     BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
-    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision,
+    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
 };
 use crate::runtime_config::{
     ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
@@ -54,9 +54,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
 use rustfs_data_usage::observed_data_usage_is_newer;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
-#[cfg(test)]
-use tokio::sync::Notify;
-use tokio::sync::mpsc;
+use tokio::sync::{Notify, mpsc};
 use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tokio_util::task::AbortOnDropHandle;
@@ -104,6 +102,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
 /// unavailable peer cannot drive a tight retry loop.
 const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
 const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
+/// A transient backend outage remains self-healing after the short retry
+/// budget is exhausted, but the probe is intentionally sparse until storage
+/// recovers or an operator reset wakes the scanner.
+const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
+/// Permanent recovery states still get a sparse status probe so a reset that
+/// races the wait registration cannot leave the scanner asleep forever.
+const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
 const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
 #[cfg(not(test))]
 const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
@@ -125,6 +130,12 @@ type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
 static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
     LazyLock::new(|| StdMutex::new(None));
 
+static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
+
+pub(super) fn notify_scanner_cycle_recovery_wake() {
+    SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
+}
+
 #[cfg(test)]
 struct ScannerCycleStatePersistTestHookGuard;
 
@@ -576,19 +587,21 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
             tokio::time::sleep(sleep_time).await;
         }
 
+        let mut transient_backoff = ScannerRetryBackoff::default();
+        let mut recovery_retry_count = 0_u32;
         loop {
             if ctx_clone.is_cancelled() {
                 break;
             }
 
-            if let Err(e) = run_data_scanner_with_maintenance_state(
+            let run_result = run_data_scanner_with_maintenance_state(
                 ctx_clone.clone(),
                 storeapi_clone.clone(),
                 startup_features,
                 startup_maintenance_generation,
             )
-            .await
-            {
+            .await;
+            if let Err(e) = &run_result {
                 error!(
                     target: "rustfs::scanner",
                     event = EVENT_SCANNER_CYCLE_STATE,
@@ -599,11 +612,52 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
                     "Scanner runtime iteration failed"
                 );
             }
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.retryable {
+                recovery_retry_count = recovery_retry_count.saturating_add(1);
+                let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
+            } else {
+                recovery_retry_count = 0;
+            }
+
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.state == "paused" {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
+                }
+                recovery_retry_count = 0;
+                continue;
+            }
+            if !recovery_status.retryable
+                && matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
+            {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
+                }
+                continue;
+            }
+
+            let retry_delay = if recovery_status.retryable || run_result.is_err() {
+                transient_backoff.record_retryable_cycle(true);
+                transient_backoff
+                    .retry_interval(scanner_cycle_interval())
+                    .unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
+            } else {
+                transient_backoff.record_retryable_cycle(false);
+                randomized_cycle_delay()
+            };
             // Backoff before retrying after lock contention or scanner-level failures.
             // Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
             tokio::select! {
                 _ = ctx_clone.cancelled() => break,
-                _ = tokio::time::sleep(randomized_cycle_delay()) => {}
+                _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                _ = tokio::time::sleep(retry_delay) => {}
             }
         }
     });
@@ -1606,40 +1660,22 @@ async fn run_data_scanner_with_maintenance_state(
         observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
     }
 
-    let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await {
-        Ok((buf, revision)) => (buf.unwrap_or_default(), revision),
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
-                state = "revision_load_failed",
-                error = %err,
-                "Scanner cycle state revision load failed"
-            );
-            global_metrics().set_cycle(None).await;
-            return Ok(());
-        }
-    };
-    let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) {
-        Ok(state) => state,
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
-                state = "cycle_decode_failed",
-                error = %err,
-                "Scanner stopped because persisted cycle state is invalid"
-            );
-            global_metrics().set_cycle(None).await;
-            return Ok(());
-        }
-    };
+    let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
+        match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
+            ScannerCycleStateStartup::Ready {
+                cycle,
+                leader_epoch,
+                revision,
+            } => (cycle, leader_epoch, revision),
+            ScannerCycleStateStartup::Blocked => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleStateStartup::Transient(err) => {
+                global_metrics().set_cycle(None).await;
+                return Err(err);
+            }
+        };
     let usage_floor = match persisted_usage_floor(storeapi.clone()).await {
         Ok(floor) => floor,
         Err(err) => {
@@ -2219,7 +2255,12 @@ pub(crate) use activity::{
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
-pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence};
+pub use cycle_state::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
+};
+pub(crate) use cycle_state::{
+    current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
+};
 pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
 pub use usage_store::store_data_usage_in_backend;
 

crates/scanner/src/scanner/leadership.rs

@@ -196,7 +196,7 @@ pub(super) async fn claim_scanner_leadership(
         if ctx.is_cancelled() {
             return false;
         }
-        let Some(claimed_epoch) = persisted_epoch.checked_add(1) else {
+        let Some(claimed_epoch) = persisted_epoch.checked_add(1).filter(|epoch| *epoch < u64::MAX) else {
             error!(
                 target: "rustfs::scanner",
                 event = EVENT_SCANNER_PERSIST_STATE,

crates/scanner/src/scanner/tests.rs

@@ -15,11 +15,12 @@
 use super::*;
 use crate::EcstoreResult;
 use crate::{
-    Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
-    ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
-    init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
+    DATA_USAGE_BLOOM_RECOVERY_PATH, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
+    ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
+    ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
+    init_local_disks_with_instance_ctx,
 };
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::Cursor;
 use std::task::Poll;
 use temp_env::{with_var, with_var_unset};
@@ -117,6 +118,15 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
     assert!(cycle_ctx.is_cancelled());
 }
 
+#[tokio::test]
+async fn scanner_cycle_recovery_wake_survives_wait_registration_race() {
+    notify_scanner_cycle_recovery_wake();
+
+    tokio::time::timeout(Duration::from_secs(1), SCANNER_CYCLE_RECOVERY_WAKE.notified())
+        .await
+        .expect("recovery wake should retain a permit until the waiter registers");
+}
+
 struct ScannerDefaultSpeedGuard;
 
 impl ScannerDefaultSpeedGuard {
@@ -151,6 +161,7 @@ impl Drop for ScannerDefaultCycleGuard {
 struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
+    non_regular_objects: Mutex<HashSet<String>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
     object_not_found_put_number: Mutex<HashMap<String, usize>>,
     error_after_commit_put_number: Mutex<HashMap<String, usize>>,
@@ -191,12 +202,16 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
             .get(&key)
             .cloned()
             .ok_or(EcstoreError::FileNotFound)?;
-        let revision = *self.revisions.lock().await.entry(key).or_insert(1);
+        let data_len = i64::try_from(data.len()).expect("memory test object length should fit in i64");
+        let revision = *self.revisions.lock().await.entry(key.clone()).or_insert(1);
+        let is_dir = self.non_regular_objects.lock().await.contains(&key);
 
         Ok(GetObjectReader {
             stream: Box::new(Cursor::new(data)),
             object_info: ObjectInfo {
                 etag: Some(format!("memory-{revision}")),
+                size: data_len,
+                is_dir,
                 ..Default::default()
             },
             buffered_body: None,
@@ -797,6 +812,10 @@ fn scanner_cycle_state_decodes_legacy_and_fenced_formats() {
     let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode");
     assert_eq!(fenced_cycle.next, 13);
     assert_eq!(fenced_epoch, 7);
+
+    let mut trailing = fenced;
+    trailing.push(0);
+    assert!(decode_scanner_cycle_state(&trailing).is_err());
 }
 
 #[test]
@@ -823,6 +842,840 @@ fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() {
     assert!(encode_scanner_cycle_state(&exhausted, 7).is_err());
 }
 
+#[tokio::test]
+async fn corrupt_cycle_state_is_quarantined_once() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key.clone(), 7);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let marker_data = store
+        .objects
+        .lock()
+        .await
+        .get(&marker_key)
+        .cloned()
+        .expect("corrupt state must leave a durable recovery marker");
+    let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should be valid JSON");
+    assert_eq!(marker.primary_revision, "memory-7");
+    assert_eq!(marker.path, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    assert_eq!(marker.quarantine_path, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    assert_eq!(marker.classification, "corrupt");
+
+    // A second startup sees the matching marker before consuming the poison body.
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+
+    // Replacing the primary object advances its revision; the stale marker must
+    // not quarantine the newer, valid state.
+    let cycle = CurrentCycle {
+        next: 9,
+        ..Default::default()
+    };
+    let encoded = encode_scanner_cycle_state(&cycle, 3).expect("valid state should encode");
+    store.objects.lock().await.insert(state_key.clone(), encoded);
+    store.revisions.lock().await.insert(state_key, 8);
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Ready {
+            cycle: CurrentCycle { next: 9, .. },
+            leader_epoch: 3,
+            ..
+        }
+    ));
+}
+
+#[tokio::test]
+async fn empty_cycle_state_object_is_quarantined_as_corrupt() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), Vec::new());
+    store.revisions.lock().await.insert(state_key, 6);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt"));
+    assert!(
+        scanner_cycle_recovery_status()
+            .reason
+            .as_deref()
+            .is_some_and(|reason| reason.contains("empty"))
+    );
+}
+
+#[tokio::test]
+async fn future_cycle_state_schema_is_recovery_required() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let mut future = 17_u64.to_le_bytes().to_vec();
+    future.extend_from_slice(b"RSCYC999");
+    future.extend_from_slice(&4_u64.to_le_bytes());
+    future.extend_from_slice(&[0x90]);
+    store.objects.lock().await.insert(state_key.clone(), future);
+    store.revisions.lock().await.insert(state_key, 13);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("future_schema"));
+}
+
+#[tokio::test]
+async fn concurrent_leaders_cannot_quarantine_newer_cycle_state() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key, 4);
+
+    let (first, second) = tokio::join!(
+        load_scanner_cycle_state_for_startup(store.clone()),
+        load_scanner_cycle_state_for_startup(store.clone()),
+    );
+    assert!(matches!(first, ScannerCycleStateStartup::Blocked));
+    assert!(matches!(second, ScannerCycleStateStartup::Blocked));
+
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let marker_data = store
+        .objects
+        .lock()
+        .await
+        .get(&marker_key)
+        .cloned()
+        .expect("one contender must publish the recovery marker");
+    let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should decode");
+    assert_eq!(marker.primary_revision, "memory-4");
+}
+
+#[tokio::test]
+async fn cleanup_pending_marker_blocks_a_rewritten_primary_after_restart() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let encoded = encode_scanner_cycle_state(
+        &CurrentCycle {
+            next: 12,
+            ..Default::default()
+        },
+        8,
+    )
+    .expect("valid state should encode");
+    store.objects.lock().await.insert(state_key.clone(), encoded);
+    store.revisions.lock().await.insert(state_key, 22);
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-21".to_string(),
+        generation: 11,
+        leader_epoch: 7,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    store
+        .objects
+        .lock()
+        .await
+        .insert(marker_key.clone(), serde_json::to_vec(&marker).expect("marker should encode"));
+    store.revisions.lock().await.insert(marker_key, 3);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().state, "cleanup-pending");
+}
+
+#[test]
+fn full_rescan_reset_accepts_unknown_marker_fields_without_trusting_cursor() {
+    let marker = br#"{
+        "schema_version": 99,
+        "primary_revision": "memory-7",
+        "generation": 9000,
+        "leader_epoch": 9000,
+        "classification": "new-future-classification",
+        "first_detected_at_unix_secs": 1,
+        "last_attempt_at_unix_secs": 2,
+        "retry_count": 9,
+        "reason": "future marker",
+        "path": "buckets/.bloomcycle.bin",
+        "quarantine_path": "buckets/.bloomcycle.bin.recovery-required.json",
+        "future_field": {"cursor": "untrusted"}
+    }"#;
+    let decoded =
+        super::cycle_state::decode_recovery_marker_for_reset(marker, &DataUsageCacheRevision::Etag("memory-3".to_string()))
+            .expect("full-rescan compatibility decoder should accept additive fields");
+    assert_eq!(decoded.primary_revision, "memory-7");
+    assert_eq!(decoded.classification, "future_schema");
+    assert_eq!(decoded.generation, 0);
+    assert_eq!(decoded.leader_epoch, 0);
+    assert_eq!(decoded.state, "blocked");
+
+    let malformed =
+        super::cycle_state::decode_recovery_marker_for_reset(b"{not-json", &DataUsageCacheRevision::Etag("memory-4".to_string()))
+            .expect("a full-rescan reset must recover even when the marker is malformed");
+    assert!(malformed.primary_revision.is_empty());
+    assert_eq!(malformed.classification, "future_schema");
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_after_malformed_marker_without_trusting_cursor() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover malformed marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0, "reset must use the verified usage floor, not marker cursor");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_ignores_epoch_from_malformed_future_primary() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let mut future_primary = vec![0; 24];
+    future_primary[8..16].copy_from_slice(b"RSCY9999");
+    future_primary[16..24].copy_from_slice(&u64::MAX.to_le_bytes());
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), future_primary)
+        .await
+        .expect("future cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover malformed future state");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1, "invalid persisted bytes must not raise the recovery epoch");
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn ecstore_exact_recovery_marker_delete_honors_etag() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v1".to_vec())
+        .await
+        .expect("initial recovery marker should be persisted");
+    let (_, stale_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("initial marker revision should load");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v2".to_vec())
+        .await
+        .expect("replacement recovery marker should be persisted");
+
+    let delete_result = store
+        .delete_config_object(
+            RUSTFS_META_BUCKET,
+            DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+            ObjectOptions {
+                http_preconditions: Some(stale_revision.preconditions()),
+                ..Default::default()
+            },
+        )
+        .await;
+    assert!(matches!(delete_result, Err(EcstoreError::PreconditionFailed)));
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("replacement marker should remain durable"),
+        b"marker-v2"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_corrupt_primary_under_stale_blocked_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let corrupt_primary = vec![0xff, 0x00, 0x01];
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), corrupt_primary.clone())
+        .await
+        .expect("corrupt cycle state should be persisted");
+    let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary revision should load");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-stale".to_string(),
+        generation: 1,
+        leader_epoch: 1,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "blocked primary changed".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "blocked".to_string(),
+    };
+    let marker_data = serde_json::to_vec(&marker).expect("blocked marker should encode");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), marker_data.clone())
+        .await
+        .expect("blocked marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "a strict marker must fail closed when its primary revision changed"
+    );
+    assert_eq!(
+        read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+            .await
+            .expect("primary should remain readable"),
+        corrupt_primary
+    );
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("blocked marker should remain durable"),
+        marker_data
+    );
+    assert!(!matches!(primary_revision, DataUsageCacheRevision::Missing));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    let old_primary_data = encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode");
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), old_primary_data.clone())
+        .await
+        .expect("valid cycle state should be persisted");
+    let (_, old_primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary state revision should load");
+    let old_usage = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(41),
+        ..Default::default()
+    };
+    let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
+    save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
+        .await
+        .expect("usage snapshot should be persisted");
+    let (_, old_usage_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage snapshot revision should load");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("reset should clear a stale malformed marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("valid primary should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("primary cycle state should decode");
+    assert_eq!(cycle.next, 42, "reset must not regress an independently fenced primary");
+    assert_eq!(leader_epoch, 8, "reset must advance the preserved primary epoch");
+    let stale_primary_save = save_config_with_preconditions(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        old_primary_data,
+        old_primary_revision.preconditions(),
+    )
+    .await;
+    assert!(matches!(stale_primary_save, Err(EcstoreError::PreconditionFailed)));
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage epoch fence should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<DataUsageInfo>(&usage)
+            .expect("fenced usage should decode")
+            .scanner_epoch,
+        Some(8)
+    );
+    let stale_save = save_config_with_preconditions(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        old_usage_data,
+        old_usage_revision.preconditions(),
+    )
+    .await;
+    assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed)));
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let completed_at = Utc::now();
+    let primary = CurrentCycle {
+        current: 3,
+        next: 42,
+        cycle_completed: vec![completed_at],
+        started: completed_at,
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    let usage = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(41),
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+    )
+    .await
+    .expect("usage snapshot should be persisted");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-old".to_string(),
+        generation: 41,
+        leader_epoch: 7,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("marker should encode"),
+    )
+    .await
+    .expect("cleanup marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("reset should resume a cleanup-pending preserved primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("preserved cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("cycle state should decode");
+    assert_eq!(cycle.current, 3, "cleanup retry must preserve the in-progress cursor");
+    assert_eq!(cycle.next, 42);
+    assert_eq!(cycle.cycle_completed, vec![completed_at]);
+    assert_eq!(cycle.started, completed_at);
+    assert_eq!(leader_epoch, 8);
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage epoch fence should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<DataUsageInfo>(&usage)
+            .expect("usage should decode")
+            .scanner_epoch,
+        Some(8)
+    );
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_oversized_regular_primary_with_malformed_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1])
+        .await
+        .expect("oversized cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("explicit full-rescan reset should replace an oversized regular primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_oversized_primary_after_cleanup_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1])
+        .await
+        .expect("oversized cycle state should be persisted");
+    let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary revision should load");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: match primary_revision {
+            DataUsageCacheRevision::Etag(etag) => etag,
+            DataUsageCacheRevision::Missing => panic!("primary revision should be present"),
+        },
+        generation: 1,
+        leader_epoch: 1,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("cleanup marker should encode"),
+    )
+    .await
+    .expect("cleanup marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("cleanup retry should rebuild an oversized primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_with_oversized_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), vec![b'x'; 64 * 1024 + 1])
+        .await
+        .expect("oversized recovery marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover an oversized marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_with_empty_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), Vec::new())
+        .await
+        .expect("empty recovery marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover an empty marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_keeps_cleanup_marker_when_preserved_epoch_is_exhausted() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, u64::MAX).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err()
+    );
+
+    let marker = read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("cleanup marker should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker)
+            .expect("cleanup marker should decode")
+            .state,
+        "cleanup-pending"
+    );
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_preserved_epoch_that_would_be_terminal() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, u64::MAX - 1).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "reset must not persist the terminal leader epoch"
+    );
+
+    let marker = read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("cleanup marker should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker)
+            .expect("cleanup marker should decode")
+            .state,
+        "cleanup-pending"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&DataUsageInfo {
+            scanner_epoch: Some(u64::MAX - 1),
+            ..Default::default()
+        })
+        .expect("usage floor should encode"),
+    )
+    .await
+    .expect("usage floor should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "reset must not persist the terminal leader epoch"
+    );
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("recovery marker should remain durable"),
+        b"{not-json"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_empty_primary_with_malformed_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), Vec::new())
+        .await
+        .expect("empty cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("explicit full-rescan reset should replace an empty primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_when_primary_cycle_state_is_missing() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-missing".to_string(),
+        generation: u64::MAX,
+        leader_epoch: u64::MAX,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 0,
+        reason: "missing primary".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "blocked".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("marker should encode"),
+    )
+    .await
+    .expect("marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recreate missing primary");
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("missing primary should be rebuilt");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn corrupt_cycle_state_rename_or_marker_failure_stays_recovery_required() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key, 9);
+    store.fail_put_number.lock().await.insert(marker_key, 1);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Transient(_)
+    ));
+    let status = scanner_cycle_recovery_status();
+    assert_eq!(status.state, "recovery-required");
+    assert!(status.retryable);
+    assert!(
+        store
+            .objects
+            .lock()
+            .await
+            .contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()))
+    );
+}
+
+#[tokio::test]
+async fn oversized_or_symlinked_cycle_state_is_rejected() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(key.clone(), vec![0; 1024 * 1024 + 1]);
+    store.revisions.lock().await.insert(key.clone(), 11);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt"));
+    assert!(
+        scanner_cycle_recovery_status()
+            .reason
+            .as_deref()
+            .is_some_and(|reason| reason.contains("oversized"))
+    );
+
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    store.objects.lock().await.remove(&marker_key);
+    store.objects.lock().await.insert(key.clone(), vec![1]);
+    store.revisions.lock().await.insert(key.clone(), 12);
+    store.non_regular_objects.lock().await.insert(key);
+    // The object contract exposes a non-regular object as `is_dir`; local
+    // backends reject symlink/reparse entries before they become an object.
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+}
+
 #[tokio::test]
 async fn scanner_startup_uses_primary_and_backup_usage_floor() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -855,6 +1708,31 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
     assert_eq!(epoch, 11);
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&DataUsageInfo {
+                scanner_epoch: Some(epoch),
+                scanner_cycle: Some(cycle),
+                ..Default::default()
+            })
+            .expect("usage snapshot should encode"),
+        );
+    }
+
+    assert_eq!(
+        persisted_usage_floor(store).await.expect("usage floor should load"),
+        PersistedUsageFloor {
+            next_cycle: 101,
+            leader_epoch: 8,
+        }
+    );
+}
+
 #[test]
 fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
     let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
@@ -987,6 +1865,15 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
 
     assert!(persisted_usage_floor(store.clone()).await.is_err());
 
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        br#"{}"#.to_vec(),
+    );
+    assert!(
+        persisted_usage_floor(store.clone()).await.is_err(),
+        "a structurally incomplete usage snapshot must not be treated as an empty floor"
+    );
+
     store.objects.lock().await.insert(
         memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
         serde_json::to_vec(&DataUsageInfo {
@@ -1244,6 +2131,22 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
     assert_eq!(store.put_counts.lock().await.get(&key), Some(&3));
 }
 
+#[tokio::test]
+async fn test_leadership_claim_rejects_terminal_epoch() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let ctx = CancellationToken::new();
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle {
+        next: 12,
+        ..Default::default()
+    };
+    let mut persisted_epoch = u64::MAX - 1;
+
+    assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
+    assert_eq!(persisted_epoch, u64::MAX - 1);
+    assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
+}
+
 #[tokio::test]
 async fn test_leadership_claim_confirms_commit_after_returned_error() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2975,6 +3878,24 @@ fn superseded_retry_backoff_grows_from_the_default_cycle() {
     }
 }
 
+#[tokio::test(start_paused = true)]
+async fn corrupt_cycle_state_backoff_uses_virtual_clock() {
+    let mut backoff = ScannerRetryBackoff::default();
+    backoff.record_retryable_cycle(true);
+    let first_delay = backoff
+        .retry_interval(Duration::from_secs(60))
+        .expect("the first recovery retry should be scheduled");
+    assert_eq!(first_delay, Duration::from_secs(5));
+
+    let deadline = Instant::now() + first_delay;
+    assert!(Instant::now() < deadline);
+    tokio::time::advance(first_delay).await;
+    assert!(Instant::now() >= deadline);
+
+    backoff.record_retryable_cycle(true);
+    assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(10)));
+}
+
 #[test]
 fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
     let runtime_config = ScannerRuntimeConfig {

protocol/agent/v1/fixtures/fixture-sets.json

@@ -25,7 +25,7 @@
     },
     {
       "name": "heartbeat",
-      "status": "populated",
+      "status": "reserved",
       "purpose": "Heartbeat payloads, Connect receive time, and freshness window behavior."
     },
     {

protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256

@@ -1,5 +0,0 @@
-975c1ca53eefeef6766a6fc0b3d3281f7408255342b0686e5e2aee5ad055414c  duplicate.json
-963529a38a02849c6c2acc6d72668dca9f63218b49c89fae41a451b584850411  overflow.json
-e3adeee1c8a19aa17e70894896fb79c072e3785bea3611b93c11e79f039ed5af  stale.json
-35b9cebd8525389a701e8fe69fbe96407bcb31aa28392fe95babf4a4886985ad  unknown.json
-37941735dbd6ad3d238258a7b2cae6f0b3aa0ecaae1d8817817c3d718d11d633  valid.json

protocol/agent/v1/fixtures/heartbeat/duplicate.json

@@ -1,9 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "heartbeat",
-  "fixture": "duplicate",
-  "description": "An exact requestId replay returns the first result and creates no second heartbeat.",
-  "first": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
-  "replay": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
-  "expected": {"decision": "DUPLICATE", "heartbeatWrites": 1, "events": 1, "sameResponse": true}
-}

protocol/agent/v1/fixtures/heartbeat/overflow.json

@@ -1,11 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "heartbeat",
-  "fixture": "overflow",
-  "description": "Values beyond frozen bounds are rejected before persistence.",
-  "vectors": [
-    {"field": "sequence", "value": 9007199254740992, "maximum": 9007199254740991},
-    {"field": "coarseNodeSummary.total", "value": 4097, "maximum": 4096}
-  ],
-  "expected": {"decision": "REJECT", "httpStatus": 422, "status": "INVALID_ARGUMENT"}
-}

protocol/agent/v1/fixtures/heartbeat/stale.json

@@ -1,9 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "heartbeat",
-  "fixture": "stale",
-  "description": "A lower heartbeat sequence is retained as history and cannot replace the current projection.",
-  "head": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
-  "late": {"requestId": "7c4d2e10-9f83-4a5b-b6c7-d8e9f0a1b2c3", "sequence": 9},
-  "expected": {"decision": "ACCEPT_HISTORY", "currentSequence": 42, "historySequence": 9}
-}

protocol/agent/v1/fixtures/heartbeat/unknown.json

@@ -1,18 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "heartbeat",
-  "fixture": "unknown",
-  "description": "Unknown optional members and capabilities are accepted, discarded before hashing, and never stored or echoed.",
-  "requestAdditions": {
-    "telemetryProfile": "extended",
-    "authorization": "Bearer non-functional-example",
-    "capabilities": ["heartbeat", "future.capability"],
-    "coarseNodeSummary": {"rackNames": ["customer-rack"]}
-  },
-  "expected": {
-    "decision": "ACCEPT",
-    "storedCapabilities": ["heartbeat"],
-    "discarded": ["authorization", "future.capability", "telemetryProfile", "coarseNodeSummary.rackNames"],
-    "echoed": []
-  }
-}

protocol/agent/v1/fixtures/heartbeat/valid.json

@@ -1,21 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "heartbeat",
-  "fixture": "valid",
-  "description": "A bounded L0 heartbeat. clientTime is advisory; Connect's receivedAt is online authority.",
-  "request": {
-    "protocolVersion": "v1",
-    "requestId": "550e8400-e29b-41d4-a716-446655440000",
-    "agentVersion": "rustfs-agent/1.19.4",
-    "capabilities": ["heartbeat", "inventory"],
-    "sequence": 42,
-    "clientTime": "2026-08-22T01:02:03Z",
-    "coarseNodeSummary": {"total": 8, "healthy": 7, "degraded": 1}
-  },
-  "expected": {
-    "decision": "ACCEPT",
-    "acceptedVersion": "v1",
-    "responseFields": ["serverTime", "acceptedVersion", "capabilityHints"],
-    "onlineAuthority": "serverTime"
-  }
-}

rustfs/src/admin/handlers/mod.rs

@@ -126,6 +126,7 @@ mod tests {
         let _list_remote_target_handler = replication::ListRemoteTargetHandler {};
         let _remove_remote_target_handler = replication::RemoveRemoteTargetHandler {};
         let _scanner_status_handler = scanner::ScannerStatusHandler {};
+        let _scanner_cycle_state_reset_handler = scanner::ScannerCycleStateResetHandler {};
         let _ilm_expiry_status_handler = scanner::IlmExpiryStatusHandler {};
         let _manual_transition_handler = ilm_transition::ManualTransitionRunHandler {};
         let _manual_transition_status_handler = ilm_transition::ManualTransitionJobStatusHandler {};

rustfs/src/admin/handlers/scanner.rs

@@ -13,8 +13,11 @@
 // limitations under the License.
 
 use crate::admin::auth::authorize_admin_request;
+use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
-use crate::admin::runtime_sources::current_scanner_metrics_report;
+use crate::admin::runtime_sources::{
+    app_context_from_req, current_object_store_handle_for_context, current_scanner_metrics_report,
+};
 use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env};
 use crate::server::ADMIN_PREFIX;
 use chrono::Utc;
@@ -22,11 +25,13 @@ use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
 use matchit::Params;
 use rustfs_common::metrics::{ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport};
+use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_credentials::Credentials;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use s3s::header::CONTENT_TYPE;
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use tokio_util::sync::CancellationToken;
 
 const JSON_CONTENT_TYPE: &str = "application/json";
 
@@ -38,6 +43,13 @@ struct ScannerStatusResponse {
     metrics: ScannerMetricsReport,
     cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus,
     runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
+    cycle_recovery: rustfs_scanner::ScannerCycleRecoveryStatus,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ScannerCycleResetRequest {
+    mode: String,
 }
 
 #[derive(Debug, Serialize)]
@@ -117,6 +129,7 @@ fn scanner_status_response(
         metrics,
         cycle_schedule,
         runtime_config,
+        cycle_recovery: rustfs_scanner::scanner::scanner_cycle_recovery_status(),
     }
 }
 
@@ -144,6 +157,11 @@ pub fn register_scanner_route(r: &mut S3Router<AdminOperation>) -> std::io::Resu
         format!("{ADMIN_PREFIX}/v3/scanner/status").as_str(),
         AdminOperation(&ScannerStatusHandler {}),
     )?;
+    r.insert(
+        Method::POST,
+        format!("{ADMIN_PREFIX}/v3/scanner/cycle-state/reset").as_str(),
+        AdminOperation(&ScannerCycleStateResetHandler {}),
+    )?;
     r.insert(
         Method::GET,
         format!("{ADMIN_PREFIX}/v3/ilm/expiry/status").as_str(),
@@ -163,6 +181,13 @@ async fn validate_scanner_status_request(req: &S3Request<Body>) -> S3Result<Cred
     authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await
 }
 
+async fn validate_scanner_reset_request(req: &S3Request<Body>) -> S3Result<Credentials> {
+    if req.credentials.is_none() {
+        return Err(s3_error!(InvalidRequest, "missing credentials"));
+    }
+    authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)]).await
+}
+
 fn json_response(body: Vec<u8>) -> S3Result<S3Response<(StatusCode, Body)>> {
     let mut headers = HeaderMap::new();
     let content_type = HeaderValue::from_str(JSON_CONTENT_TYPE)
@@ -192,6 +217,37 @@ impl Operation for ScannerStatusHandler {
 
 pub struct IlmExpiryStatusHandler {}
 
+pub struct ScannerCycleStateResetHandler {}
+
+#[async_trait::async_trait]
+impl Operation for ScannerCycleStateResetHandler {
+    async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let _cred = validate_scanner_reset_request(&req).await?;
+        let body = req
+            .input
+            .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
+            .await
+            .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?;
+        let reset = serde_json::from_slice::<ScannerCycleResetRequest>(&body)
+            .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?;
+        if reset.mode != "full-rescan" {
+            return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "reset mode must be full-rescan"));
+        }
+        let context = app_context_from_req(&req)
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?;
+        let store = current_object_store_handle_for_context(Some(context.as_ref()))
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?;
+        supervise_admin_mutation("scanner cycle state reset", async move {
+            rustfs_scanner::scanner::reset_scanner_cycle_recovery(CancellationToken::new(), store)
+                .await
+                .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
+            Ok::<_, S3Error>(())
+        })
+        .await?;
+        json_response(br#"{"status":"reset","mode":"full-rescan"}"#.to_vec())
+    }
+}
+
 #[async_trait::async_trait]
 impl Operation for IlmExpiryStatusHandler {
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
@@ -237,6 +293,38 @@ mod tests {
         assert_eq!(err.message(), Some("missing credentials"));
     }
 
+    #[tokio::test]
+    async fn scanner_reset_gate_rejects_missing_credentials() {
+        let req = S3Request {
+            input: Body::from(String::new()),
+            method: Method::POST,
+            uri: http::Uri::from_static("/rustfs/admin/v3/scanner/cycle-state/reset"),
+            headers: HeaderMap::new(),
+            extensions: http::Extensions::new(),
+            credentials: None,
+            region: None,
+            service: None,
+            trailing_headers: None,
+        };
+
+        let err = validate_scanner_reset_request(&req)
+            .await
+            .expect_err("a reset request without credentials must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("missing credentials"));
+    }
+
+    #[test]
+    fn admin_reset_requires_full_rescan_or_verified_cursor() {
+        let full_rescan: ScannerCycleResetRequest =
+            serde_json::from_str(r#"{"mode":"full-rescan"}"#).expect("full rescan must be accepted");
+        assert_eq!(full_rescan.mode, "full-rescan");
+        let cursor: ScannerCycleResetRequest =
+            serde_json::from_str(r#"{"mode":"cursor"}"#).expect("mode validation belongs to the handler");
+        assert_ne!(cursor.mode, "full-rescan");
+        assert!(serde_json::from_str::<ScannerCycleResetRequest>(r#"{"mode":"full-rescan","cursor":"untrusted"}"#).is_err());
+    }
+
     #[test]
     fn scanner_disabled_reason_reports_startup_env_key() {
         assert_eq!(scanner_disabled_reason(true), None);
@@ -304,6 +392,11 @@ mod tests {
         assert_eq!(encoded["cycle_schedule"]["effective_interval_seconds"], 0);
         assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_enabled"], false);
         assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_multiplier"], 1);
+        assert_eq!(encoded["cycle_recovery"]["state"], "healthy");
+        assert_eq!(
+            encoded["cycle_recovery"]["quarantine_path"],
+            rustfs_scanner::DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()
+        );
     }
 
     #[test]

rustfs/src/admin/route_policy.rs

@@ -428,6 +428,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
     admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High),
     admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High),
     admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive),
+    admin(
+        HttpMethod::Post,
+        "/rustfs/admin/v3/scanner/cycle-state/reset",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/v3/ilm/expiry/status",
@@ -2020,6 +2026,12 @@ mod tests {
         assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/expiry/status", SET_TIER);
     }
 
+    #[test]
+    fn route_policy_requires_config_update_for_scanner_cycle_reset() {
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", CONFIG_UPDATE);
+        assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", SERVER_INFO);
+    }
+
     #[test]
     fn route_policy_uses_tier_actions_for_transition_routes() {
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);

rustfs/src/admin/route_registration_test.rs

@@ -243,6 +243,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/config"),
         admin_route(Method::PUT, "/v3/config"),
         admin_route(Method::GET, "/v3/scanner/status"),
+        admin_route(Method::POST, "/v3/scanner/cycle-state/reset"),
         admin_route(Method::GET, "/v3/audit/target/list"),
         admin_route_sample(
             Method::PUT,
@@ -879,6 +880,7 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/config"));
     assert_route(&router, Method::PUT, &admin_path("/v3/config"));
     assert_route(&router, Method::GET, &admin_path("/v3/scanner/status"));
+    assert_route(&router, Method::POST, &admin_path("/v3/scanner/cycle-state/reset"));
     assert_route(&router, Method::GET, &admin_path("/v3/ilm/expiry/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/run"));
     assert_route(
@@ -1367,6 +1369,7 @@ fn test_admin_alias_paths_match_existing_admin_routes() {
         (Method::GET, compat_admin_alias_path("/v3/config")),
         (Method::PUT, compat_admin_alias_path("/v3/config")),
         (Method::GET, compat_admin_alias_path("/v3/scanner/status")),
+        (Method::POST, compat_admin_alias_path("/v3/scanner/cycle-state/reset")),
         (Method::GET, compat_admin_alias_path("/v3/ilm/expiry/status")),
     ] {
         assert!(

rustfs/src/connect/config.rs

@@ -1,173 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::env;
-use std::ffi::OsString;
-use std::fs;
-use std::path::PathBuf;
-use std::time::Duration;
-
-use super::{CredentialStore, IdentityStore};
-
-pub const ENV_CONNECT_ENDPOINT: &str = "RUSTFS_CONNECT_ENDPOINT";
-pub const ENV_CONNECT_ROOT_CA_FILE: &str = "RUSTFS_CONNECT_ROOT_CA_FILE";
-pub const ENV_CONNECT_STATE_DIR: &str = "RUSTFS_CONNECT_STATE_DIR";
-
-#[derive(Clone, Copy, Debug)]
-pub struct HeartbeatSchedule {
-    pub cadence: Duration,
-    pub jitter: Duration,
-    pub timeout: Duration,
-    pub initial_backoff: Duration,
-    pub max_backoff: Duration,
-}
-
-impl Default for HeartbeatSchedule {
-    fn default() -> Self {
-        Self {
-            cadence: Duration::from_secs(30),
-            jitter: Duration::from_secs(3),
-            timeout: Duration::from_secs(5),
-            initial_backoff: Duration::from_secs(1),
-            max_backoff: Duration::from_secs(5 * 60),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct HeartbeatConfig {
-    pub endpoint: String,
-    pub root_ca_pem: Vec<u8>,
-    pub identity_store: IdentityStore,
-    pub credential_store: CredentialStore,
-    pub state_path: PathBuf,
-    pub schedule: HeartbeatSchedule,
-}
-
-impl HeartbeatConfig {
-    pub fn new(
-        endpoint: impl Into<String>,
-        root_ca_pem: impl Into<Vec<u8>>,
-        identity_store: IdentityStore,
-        credential_store: CredentialStore,
-        state_path: impl Into<PathBuf>,
-    ) -> Self {
-        Self {
-            endpoint: endpoint.into(),
-            root_ca_pem: root_ca_pem.into(),
-            identity_store,
-            credential_store,
-            state_path: state_path.into(),
-            schedule: HeartbeatSchedule::default(),
-        }
-    }
-
-    pub fn from_env() -> Result<Option<Self>, HeartbeatConfigError> {
-        Self::from_env_values(
-            env::var_os(ENV_CONNECT_ENDPOINT),
-            env::var_os(ENV_CONNECT_ROOT_CA_FILE),
-            env::var_os(ENV_CONNECT_STATE_DIR),
-        )
-    }
-
-    fn from_env_values(
-        endpoint: Option<OsString>,
-        root_ca_file: Option<OsString>,
-        state_dir: Option<OsString>,
-    ) -> Result<Option<Self>, HeartbeatConfigError> {
-        let configured = endpoint.is_some() || root_ca_file.is_some() || state_dir.is_some();
-        if !configured {
-            return Ok(None);
-        }
-        let (Some(endpoint), Some(root_ca_file), Some(state_dir)) = (endpoint, root_ca_file, state_dir) else {
-            return Err(HeartbeatConfigError::Partial);
-        };
-        let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?;
-        let root_ca_file = PathBuf::from(root_ca_file);
-        let state_dir = PathBuf::from(state_dir);
-        if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() || state_dir.as_os_str().is_empty() {
-            return Err(HeartbeatConfigError::Partial);
-        }
-        let root_ca_pem = fs::read(&root_ca_file).map_err(|source| HeartbeatConfigError::RootCertificate {
-            path: root_ca_file,
-            source,
-        })?;
-        Ok(Some(Self::new(
-            endpoint,
-            root_ca_pem,
-            IdentityStore::new(state_dir.join("identity")),
-            CredentialStore::new(state_dir.join("credential")),
-            state_dir.join("heartbeat/state.json"),
-        )))
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum HeartbeatConfigError {
-    #[error(
-        "Connect heartbeat configuration requires RUSTFS_CONNECT_ENDPOINT, RUSTFS_CONNECT_ROOT_CA_FILE, and RUSTFS_CONNECT_STATE_DIR"
-    )]
-    Partial,
-    #[error("RUSTFS_CONNECT_ENDPOINT is not valid UTF-8")]
-    EndpointEncoding,
-    #[error("failed to read the Connect root CA at {path}: {source}")]
-    RootCertificate {
-        path: PathBuf,
-        #[source]
-        source: std::io::Error,
-    },
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{HeartbeatConfig, HeartbeatConfigError};
-    use std::ffi::OsString;
-
-    #[test]
-    fn absent_environment_is_disabled_without_side_effects() {
-        assert!(
-            HeartbeatConfig::from_env_values(None, None, None)
-                .expect("absent config")
-                .is_none()
-        );
-    }
-
-    #[test]
-    fn partial_environment_is_rejected() {
-        assert!(matches!(
-            HeartbeatConfig::from_env_values(Some(OsString::from("https://connect.example/agent/")), None, None),
-            Err(HeartbeatConfigError::Partial)
-        ));
-    }
-
-    #[test]
-    fn complete_environment_builds_the_durable_paths() {
-        let temp = tempfile::tempdir().expect("tempdir");
-        let root = temp.path().join("root.pem");
-        std::fs::write(&root, b"root certificate").expect("root CA");
-        let state = temp.path().join("state");
-        let config = HeartbeatConfig::from_env_values(
-            Some(OsString::from("https://connect.example/agent/")),
-            Some(root.into_os_string()),
-            Some(state.clone().into_os_string()),
-        )
-        .expect("complete config")
-        .expect("enabled config");
-
-        assert_eq!(config.endpoint, "https://connect.example/agent/");
-        assert_eq!(config.root_ca_pem, b"root certificate");
-        assert_eq!(config.state_path, state.join("heartbeat/state.json"));
-        assert!(!state.exists(), "parsing configuration must not create state");
-    }
-}

rustfs/src/connect/heartbeat.rs

@@ -1,585 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fs;
-use std::io::{self, Write as _};
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::Duration;
-
-use chrono::{DateTime, SecondsFormat, Utc};
-use reqwest::{Client, StatusCode, Url, header};
-use rustls::RootCertStore;
-use rustls::pki_types::{CertificateDer, pem::PemObject as _};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-use zeroize::Zeroizing;
-
-use super::config::HeartbeatConfig;
-use super::credential_store::{CredentialStoreError, DeviceCredential};
-use super::identity::IdentityError;
-use super::identity_store::StoreError;
-use super::registration::{CredentialValidationError, validate_stored_credential};
-
-const PROTOCOL_VERSION: &str = "v1";
-const AGENT_VERSION: &str = concat!("rustfs-agent/", env!("CARGO_PKG_VERSION"));
-const MAX_SEQUENCE: u64 = 9_007_199_254_740_991;
-const MAX_RESPONSE_BYTES: usize = 64 * 1024;
-#[cfg(unix)]
-const FILE_MODE: u32 = 0o600;
-static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0);
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields, rename_all = "camelCase")]
-pub struct CoarseNodeSummary {
-    total: u16,
-    healthy: u16,
-    degraded: u16,
-}
-
-impl CoarseNodeSummary {
-    pub fn new(total: u16, healthy: u16, degraded: u16) -> Result<Self, HeartbeatError> {
-        let summary = Self {
-            total,
-            healthy,
-            degraded,
-        };
-        if !summary.is_valid() {
-            return Err(HeartbeatError::NodeSummary);
-        }
-        Ok(summary)
-    }
-
-    fn is_valid(&self) -> bool {
-        self.total != 0
-            && self.total <= 4096
-            && self.healthy <= 4096
-            && self.degraded <= 4096
-            && self.healthy.saturating_add(self.degraded) <= self.total
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum HeartbeatStatus {
-    Starting,
-    Online { server_time: String },
-    BackingOff { delay: Duration },
-    AuthenticationStopped { status: u16, reason: Option<String> },
-    Failed { reason: String },
-    Stopped,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields, rename_all = "camelCase")]
-pub(crate) struct PendingHeartbeat {
-    protocol_version: String,
-    request_id: String,
-    agent_version: String,
-    capabilities: [String; 1],
-    sequence: u64,
-    client_time: String,
-    coarse_node_summary: CoarseNodeSummary,
-}
-
-impl PendingHeartbeat {
-    fn is_valid(&self) -> bool {
-        self.protocol_version == PROTOCOL_VERSION
-            && self.agent_version == AGENT_VERSION
-            && self.capabilities[0] == "heartbeat"
-            && self.sequence <= MAX_SEQUENCE
-            && self.coarse_node_summary.is_valid()
-            && is_exact_utc_seconds(&self.client_time)
-            && Uuid::parse_str(&self.request_id)
-                .is_ok_and(|request_id| request_id.get_version_num() == 4 && request_id.to_string() == self.request_id)
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct HeartbeatResponse {
-    server_time: String,
-    accepted_version: String,
-    #[serde(default)]
-    capability_hints: Vec<String>,
-}
-
-pub(crate) enum Delivery {
-    Accepted { server_time: String },
-    Retry { retry_after: Option<Duration> },
-    AuthenticationStopped { status: u16, reason: Option<String> },
-    Rejected { status: u16, reason: Option<String> },
-}
-
-pub(crate) struct HeartbeatSender {
-    endpoint: Url,
-    root_store: RootCertStore,
-    roots: Vec<CertificateDer<'static>>,
-    config: HeartbeatConfig,
-}
-
-impl HeartbeatSender {
-    pub(crate) fn new(config: HeartbeatConfig) -> Result<Self, HeartbeatError> {
-        let mut endpoint = Url::parse(&config.endpoint).map_err(|_| HeartbeatError::Endpoint)?;
-        if endpoint.scheme() != "https"
-            || endpoint.cannot_be_a_base()
-            || !endpoint.username().is_empty()
-            || endpoint.password().is_some()
-            || endpoint.query().is_some()
-            || endpoint.fragment().is_some()
-        {
-            return Err(HeartbeatError::Endpoint);
-        }
-        if !endpoint.path().ends_with('/') {
-            endpoint.set_path(&format!("{}/", endpoint.path()));
-        }
-        let roots = CertificateDer::pem_slice_iter(&config.root_ca_pem)
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|_| HeartbeatError::RootCertificate)?;
-        if roots.is_empty() {
-            return Err(HeartbeatError::RootCertificate);
-        }
-        let mut root_store = RootCertStore::empty();
-        let (accepted, rejected) = root_store.add_parsable_certificates(roots.clone());
-        if accepted != roots.len() || rejected != 0 {
-            return Err(HeartbeatError::RootCertificate);
-        }
-        let schedule = config.schedule;
-        if schedule.cadence.is_zero()
-            || schedule.timeout.is_zero()
-            || schedule.timeout > Duration::from_secs(5)
-            || schedule.initial_backoff.is_zero()
-            || schedule.max_backoff < schedule.initial_backoff
-            || schedule.max_backoff > Duration::from_secs(5 * 60)
-            || schedule.jitter > schedule.cadence
-        {
-            return Err(HeartbeatError::Schedule);
-        }
-        Ok(Self {
-            endpoint,
-            root_store,
-            roots,
-            config,
-        })
-    }
-
-    pub(crate) async fn send(&self, heartbeat: &PendingHeartbeat) -> Result<Delivery, HeartbeatError> {
-        let (cluster_uid, client) = {
-            let _lock = self.config.credential_store.lock().await?;
-            let credential = self.config.credential_store.load()?.ok_or(HeartbeatError::NotRegistered)?;
-            let identity = self.config.identity_store.load()?.ok_or(HeartbeatError::IdentityMissing)?;
-            validate_stored_credential(&credential, &identity, &self.root_store, &self.roots)?;
-            let now = Utc::now().timestamp();
-            if now < credential.not_before_unix || now >= credential.not_after_unix {
-                return Err(HeartbeatError::CredentialExpired);
-            }
-            let cluster_uid = cluster_uid(&credential)?.to_owned();
-            let client = self.client(&credential, &identity.to_pkcs8_pem()?)?;
-            (cluster_uid, client)
-        };
-        let url = self.endpoint.join(&format!("clusters/{cluster_uid}/heartbeats"))?;
-        let response = match client.post(url).json(heartbeat).send().await {
-            Ok(response) => response,
-            Err(error) if error.is_timeout() || error.is_connect() || error.is_request() => {
-                return Ok(Delivery::Retry { retry_after: None });
-            }
-            Err(error) => return Err(error.into()),
-        };
-        let status = response.status();
-        if status == StatusCode::TOO_MANY_REQUESTS {
-            return Ok(Delivery::Retry {
-                retry_after: retry_after(response.headers(), Utc::now(), self.config.schedule.max_backoff),
-            });
-        }
-        if status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() {
-            return Ok(Delivery::Retry { retry_after: None });
-        }
-        if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
-            return Ok(Delivery::AuthenticationStopped {
-                status: status.as_u16(),
-                reason: response_reason(response).await,
-            });
-        }
-        if status != StatusCode::OK {
-            return Ok(Delivery::Rejected {
-                status: status.as_u16(),
-                reason: response_reason(response).await,
-            });
-        }
-        let accepted: HeartbeatResponse =
-            serde_json::from_slice(&bounded_body(response).await?).map_err(|_| HeartbeatError::Response)?;
-        if accepted.accepted_version != PROTOCOL_VERSION
-            || accepted.capability_hints.len() > 32
-            || accepted.capability_hints.iter().any(|hint| hint.len() > 32)
-            || !is_exact_utc_seconds(&accepted.server_time)
-        {
-            return Err(HeartbeatError::Response);
-        }
-        Ok(Delivery::Accepted {
-            server_time: accepted.server_time,
-        })
-    }
-
-    fn client(&self, credential: &DeviceCredential, key: &Zeroizing<String>) -> Result<Client, HeartbeatError> {
-        let mut pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + key.len() + 1));
-        pem.extend_from_slice(credential.certificate_chain.as_bytes());
-        pem.push(b'\n');
-        pem.extend_from_slice(key.as_bytes());
-        let identity = reqwest::Identity::from_pem(&pem).map_err(|_| HeartbeatError::IdentityCertificate)?;
-        let roots = self
-            .roots
-            .iter()
-            .map(|root| reqwest::Certificate::from_der(root.as_ref()))
-            .collect::<Result<Vec<_>, _>>()?;
-        Client::builder()
-            .https_only(true)
-            .redirect(reqwest::redirect::Policy::none())
-            .timeout(self.config.schedule.timeout)
-            .tls_certs_only(roots)
-            .identity(identity)
-            .build()
-            .map_err(Into::into)
-    }
-}
-
-#[derive(Clone)]
-pub(crate) struct HeartbeatStateStore {
-    path: PathBuf,
-}
-
-#[derive(Default, Serialize, Deserialize)]
-#[serde(deny_unknown_fields, rename_all = "camelCase")]
-struct HeartbeatState {
-    next_sequence: u64,
-    pending: Option<PendingHeartbeat>,
-}
-
-impl HeartbeatStateStore {
-    pub(crate) fn new(path: PathBuf) -> Self {
-        Self { path }
-    }
-
-    pub(crate) fn try_runtime_lock(&self) -> Result<fs::File, HeartbeatError> {
-        let directory = parent(&self.path)?;
-        fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?;
-        let name = filename(&self.path)?;
-        let path = directory.join(format!(".{name}.lock"));
-        let mut options = fs::OpenOptions::new();
-        options.create(true).truncate(false).read(true).write(true);
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::OpenOptionsExt as _;
-            options.mode(FILE_MODE);
-        }
-        let lock = options.open(&path).map_err(|source| state_io(&path, source))?;
-        check_mode(&path)?;
-        lock.try_lock().map_err(|_| HeartbeatError::AlreadyRunning)?;
-        Ok(lock)
-    }
-
-    pub(crate) async fn prepare(
-        &self,
-        summary: CoarseNodeSummary,
-        now: DateTime<Utc>,
-    ) -> Result<PendingHeartbeat, HeartbeatError> {
-        let store = self.clone();
-        tokio::task::spawn_blocking(move || store.prepare_sync(summary, now))
-            .await
-            .map_err(|source| state_io(&self.path, io::Error::other(source)))?
-    }
-
-    pub(crate) async fn mark_accepted(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> {
-        let store = self.clone();
-        let accepted = accepted.clone();
-        tokio::task::spawn_blocking(move || store.mark_accepted_sync(&accepted))
-            .await
-            .map_err(|source| state_io(&self.path, io::Error::other(source)))?
-    }
-
-    fn prepare_sync(&self, summary: CoarseNodeSummary, now: DateTime<Utc>) -> Result<PendingHeartbeat, HeartbeatError> {
-        let mut state = self.read()?;
-        if let Some(pending) = state.pending {
-            return Ok(pending);
-        }
-        if state.next_sequence > MAX_SEQUENCE {
-            return Err(HeartbeatError::SequenceExhausted);
-        }
-        let pending = PendingHeartbeat {
-            protocol_version: PROTOCOL_VERSION.to_owned(),
-            request_id: Uuid::new_v4().to_string(),
-            agent_version: AGENT_VERSION.to_owned(),
-            capabilities: ["heartbeat".to_owned()],
-            sequence: state.next_sequence,
-            client_time: now.to_rfc3339_opts(SecondsFormat::Secs, true),
-            coarse_node_summary: summary,
-        };
-        state.pending = Some(pending.clone());
-        self.write(&state)?;
-        Ok(pending)
-    }
-
-    fn mark_accepted_sync(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> {
-        let mut state = self.read()?;
-        if state.pending.as_ref() != Some(accepted) {
-            return Err(HeartbeatError::StateConflict);
-        }
-        state.next_sequence = accepted.sequence.checked_add(1).ok_or(HeartbeatError::SequenceExhausted)?;
-        state.pending = None;
-        self.write(&state)
-    }
-
-    fn read(&self) -> Result<HeartbeatState, HeartbeatError> {
-        let bytes = match fs::read(&self.path) {
-            Ok(bytes) => bytes,
-            Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(HeartbeatState::default()),
-            Err(source) => return Err(state_io(&self.path, source)),
-        };
-        check_mode(&self.path)?;
-        let state: HeartbeatState = serde_json::from_slice(&bytes).map_err(|source| HeartbeatError::StateInvalid {
-            path: self.path.clone(),
-            source,
-        })?;
-        if state.next_sequence > MAX_SEQUENCE + 1
-            || state
-                .pending
-                .as_ref()
-                .is_some_and(|pending| pending.sequence != state.next_sequence || !pending.is_valid())
-        {
-            return Err(HeartbeatError::StateCorrupt { path: self.path.clone() });
-        }
-        Ok(state)
-    }
-
-    fn write(&self, state: &HeartbeatState) -> Result<(), HeartbeatError> {
-        let bytes = serde_json::to_vec(state).map_err(|source| HeartbeatError::StateInvalid {
-            path: self.path.clone(),
-            source,
-        })?;
-        let directory = parent(&self.path)?;
-        fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?;
-        let temp = stage(directory, &self.path, &bytes)?;
-        let result = fs::rename(&temp, &self.path)
-            .map_err(|source| state_io(&self.path, source))
-            .and_then(|()| fsync_dir(directory).map_err(|source| state_io(directory, source)));
-        if result.is_err() {
-            let _ = fs::remove_file(temp);
-        }
-        result
-    }
-}
-
-fn cluster_uid(credential: &DeviceCredential) -> Result<&str, HeartbeatError> {
-    let mut parts = credential.name.split('/');
-    let valid = parts.next() == Some("organizations");
-    let organization_uid = parts.next();
-    let valid = valid && parts.next() == Some("clusters");
-    let cluster_uid = parts.next();
-    let valid = valid && parts.next() == Some("clusterDevices");
-    let device_uid = parts.next();
-    if !valid
-        || organization_uid.is_none_or(str::is_empty)
-        || cluster_uid.is_none_or(str::is_empty)
-        || device_uid != Some(credential.uid.as_str())
-        || parts.next().is_some()
-    {
-        return Err(HeartbeatError::CredentialName);
-    }
-    cluster_uid.ok_or(HeartbeatError::CredentialName)
-}
-
-fn retry_after(headers: &header::HeaderMap, now: DateTime<Utc>, maximum: Duration) -> Option<Duration> {
-    let value = headers.get(header::RETRY_AFTER)?.to_str().ok()?;
-    let delay = value.parse::<u64>().ok().map(Duration::from_secs).or_else(|| {
-        DateTime::parse_from_rfc2822(value)
-            .ok()
-            .and_then(|at| (at.with_timezone(&Utc) - now).to_std().ok())
-    })?;
-    Some(delay.min(maximum))
-}
-
-fn is_exact_utc_seconds(value: &str) -> bool {
-    DateTime::parse_from_rfc3339(value).is_ok_and(|time| {
-        time.offset().local_minus_utc() == 0
-            && value.ends_with('Z')
-            && time.with_timezone(&Utc).to_rfc3339_opts(SecondsFormat::Secs, true) == value
-    })
-}
-
-async fn response_reason(response: reqwest::Response) -> Option<String> {
-    #[derive(Deserialize)]
-    struct Envelope {
-        #[serde(default)]
-        details: Vec<Detail>,
-    }
-    #[derive(Deserialize)]
-    struct Detail {
-        #[serde(default)]
-        reason: String,
-    }
-
-    serde_json::from_slice::<Envelope>(&bounded_body(response).await.ok()?)
-        .ok()?
-        .details
-        .into_iter()
-        .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason))
-}
-
-async fn bounded_body(mut response: reqwest::Response) -> Result<Vec<u8>, HeartbeatError> {
-    let mut body = Vec::new();
-    while let Some(chunk) = response.chunk().await? {
-        if body.len().saturating_add(chunk.len()) > MAX_RESPONSE_BYTES {
-            return Err(HeartbeatError::ResponseTooLarge);
-        }
-        body.extend_from_slice(&chunk);
-    }
-    Ok(body)
-}
-
-fn parent(path: &Path) -> Result<&Path, HeartbeatError> {
-    path.parent()
-        .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state path has no parent")))
-}
-
-fn filename(path: &Path) -> Result<&str, HeartbeatError> {
-    path.file_name()
-        .and_then(|name| name.to_str())
-        .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state filename is invalid")))
-}
-
-fn stage(directory: &Path, destination: &Path, bytes: &[u8]) -> Result<PathBuf, HeartbeatError> {
-    let name = filename(destination)?;
-    loop {
-        let path = directory.join(format!(
-            ".{name}.{}.{}.tmp",
-            std::process::id(),
-            STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
-        ));
-        let mut options = fs::OpenOptions::new();
-        options.write(true).create_new(true);
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::OpenOptionsExt as _;
-            options.mode(FILE_MODE);
-        }
-        let mut file = match options.open(&path) {
-            Ok(file) => file,
-            Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
-            Err(source) => return Err(state_io(&path, source)),
-        };
-        if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) {
-            let _ = fs::remove_file(&path);
-            return Err(state_io(&path, source));
-        }
-        return Ok(path);
-    }
-}
-
-fn state_io(path: &Path, source: io::Error) -> HeartbeatError {
-    HeartbeatError::StateIo {
-        path: path.to_path_buf(),
-        source,
-    }
-}
-
-#[cfg(unix)]
-fn check_mode(path: &Path) -> Result<(), HeartbeatError> {
-    use std::os::unix::fs::PermissionsExt as _;
-
-    let mode = fs::metadata(path)
-        .map_err(|source| state_io(path, source))?
-        .permissions()
-        .mode()
-        & 0o7777;
-    if mode != FILE_MODE {
-        return Err(HeartbeatError::StatePermissions {
-            path: path.to_path_buf(),
-            mode,
-            expected: FILE_MODE,
-        });
-    }
-    Ok(())
-}
-
-#[cfg(not(unix))]
-fn check_mode(_path: &Path) -> Result<(), HeartbeatError> {
-    Ok(())
-}
-
-fn fsync_dir(directory: &Path) -> io::Result<()> {
-    #[cfg(unix)]
-    fs::File::open(directory)?.sync_all()?;
-    #[cfg(not(unix))]
-    let _ = directory;
-    Ok(())
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum HeartbeatError {
-    #[error("Connect heartbeat endpoint must be an HTTPS base URL without credentials, query, or fragment")]
-    Endpoint,
-    #[error("Connect heartbeat root CA configuration is invalid")]
-    RootCertificate,
-    #[error("Connect heartbeat schedule is invalid")]
-    Schedule,
-    #[error("RustFS is not registered with Connect")]
-    NotRegistered,
-    #[error("the Connect device private key is missing")]
-    IdentityMissing,
-    #[error("the stored Connect certificate and device private key cannot form a TLS identity")]
-    IdentityCertificate,
-    #[error("the stored Connect credential name is invalid")]
-    CredentialName,
-    #[error("the stored Connect device certificate is not currently valid")]
-    CredentialExpired,
-    #[error("the Connect heartbeat node summary is outside protocol bounds")]
-    NodeSummary,
-    #[error("the Connect heartbeat sequence is exhausted")]
-    SequenceExhausted,
-    #[error("a Connect heartbeat runtime already owns this state")]
-    AlreadyRunning,
-    #[error("the persisted Connect heartbeat changed while delivery was in flight")]
-    StateConflict,
-    #[error("Connect heartbeat state I/O failed at {path}: {source}")]
-    StateIo {
-        path: PathBuf,
-        #[source]
-        source: io::Error,
-    },
-    #[error("Connect heartbeat state at {path} is invalid: {source}")]
-    StateInvalid {
-        path: PathBuf,
-        #[source]
-        source: serde_json::Error,
-    },
-    #[error("Connect heartbeat state at {path} violates the protocol invariants")]
-    StateCorrupt { path: PathBuf },
-    #[cfg(unix)]
-    #[error("Connect heartbeat state at {path} has mode {mode:o}, expected {expected:o}")]
-    StatePermissions { path: PathBuf, mode: u32, expected: u32 },
-    #[error("Connect heartbeat response exceeded 64 KiB")]
-    ResponseTooLarge,
-    #[error("Connect returned an invalid heartbeat response")]
-    Response,
-    #[error(transparent)]
-    Url(#[from] url::ParseError),
-    #[error(transparent)]
-    Transport(#[from] reqwest::Error),
-    #[error(transparent)]
-    Identity(#[from] IdentityError),
-    #[error(transparent)]
-    IdentityStore(#[from] StoreError),
-    #[error(transparent)]
-    CredentialStore(#[from] CredentialStoreError),
-    #[error(transparent)]
-    CredentialValidation(#[from] CredentialValidationError),
-}

rustfs/src/connect/mod.rs

@@ -21,26 +21,20 @@
 //! canonical transcript frozen by
 //! `protocol/agent/v1/registration-proof.md`.
 //!
-//! Enrolled deployments may start the optional outbound heartbeat runtime.
-//! An unconfigured server starts no Connect task, generates no key, and holds
-//! no Connect identity.
+//! Nothing here contacts the network or starts a task. A deployment that has
+//! not been enrolled into a Connect control plane never calls into it, so an
+//! unconfigured server generates no key and holds no identity.
 
 pub mod client;
-pub mod config;
 pub mod credential_store;
-pub mod heartbeat;
 pub mod identity;
 pub mod identity_store;
 pub mod offline;
 pub mod registration;
-pub mod runtime;
 
 pub use client::{ClientError, ConnectClient, ConnectConfig};
-pub use config::{HeartbeatConfig, HeartbeatConfigError, HeartbeatSchedule};
 pub use credential_store::{CredentialStore, DeviceCredential};
-pub use heartbeat::{CoarseNodeSummary, HeartbeatError, HeartbeatStatus};
 pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
 pub use identity_store::{IdentityStore, StoreError};
 pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};
 pub use registration::{RegistrationToken, TokenError};
-pub use runtime::{HeartbeatRuntime, spawn_heartbeat_runtime};

rustfs/src/connect/runtime.rs

@@ -1,156 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::future::Future;
-use std::time::Duration;
-
-use chrono::Utc;
-use rand::RngExt as _;
-use tokio::sync::watch;
-use tokio::task::JoinHandle;
-use tokio_util::sync::CancellationToken;
-
-use super::config::HeartbeatConfig;
-use super::heartbeat::{CoarseNodeSummary, Delivery, HeartbeatError, HeartbeatSender, HeartbeatStateStore, HeartbeatStatus};
-
-pub struct HeartbeatRuntime {
-    shutdown: CancellationToken,
-    status: watch::Receiver<HeartbeatStatus>,
-    task: Option<JoinHandle<()>>,
-}
-
-impl HeartbeatRuntime {
-    pub fn status(&self) -> watch::Receiver<HeartbeatStatus> {
-        self.status.clone()
-    }
-
-    pub async fn shutdown(mut self) {
-        self.shutdown.cancel();
-        if let Some(task) = self.task.take() {
-            let _ = task.await;
-        }
-    }
-}
-
-impl Drop for HeartbeatRuntime {
-    fn drop(&mut self) {
-        self.shutdown.cancel();
-    }
-}
-
-pub fn spawn_heartbeat_runtime<F>(
-    config: Option<HeartbeatConfig>,
-    parent_shutdown: &CancellationToken,
-    sample: F,
-) -> Result<Option<HeartbeatRuntime>, HeartbeatError>
-where
-    F: Fn() -> CoarseNodeSummary + Send + Sync + 'static,
-{
-    let Some(config) = config else {
-        return Ok(None);
-    };
-    let sender = HeartbeatSender::new(config.clone())?;
-    let store = HeartbeatStateStore::new(config.state_path.clone());
-    let lock = store.try_runtime_lock()?;
-    let schedule = config.schedule;
-    let shutdown = parent_shutdown.child_token();
-    let task_shutdown = shutdown.clone();
-    let (status_tx, status_rx) = watch::channel(HeartbeatStatus::Starting);
-    let task = tokio::spawn(async move {
-        let _lock = lock;
-        let mut backoff = schedule.initial_backoff;
-        loop {
-            if task_shutdown.is_cancelled() {
-                break;
-            }
-            let pending = match store.prepare(sample(), Utc::now()).await {
-                Ok(pending) => pending,
-                Err(error) => return failed(&status_tx, error),
-            };
-            let delivery = match cancellable(&task_shutdown, sender.send(&pending)).await {
-                Some(Ok(delivery)) => delivery,
-                Some(Err(error)) => return failed(&status_tx, error),
-                None => break,
-            };
-            let delay = match delivery {
-                Delivery::Accepted { server_time } => {
-                    if let Err(error) = store.mark_accepted(&pending).await {
-                        return failed(&status_tx, error);
-                    }
-                    backoff = schedule.initial_backoff;
-                    let _ = status_tx.send(HeartbeatStatus::Online { server_time });
-                    schedule.cadence.saturating_add(jitter(schedule.jitter))
-                }
-                Delivery::Retry { retry_after } => {
-                    let delay = retry_after
-                        .unwrap_or(backoff)
-                        .clamp(schedule.initial_backoff, schedule.max_backoff);
-                    backoff = backoff.saturating_mul(2).min(schedule.max_backoff);
-                    let _ = status_tx.send(HeartbeatStatus::BackingOff { delay });
-                    delay
-                }
-                Delivery::AuthenticationStopped { status, reason } => {
-                    let _ = status_tx.send(HeartbeatStatus::AuthenticationStopped { status, reason });
-                    return;
-                }
-                Delivery::Rejected { status, reason } => {
-                    let suffix = reason.map_or_else(String::new, |reason| format!("; reason={reason}"));
-                    let _ = status_tx.send(HeartbeatStatus::Failed {
-                        reason: format!("Connect rejected heartbeat with HTTP {status}{suffix}"),
-                    });
-                    return;
-                }
-            };
-            if sleep_or_cancel(&task_shutdown, delay).await {
-                break;
-            }
-        }
-        let _ = status_tx.send(HeartbeatStatus::Stopped);
-    });
-    Ok(Some(HeartbeatRuntime {
-        shutdown,
-        status: status_rx,
-        task: Some(task),
-    }))
-}
-
-fn failed(status: &watch::Sender<HeartbeatStatus>, error: HeartbeatError) {
-    let _ = status.send(HeartbeatStatus::Failed {
-        reason: error.to_string(),
-    });
-}
-
-fn jitter(maximum: Duration) -> Duration {
-    if maximum.is_zero() {
-        Duration::ZERO
-    } else {
-        maximum.mul_f64(rand::rng().random_range(0.0..=1.0))
-    }
-}
-
-async fn cancellable<T>(shutdown: &CancellationToken, future: impl Future<Output = T>) -> Option<T> {
-    tokio::select! {
-        biased;
-        () = shutdown.cancelled() => None,
-        value = future => Some(value),
-    }
-}
-
-async fn sleep_or_cancel(shutdown: &CancellationToken, delay: Duration) -> bool {
-    tokio::select! {
-        biased;
-        () = shutdown.cancelled() => true,
-        () = tokio::time::sleep(delay) => false,
-    }
-}

rustfs/src/startup_lifecycle.rs

@@ -128,7 +128,6 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
     } = lifecycle;
     let StartupServiceRuntime {
         optional_runtimes,
-        heartbeat,
         iam_bootstrap,
         enable_scanner,
     } = service_runtime;
@@ -163,9 +162,6 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
         shutdown_token,
     )
     .await;
-    if let Some(heartbeat) = heartbeat {
-        heartbeat.shutdown().await;
-    }
     if let Err(err) = event_notifier_reconciler.await {
         tracing::warn!(
             target: "rustfs::main::run",

rustfs/src/startup_services.rs

@@ -16,7 +16,6 @@ use crate::site_replication_reconcile::spawn_site_replication_reconcile_task;
 use crate::storage_api::startup::services::{ECStore, EndpointServerPools, ServerContextSlot};
 use crate::{
     config::Config,
-    connect::{CoarseNodeSummary, HeartbeatConfig, HeartbeatRuntime, spawn_heartbeat_runtime},
     init::{init_buffer_profile_system, init_kms_system},
     server::ServiceStateManager,
     startup_audit::init_audit_runtime,
@@ -36,7 +35,6 @@ use tokio_util::sync::CancellationToken;
 
 pub(crate) struct StartupServiceRuntime {
     pub(crate) optional_runtimes: OptionalRuntimeServices,
-    pub(crate) heartbeat: Option<HeartbeatRuntime>,
     pub(crate) iam_bootstrap: IamBootstrapDisposition,
     pub(crate) enable_scanner: bool,
 }
@@ -75,8 +73,6 @@ pub(crate) async fn init_startup_runtime_services(
     init_kms_system(config).await?;
 
     let optional_runtimes = init_optional_runtime_services().await?;
-    let heartbeat_config = HeartbeatConfig::from_env().map_err(std::io::Error::other)?;
-    let heartbeat_nodes = heartbeat_config.as_ref().map(|_| endpoint_pools.get_nodes().len());
 
     init_buffer_profile_system(config);
     init_deadlock_detector_runtime();
@@ -96,27 +92,10 @@ pub(crate) async fn init_startup_runtime_services(
     init_notification_runtime(endpoint_pools, buckets).await?;
     let enable_scanner = init_background_service_runtime(store.clone()).await?;
     init_observability_runtime(store.clone(), ctx.clone()).await;
-    let heartbeat = start_heartbeat_runtime(heartbeat_config, heartbeat_nodes, &ctx)?;
 
     Ok(StartupServiceRuntime {
         optional_runtimes,
-        heartbeat,
         iam_bootstrap,
         enable_scanner,
     })
 }
-
-fn start_heartbeat_runtime(
-    config: Option<HeartbeatConfig>,
-    node_count: Option<usize>,
-    shutdown: &CancellationToken,
-) -> Result<Option<HeartbeatRuntime>> {
-    let Some(config) = config else {
-        return Ok(None);
-    };
-    let summary = u16::try_from(node_count.unwrap_or_default())
-        .ok()
-        .and_then(|total| CoarseNodeSummary::new(total, 0, 0).ok())
-        .ok_or_else(|| std::io::Error::other("Connect heartbeat node count is outside protocol bounds"))?;
-    spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(std::io::Error::other)
-}

rustfs/tests/connect_heartbeat.rs

@@ -1,687 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::VecDeque;
-use std::fs;
-use std::path::Path;
-use std::sync::{Arc, Mutex};
-use std::time::Duration;
-
-use bytes::Bytes;
-use http_body_util::{BodyExt as _, Full};
-use hyper::service::service_fn;
-use hyper::{Request, Response, StatusCode};
-use hyper_util::rt::TokioIo;
-use rcgen::{
-    BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
-    KeyUsagePurpose, SanType,
-};
-use rustfs::connect::{
-    CoarseNodeSummary, CredentialStore, DeviceCredential, HeartbeatConfig, HeartbeatSchedule, HeartbeatStatus, IdentityStore,
-    spawn_heartbeat_runtime,
-};
-use rustls::RootCertStore;
-use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
-use rustls::server::WebPkiClientVerifier;
-use serde_json::{Value, json};
-use time::OffsetDateTime;
-use tokio::net::TcpListener;
-use tokio::sync::watch;
-use tokio_rustls::TlsAcceptor;
-use tokio_util::sync::CancellationToken;
-
-const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
-const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
-const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
-
-struct TestPki {
-    root_params: CertificateParams,
-    root_key: KeyPair,
-    root_der: CertificateDer<'static>,
-    root_pem: String,
-    server_der: CertificateDer<'static>,
-    server_key: PrivatePkcs8KeyDer<'static>,
-}
-
-impl TestPki {
-    fn new() -> Self {
-        let now = OffsetDateTime::now_utc();
-        let root_key = KeyPair::generate().expect("generate root key");
-        let mut root_params = CertificateParams::default();
-        root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
-        root_params.not_before = now - time::Duration::days(30);
-        root_params.not_after = now + time::Duration::days(30);
-        root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature];
-        let root = root_params.self_signed(&root_key).expect("sign root");
-
-        let server_key = KeyPair::generate().expect("generate server key");
-        let mut server_params = CertificateParams::default();
-        server_params.not_before = now - time::Duration::hours(1);
-        server_params.not_after = now + time::Duration::days(2);
-        server_params
-            .subject_alt_names
-            .push(SanType::DnsName("localhost".try_into().expect("valid DNS name")));
-        server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
-        let server = server_params
-            .signed_by(&server_key, &Issuer::from_params(&root_params, &root_key))
-            .expect("sign server certificate");
-        Self {
-            root_params,
-            root_key,
-            root_der: root.der().clone(),
-            root_pem: root.pem(),
-            server_der: server.der().clone(),
-            server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()),
-        }
-    }
-
-    fn server_config(&self) -> rustls::ServerConfig {
-        let mut roots = RootCertStore::empty();
-        roots.add(self.root_der.clone()).expect("add client root");
-        let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
-            .build()
-            .expect("client verifier");
-        rustls::ServerConfig::builder()
-            .with_client_cert_verifier(verifier)
-            .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key()))
-            .expect("server TLS")
-    }
-
-    fn stores(&self, temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) {
-        let now = OffsetDateTime::now_utc();
-        self.stores_with_certificate(temp, now - time::Duration::hours(1), now + time::Duration::hours(23), true)
-    }
-
-    fn stores_with_certificate(
-        &self,
-        temp: &tempfile::TempDir,
-        not_before: OffsetDateTime,
-        not_after: OffsetDateTime,
-        bind_identity: bool,
-    ) -> (IdentityStore, CredentialStore) {
-        let identity_store = IdentityStore::new(temp.path().join("identity"));
-        let identity = identity_store.load_or_create().expect("create identity");
-        let private_key = PrivatePkcs8KeyDer::from(identity.to_pkcs8_der().expect("serialize key").to_vec());
-        let device_key = if bind_identity {
-            KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("device key")
-        } else {
-            KeyPair::generate().expect("mismatched device key")
-        };
-        let mut params = CertificateParams::default();
-        params.not_before = not_before;
-        params.not_after = not_after;
-        params.serial_number = Some(vec![1; 16].into());
-        params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
-        params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
-        params.distinguished_name = DistinguishedName::new();
-        params.distinguished_name.push(DnType::CommonName, DEVICE_UID);
-        params.subject_alt_names.push(SanType::URI(
-            format!("urn:rustfs:connect:device:{DEVICE_UID}")
-                .try_into()
-                .expect("device URI"),
-        ));
-        let certificate = params
-            .signed_by(&device_key, &Issuer::from_params(&self.root_params, &self.root_key))
-            .expect("device certificate");
-        let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}");
-        let credential = DeviceCredential {
-            name: format!("{cluster}/clusterDevices/{DEVICE_UID}"),
-            uid: DEVICE_UID.to_owned(),
-            protocol_version: "v1".to_owned(),
-            key_id: format!("x509-{}", "01".repeat(16)),
-            certificate_serial: "01".repeat(16),
-            certificate: certificate.pem(),
-            certificate_chain: certificate.pem(),
-            not_before_unix: not_before.unix_timestamp(),
-            not_after_unix: not_after.unix_timestamp(),
-        };
-        let directory = temp.path().join("credential");
-        fs::create_dir_all(&directory).expect("credential directory");
-        let path = directory.join("device.crt.json");
-        fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("write credential");
-        private_mode(&path);
-        (identity_store, CredentialStore::new(directory))
-    }
-}
-
-#[derive(Clone)]
-struct Reply {
-    status: StatusCode,
-    body: Value,
-    retry_after: Option<&'static str>,
-    delay: Duration,
-}
-
-impl Reply {
-    fn ok(time: &str) -> Self {
-        Self {
-            status: StatusCode::OK,
-            body: json!({
-                "serverTime": time,
-                "acceptedVersion": "v1",
-                "capabilityHints": [],
-                "futureField": true
-            }),
-            retry_after: None,
-            delay: Duration::ZERO,
-        }
-    }
-
-    fn error(status: StatusCode) -> Self {
-        Self {
-            status,
-            body: json!({"details": []}),
-            retry_after: None,
-            delay: Duration::ZERO,
-        }
-    }
-}
-
-struct TestServer {
-    endpoint: String,
-    seen: Arc<Mutex<Vec<Value>>>,
-    task: tokio::task::JoinHandle<()>,
-}
-
-impl Drop for TestServer {
-    fn drop(&mut self) {
-        self.task.abort();
-    }
-}
-
-async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
-    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind server");
-    let address = listener.local_addr().expect("server address");
-    let acceptor = TlsAcceptor::from(Arc::new(pki.server_config()));
-    let replies = Arc::new(Mutex::new(VecDeque::from(replies)));
-    let seen = Arc::new(Mutex::new(Vec::new()));
-    let captured = seen.clone();
-    let task = tokio::spawn(async move {
-        while let Ok((stream, _)) = listener.accept().await {
-            let acceptor = acceptor.clone();
-            let replies = replies.clone();
-            let seen = captured.clone();
-            tokio::spawn(async move {
-                let Ok(stream) = acceptor.accept(stream).await else { return };
-                let service = service_fn(move |request: Request<hyper::body::Incoming>| {
-                    let replies = replies.clone();
-                    let seen = seen.clone();
-                    async move {
-                        assert_eq!(request.uri().path(), format!("/agent/clusters/{CLUSTER_UID}/heartbeats"));
-                        let body = request.into_body().collect().await.expect("request body").to_bytes();
-                        seen.lock()
-                            .expect("seen lock")
-                            .push(serde_json::from_slice(&body).expect("request JSON"));
-                        let reply = replies
-                            .lock()
-                            .expect("reply lock")
-                            .pop_front()
-                            .unwrap_or_else(|| Reply::error(StatusCode::SERVICE_UNAVAILABLE));
-                        if !reply.delay.is_zero() {
-                            tokio::time::sleep(reply.delay).await;
-                        }
-                        let mut builder = Response::builder()
-                            .status(reply.status)
-                            .header("content-type", "application/json");
-                        if let Some(value) = reply.retry_after {
-                            builder = builder.header("retry-after", value);
-                        }
-                        Ok::<_, hyper::Error>(
-                            builder
-                                .body(Full::new(Bytes::from(serde_json::to_vec(&reply.body).expect("reply JSON"))))
-                                .expect("reply"),
-                        )
-                    }
-                });
-                let _ = hyper::server::conn::http1::Builder::new()
-                    .serve_connection(TokioIo::new(stream), service)
-                    .await;
-            });
-        }
-    });
-    TestServer {
-        endpoint: format!("https://localhost:{}/agent/", address.port()),
-        seen,
-        task,
-    }
-}
-
-fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> HeartbeatConfig {
-    let (identity_store, credential_store) = pki.stores(temp);
-    config_with_stores(temp, pki, server, identity_store, credential_store)
-}
-
-fn config_with_stores(
-    temp: &tempfile::TempDir,
-    pki: &TestPki,
-    server: &TestServer,
-    identity_store: IdentityStore,
-    credential_store: CredentialStore,
-) -> HeartbeatConfig {
-    HeartbeatConfig {
-        endpoint: server.endpoint.clone(),
-        root_ca_pem: pki.root_pem.as_bytes().to_vec(),
-        identity_store,
-        credential_store,
-        state_path: temp.path().join("heartbeat/state.json"),
-        schedule: HeartbeatSchedule {
-            cadence: Duration::from_millis(40),
-            jitter: Duration::ZERO,
-            timeout: Duration::from_millis(200),
-            initial_backoff: Duration::from_millis(20),
-            max_backoff: Duration::from_millis(80),
-        },
-    }
-}
-
-fn rewrite_credential(temp: &tempfile::TempDir, update: impl FnOnce(&mut DeviceCredential)) {
-    let path = temp.path().join("credential/device.crt.json");
-    let mut credential: DeviceCredential =
-        serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("parse credential");
-    update(&mut credential);
-    fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("rewrite credential");
-    private_mode(&path);
-}
-
-fn summary() -> CoarseNodeSummary {
-    CoarseNodeSummary::new(8, 7, 1).expect("node summary")
-}
-
-async fn wait_for(
-    status: &mut watch::Receiver<HeartbeatStatus>,
-    predicate: impl Fn(&HeartbeatStatus) -> bool,
-) -> HeartbeatStatus {
-    tokio::time::timeout(Duration::from_secs(3), async {
-        loop {
-            let current = status.borrow_and_update().clone();
-            if predicate(&current) {
-                return current;
-            }
-            status.changed().await.expect("status channel");
-        }
-    })
-    .await
-    .expect("heartbeat status timeout")
-}
-
-async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer, expected: &str) {
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-    assert!(matches!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
-        HeartbeatStatus::Failed { reason } if reason.contains(expected)
-    ));
-    assert!(server.seen.lock().expect("seen lock").is_empty());
-    runtime.shutdown().await;
-}
-
-#[tokio::test]
-async fn connect_config_absent_starts_no_task() {
-    let shutdown = CancellationToken::new();
-    let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-    let sampled = calls.clone();
-    let runtime = spawn_heartbeat_runtime(None, &shutdown, move || {
-        sampled.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-        summary()
-    })
-    .expect("absent config");
-
-    assert!(runtime.is_none());
-    tokio::task::yield_now().await;
-    assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 0);
-}
-
-#[tokio::test]
-async fn duplicate_runtime_is_rejected_without_a_second_task() {
-    let pki = TestPki::new();
-    let mut reply = Reply::ok("2026-08-22T01:02:03Z");
-    reply.delay = Duration::from_secs(5);
-    let server = server(&pki, vec![reply]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let config = config(&temp, &pki, &server);
-    let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary)
-        .expect("first runtime")
-        .expect("configured runtime");
-
-    assert!(matches!(
-        spawn_heartbeat_runtime(Some(config), &shutdown, summary),
-        Err(rustfs::connect::HeartbeatError::AlreadyRunning)
-    ));
-    runtime.shutdown().await;
-}
-
-#[tokio::test(flavor = "current_thread")]
-async fn dropped_runtime_keeps_the_lock_until_its_task_stops() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let config = config(&temp, &pki, &server);
-    let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary)
-        .expect("first runtime")
-        .expect("configured runtime");
-
-    drop(runtime);
-    assert!(matches!(
-        spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary),
-        Err(rustfs::connect::HeartbeatError::AlreadyRunning)
-    ));
-
-    let replacement = tokio::time::timeout(Duration::from_secs(3), async {
-        loop {
-            match spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) {
-                Ok(Some(runtime)) => break runtime,
-                Err(rustfs::connect::HeartbeatError::AlreadyRunning) => tokio::task::yield_now().await,
-                Ok(None) => panic!("configured replacement returned no runtime"),
-                Err(error) => panic!("unexpected replacement error: {error}"),
-            }
-        }
-    })
-    .await
-    .expect("dropped runtime releases its lock after stopping");
-    replacement.shutdown().await;
-}
-
-#[tokio::test]
-async fn corrupt_persisted_state_is_rejected_before_network_delivery() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let config = config(&temp, &pki, &server);
-    let directory = config.state_path.parent().expect("state directory");
-    fs::create_dir_all(directory).expect("create state directory");
-    fs::write(
-        &config.state_path,
-        br#"{"nextSequence":0,"pending":{"protocolVersion":"v1","requestId":"550e8400-e29b-41d4-a716-446655440000","agentVersion":"rustfs-agent/1.0.0-rc.3","capabilities":["heartbeat"],"sequence":0,"clientTime":"2026-08-22T01:02:03Z","coarseNodeSummary":{"total":0,"healthy":0,"degraded":0}}}"#,
-    )
-    .expect("write corrupt state");
-    private_mode(&config.state_path);
-    let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-
-    assert!(matches!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
-        HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants")
-    ));
-    assert!(server.seen.lock().expect("seen lock").is_empty());
-    runtime.shutdown().await;
-}
-
-#[tokio::test]
-async fn invalid_stored_resource_name_is_rejected_before_network_delivery() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let config = config(&temp, &pki, &server);
-    rewrite_credential(&temp, |credential| {
-        credential.name = format!("organizations/{ORGANIZATION_UID}/clusters/not-a-uuid/clusterDevices/{DEVICE_UID}");
-    });
-
-    assert_credential_failure(config, &server, "wrong device identity").await;
-}
-
-#[tokio::test]
-async fn invalid_stored_protocol_is_rejected_before_network_delivery() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let config = config(&temp, &pki, &server);
-    rewrite_credential(&temp, |credential| credential.protocol_version = "v2".to_owned());
-
-    assert_credential_failure(config, &server, "wrong device identity").await;
-}
-
-#[tokio::test]
-async fn stored_certificate_key_mismatch_is_rejected_before_network_delivery() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let now = OffsetDateTime::now_utc();
-    let (identity_store, credential_store) =
-        pki.stores_with_certificate(&temp, now - time::Duration::hours(1), now + time::Duration::hours(23), false);
-    let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store);
-
-    assert_credential_failure(config, &server, "different device key").await;
-}
-
-#[tokio::test]
-async fn expired_stored_certificate_is_rejected_before_network_delivery() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let now = OffsetDateTime::now_utc();
-    let (identity_store, credential_store) =
-        pki.stores_with_certificate(&temp, now - time::Duration::days(2), now - time::Duration::days(1), true);
-    let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store);
-
-    assert_credential_failure(config, &server, "not currently valid").await;
-}
-
-#[tokio::test]
-async fn sends_only_l0_fields_and_accepts_additive_response_fields() {
-    let pki = TestPki::new();
-    let server = server(&pki, vec![Reply::ok("2038-01-19T03:14:07Z")]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-
-    assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
-        HeartbeatStatus::Online {
-            server_time: "2038-01-19T03:14:07Z".to_owned()
-        }
-    );
-    runtime.shutdown().await;
-    let seen = server.seen.lock().expect("seen lock");
-    let request = &seen[0];
-    let mut keys = request
-        .as_object()
-        .expect("heartbeat object")
-        .keys()
-        .map(String::as_str)
-        .collect::<Vec<_>>();
-    keys.sort_unstable();
-    assert_eq!(
-        keys,
-        [
-            "agentVersion",
-            "capabilities",
-            "clientTime",
-            "coarseNodeSummary",
-            "protocolVersion",
-            "requestId",
-            "sequence"
-        ]
-    );
-    assert_eq!(request["capabilities"], json!(["heartbeat"]));
-    assert_eq!(request["coarseNodeSummary"], json!({"total": 8, "healthy": 7, "degraded": 1}));
-    assert_ne!(request["clientTime"], "2038-01-19T03:14:07Z");
-    assert!(request.get("authorization").is_none());
-}
-
-#[tokio::test]
-async fn restart_replays_pending_request_then_advances_sequence() {
-    let pki = TestPki::new();
-    let first_server = server(&pki, vec![Reply::error(StatusCode::SERVICE_UNAVAILABLE)]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let first_config = config(&temp, &pki, &first_server);
-    let runtime = spawn_heartbeat_runtime(Some(first_config.clone()), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-    wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await;
-    runtime.shutdown().await;
-    let first = first_server.seen.lock().expect("seen lock")[0].clone();
-    drop(first_server);
-
-    let second_server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z"), Reply::ok("2026-08-22T01:02:04Z")]).await;
-    let mut second_config = first_config;
-    second_config.endpoint = second_server.endpoint.clone();
-    let runtime = spawn_heartbeat_runtime(Some(second_config), &shutdown, summary)
-        .expect("restart runtime")
-        .expect("configured runtime");
-    tokio::time::timeout(Duration::from_secs(3), async {
-        while second_server.seen.lock().expect("seen lock").len() < 2 {
-            tokio::time::sleep(Duration::from_millis(10)).await;
-        }
-    })
-    .await
-    .expect("two heartbeats");
-    runtime.shutdown().await;
-
-    let seen = second_server.seen.lock().expect("seen lock");
-    assert_eq!(seen[0]["requestId"], first["requestId"]);
-    assert_eq!(seen[0]["sequence"], first["sequence"]);
-    assert_ne!(seen[1]["requestId"], seen[0]["requestId"]);
-    assert_eq!(seen[1]["sequence"].as_u64(), seen[0]["sequence"].as_u64().map(|value| value + 1));
-}
-
-#[tokio::test]
-async fn retry_after_is_respected_with_the_local_upper_bound() {
-    let pki = TestPki::new();
-    let mut reply = Reply::error(StatusCode::TOO_MANY_REQUESTS);
-    reply.retry_after = Some("300");
-    let server = server(&pki, vec![reply]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-    assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await,
-        HeartbeatStatus::BackingOff {
-            delay: Duration::from_millis(80)
-        }
-    );
-    runtime.shutdown().await;
-}
-
-#[tokio::test]
-async fn disconnects_use_exponential_backoff_with_a_cap() {
-    let pki = TestPki::new();
-    let server = server(
-        &pki,
-        vec![
-            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
-            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
-            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
-        ],
-    )
-    .await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-    for delay in [20, 40, 80] {
-        assert_eq!(
-            wait_for(&mut status, |status| {
-                matches!(status, HeartbeatStatus::BackingOff { delay: observed } if *observed == Duration::from_millis(delay))
-            })
-            .await,
-            HeartbeatStatus::BackingOff {
-                delay: Duration::from_millis(delay)
-            }
-        );
-    }
-    runtime.shutdown().await;
-}
-
-#[tokio::test]
-async fn revoked_credential_stops_and_exposes_local_status() {
-    let pki = TestPki::new();
-    let mut reply = Reply::error(StatusCode::UNAUTHORIZED);
-    reply.body = json!({"details": [{"reason": "CREDENTIAL_REVOKED"}]});
-    let server = server(&pki, vec![reply]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    let mut status = runtime.status();
-    assert_eq!(
-        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await,
-        HeartbeatStatus::AuthenticationStopped {
-            status: 401,
-            reason: Some("CREDENTIAL_REVOKED".to_owned())
-        }
-    );
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    assert_eq!(server.seen.lock().expect("seen lock").len(), 1);
-    runtime.shutdown().await;
-}
-
-#[tokio::test]
-async fn shutdown_cancels_an_in_flight_request() {
-    let pki = TestPki::new();
-    let mut reply = Reply::ok("2026-08-22T01:02:03Z");
-    reply.delay = Duration::from_secs(5);
-    let server = server(&pki, vec![reply]).await;
-    let temp = tempfile::tempdir().expect("tempdir");
-    let shutdown = CancellationToken::new();
-    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
-        .expect("start runtime")
-        .expect("configured runtime");
-    tokio::time::timeout(Duration::from_secs(3), async {
-        while server.seen.lock().expect("seen lock").is_empty() {
-            tokio::task::yield_now().await;
-        }
-    })
-    .await
-    .expect("request reached server");
-    tokio::time::timeout(Duration::from_millis(250), runtime.shutdown())
-        .await
-        .expect("cancellable shutdown");
-}
-
-#[test]
-fn consumes_the_frozen_heartbeat_fixtures() {
-    let registry: Value =
-        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/fixture-sets.json")).expect("fixture registry");
-    let heartbeat = registry["sets"]
-        .as_array()
-        .expect("fixture sets")
-        .iter()
-        .find(|set| set["name"] == "heartbeat")
-        .expect("heartbeat fixture set");
-    assert_eq!(heartbeat["status"], "populated");
-    let valid: Value =
-        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/valid.json")).expect("valid fixture");
-    assert_eq!(valid["request"]["protocolVersion"], "v1");
-    let overflow: Value =
-        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/overflow.json")).expect("overflow fixture");
-    assert_eq!(overflow["expected"]["httpStatus"], 422);
-}
-
-#[cfg(unix)]
-fn private_mode(path: &Path) {
-    use std::os::unix::fs::PermissionsExt as _;
-    fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("private mode");
-}
-
-#[cfg(not(unix))]
-fn private_mode(_path: &Path) {}

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!