test(kms): validate Vault circuit recovery

* fix(scanner): isolate corrupt cycle state

* fix(scanner): preserve newer state during recovery reset

* fix(scanner): fence recovery reset state

* fix(scanner): reject terminal recovery epochs

* fix(scanner): reject trailing cycle state bytes

* fix(scanner): reject terminal leadership epochs

* fix(scanner): retain recovery wake notifications

* fix(scanner): recover from oversized markers

.github/actions/schedule-failure-issue/action.yml

@@ -14,10 +14,9 @@
 
 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails or
-  does not complete normally.
+  Open (or update) a tracking issue when a scheduled workflow run fails.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the result is
+  "[scheduled-failure] <workflow name>" already exists, the failure is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
@@ -39,26 +38,6 @@ inputs:
       Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
-  source-run-id:
-    description: "Run ID to report. Defaults to the current workflow run."
-    required: false
-    default: ${{ github.run_id }}
-  source-run-attempt:
-    description: "Run attempt to report. Defaults to the current attempt."
-    required: false
-    default: ${{ github.run_attempt }}
-  source-event:
-    description: "Trigger event of the run being reported."
-    required: false
-    default: ${{ github.event_name }}
-  source-ref-name:
-    description: "Ref name of the run being reported."
-    required: false
-    default: ${{ github.ref_name }}
-  source-sha:
-    description: "Commit SHA of the run being reported."
-    required: false
-    default: ${{ github.sha }}
 
 runs:
   using: "composite"
@@ -69,21 +48,17 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
-        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
-        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
-        SOURCE_EVENT: ${{ inputs.source-event }}
-        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
-        SOURCE_SHA: ${{ inputs.source-sha }}
       run: |
         set -euo pipefail
 
         title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
 
-        # Inspect the reported run attempt. It can be the current in-workflow
-        # failure or a completed run observed by the external watchdog.
+        # Failed job names for this run attempt. The alert job runs while the
+        # run as a whole is still in progress, so inspect the jobs that have
+        # already completed with a non-success conclusion.
         failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
                 | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -93,13 +68,13 @@ runs:
         fi
 
         body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** did not complete successfully.
+        Scheduled run of **${WORKFLOW_NAME}** failed.
 
-        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
-        - Event: \`${SOURCE_EVENT}\`
-        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
+        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
+        - Event: \`${GITHUB_EVENT_NAME}\`
+        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
 
-        Non-success jobs:
+        Failed jobs:
         ${failed_jobs}
         EOF
         )"

.github/workflows/build.yml

@@ -1032,23 +1032,3 @@ jobs:
 
           echo "🎉 Released $TAG successfully!"
           echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1032,37 +1032,3 @@ jobs:
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs:
-      - typos
-      - quick-checks
-      - test-and-lint
-      - test-ilm-integration-serial
-      - test-and-lint-rio-v2
-      - test-and-lint-protocols
-      - build-rustfs-debug-binary
-      - build-rustfs-debug-binary-rio-v2
-      - uring-integration
-      - e2e-tests
-      - e2e-full
-      - e2e-tests-rio-v2
-      - s3-implemented-tests
-      - s3-lifecycle-behavior-tests
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/minio-interop.yml

@@ -121,21 +121,3 @@ jobs:
           cargo nextest run --run-ignored ignored-only --no-tests=fail \
             -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
             -E "$INTEROP_FILTER"
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [minio-interop]
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -194,23 +191,3 @@ jobs:
 
       - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
         run: bash scripts/test/vault_ha_kms_live.sh
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [build, kms-vault-lane, kms-vault-ha-failover]
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scheduled-validation-watchdog.yml

@@ -1,63 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Scheduled Validation Watchdog
-
-on:
-  workflow_run:
-    workflows:
-      - "Security Audit"
-      - "Build and Release"
-      - "Continuous Integration"
-      - "coverage"
-      - "e2e-nightly"
-      - "e2e-s3tests"
-      - "Fuzz"
-      - "mint"
-      - "minio-interop"
-      - "Nightly GNU Build"
-      - "Performance A/B"
-      - "Runner Hygiene"
-    types: [completed]
-
-permissions:
-  contents: read
-
-jobs:
-  alert-on-incomplete-run:
-    name: Alert on incomplete scheduled run
-    if: >-
-      github.event.workflow_run.event == 'schedule' &&
-      github.event.workflow_run.conclusion != 'success' &&
-      github.event.workflow_run.conclusion != 'failure'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      actions: read
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update incomplete-run issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: ${{ github.event.workflow_run.name }}
-          source-run-id: ${{ github.event.workflow_run.id }}
-          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
-          source-event: ${{ github.event.workflow_run.event }}
-          source-ref-name: ${{ github.event.workflow_run.head_branch }}
-          source-sha: ${{ github.event.workflow_run.head_sha }}

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

crates/scanner/src/data_usage_define.rs

@@ -125,6 +125,34 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
     }
 }
 
+/// Read only the object revision without materializing its body.
+pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
+    match store
+        .get_object_reader(
+            RUSTFS_META_BUCKET,
+            path,
+            None,
+            HeaderMap::new(),
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(reader) => reader
+            .object_info
+            .etag
+            .filter(|etag| !etag.is_empty())
+            .map(DataUsageCacheRevision::Etag)
+            .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
+        Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
+            Ok(DataUsageCacheRevision::Missing)
+        }
+        Err(err) => Err(err),
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct DataUsageCacheRevisions {
     main: DataUsageCacheRevision,
@@ -146,6 +174,11 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
 pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
 
+/// Durable companion object for a cycle-state object which cannot be decoded.
+/// The primary object is deliberately never replaced or deleted by recovery.
+pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock<String> =
+    LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str()));
+
 pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
 

crates/scanner/src/data_usage_define/persistence.rs

@@ -74,7 +74,7 @@ impl DataUsageCache {
         let loaded = Self::load_cache(store.clone(), name).await?;
         let backup = match loaded.backup_revision {
             Some(revision) => Some(revision),
-            None => match Self::revision_for_path(store, &backup_path).await {
+            None => match read_config_revision(store, &backup_path).await {
                 Ok(revision) => Some(revision),
                 Err(err) => {
                     counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
@@ -336,33 +336,6 @@ impl DataUsageCache {
         }
     }
 
-    async fn revision_for_path<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
-        match store
-            .get_object_reader(
-                RUSTFS_META_BUCKET,
-                path,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    no_lock: true,
-                    ..Default::default()
-                },
-            )
-            .await
-        {
-            Ok(reader) => reader
-                .object_info
-                .etag
-                .filter(|etag| !etag.is_empty())
-                .map(DataUsageCacheRevision::Etag)
-                .ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))),
-            Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
-                Ok(DataUsageCacheRevision::Missing)
-            }
-            Err(err) => Err(err),
-        }
-    }
-
     pub(super) fn cache_save_timeout() -> Duration {
         crate::runtime_config::scanner_cache_save_timeout()
     }

crates/scanner/src/lib.rs

@@ -75,7 +75,10 @@ pub use remote_scanner::{
 };
 pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
 pub use rustfs_common::last_minute;
-pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
+pub use scanner::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
+    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
+};
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
     record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,

crates/scanner/src/scanner.rs

@@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock};
 
 use crate::data_usage_define::{
     BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
-    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision,
+    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
 };
 use crate::runtime_config::{
     ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
@@ -54,9 +54,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
 use rustfs_data_usage::observed_data_usage_is_newer;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
-#[cfg(test)]
-use tokio::sync::Notify;
-use tokio::sync::mpsc;
+use tokio::sync::{Notify, mpsc};
 use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tokio_util::task::AbortOnDropHandle;
@@ -104,6 +102,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
 /// unavailable peer cannot drive a tight retry loop.
 const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
 const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
+/// A transient backend outage remains self-healing after the short retry
+/// budget is exhausted, but the probe is intentionally sparse until storage
+/// recovers or an operator reset wakes the scanner.
+const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
+/// Permanent recovery states still get a sparse status probe so a reset that
+/// races the wait registration cannot leave the scanner asleep forever.
+const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
 const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
 #[cfg(not(test))]
 const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
@@ -125,6 +130,12 @@ type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
 static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
     LazyLock::new(|| StdMutex::new(None));
 
+static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
+
+pub(super) fn notify_scanner_cycle_recovery_wake() {
+    SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
+}
+
 #[cfg(test)]
 struct ScannerCycleStatePersistTestHookGuard;
 
@@ -576,19 +587,21 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
             tokio::time::sleep(sleep_time).await;
         }
 
+        let mut transient_backoff = ScannerRetryBackoff::default();
+        let mut recovery_retry_count = 0_u32;
         loop {
             if ctx_clone.is_cancelled() {
                 break;
             }
 
-            if let Err(e) = run_data_scanner_with_maintenance_state(
+            let run_result = run_data_scanner_with_maintenance_state(
                 ctx_clone.clone(),
                 storeapi_clone.clone(),
                 startup_features,
                 startup_maintenance_generation,
             )
-            .await
-            {
+            .await;
+            if let Err(e) = &run_result {
                 error!(
                     target: "rustfs::scanner",
                     event = EVENT_SCANNER_CYCLE_STATE,
@@ -599,11 +612,52 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
                     "Scanner runtime iteration failed"
                 );
             }
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.retryable {
+                recovery_retry_count = recovery_retry_count.saturating_add(1);
+                let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
+            } else {
+                recovery_retry_count = 0;
+            }
+
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.state == "paused" {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
+                }
+                recovery_retry_count = 0;
+                continue;
+            }
+            if !recovery_status.retryable
+                && matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
+            {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
+                }
+                continue;
+            }
+
+            let retry_delay = if recovery_status.retryable || run_result.is_err() {
+                transient_backoff.record_retryable_cycle(true);
+                transient_backoff
+                    .retry_interval(scanner_cycle_interval())
+                    .unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
+            } else {
+                transient_backoff.record_retryable_cycle(false);
+                randomized_cycle_delay()
+            };
             // Backoff before retrying after lock contention or scanner-level failures.
             // Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
             tokio::select! {
                 _ = ctx_clone.cancelled() => break,
-                _ = tokio::time::sleep(randomized_cycle_delay()) => {}
+                _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                _ = tokio::time::sleep(retry_delay) => {}
             }
         }
     });
@@ -1606,40 +1660,22 @@ async fn run_data_scanner_with_maintenance_state(
         observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
     }
 
-    let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await {
-        Ok((buf, revision)) => (buf.unwrap_or_default(), revision),
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
-                state = "revision_load_failed",
-                error = %err,
-                "Scanner cycle state revision load failed"
-            );
-            global_metrics().set_cycle(None).await;
-            return Ok(());
-        }
-    };
-    let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) {
-        Ok(state) => state,
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
-                state = "cycle_decode_failed",
-                error = %err,
-                "Scanner stopped because persisted cycle state is invalid"
-            );
-            global_metrics().set_cycle(None).await;
-            return Ok(());
-        }
-    };
+    let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
+        match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
+            ScannerCycleStateStartup::Ready {
+                cycle,
+                leader_epoch,
+                revision,
+            } => (cycle, leader_epoch, revision),
+            ScannerCycleStateStartup::Blocked => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleStateStartup::Transient(err) => {
+                global_metrics().set_cycle(None).await;
+                return Err(err);
+            }
+        };
     let usage_floor = match persisted_usage_floor(storeapi.clone()).await {
         Ok(floor) => floor,
         Err(err) => {
@@ -2219,7 +2255,12 @@ pub(crate) use activity::{
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
-pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence};
+pub use cycle_state::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
+};
+pub(crate) use cycle_state::{
+    current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
+};
 pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
 pub use usage_store::store_data_usage_in_backend;
 

crates/scanner/src/scanner/leadership.rs

@@ -196,7 +196,7 @@ pub(super) async fn claim_scanner_leadership(
         if ctx.is_cancelled() {
             return false;
         }
-        let Some(claimed_epoch) = persisted_epoch.checked_add(1) else {
+        let Some(claimed_epoch) = persisted_epoch.checked_add(1).filter(|epoch| *epoch < u64::MAX) else {
             error!(
                 target: "rustfs::scanner",
                 event = EVENT_SCANNER_PERSIST_STATE,

crates/scanner/src/scanner/tests.rs

@@ -15,11 +15,12 @@
 use super::*;
 use crate::EcstoreResult;
 use crate::{
-    Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
-    ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
-    init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
+    DATA_USAGE_BLOOM_RECOVERY_PATH, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
+    ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
+    ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
+    init_local_disks_with_instance_ctx,
 };
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::Cursor;
 use std::task::Poll;
 use temp_env::{with_var, with_var_unset};
@@ -117,6 +118,15 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
     assert!(cycle_ctx.is_cancelled());
 }
 
+#[tokio::test]
+async fn scanner_cycle_recovery_wake_survives_wait_registration_race() {
+    notify_scanner_cycle_recovery_wake();
+
+    tokio::time::timeout(Duration::from_secs(1), SCANNER_CYCLE_RECOVERY_WAKE.notified())
+        .await
+        .expect("recovery wake should retain a permit until the waiter registers");
+}
+
 struct ScannerDefaultSpeedGuard;
 
 impl ScannerDefaultSpeedGuard {
@@ -151,6 +161,7 @@ impl Drop for ScannerDefaultCycleGuard {
 struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
+    non_regular_objects: Mutex<HashSet<String>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
     object_not_found_put_number: Mutex<HashMap<String, usize>>,
     error_after_commit_put_number: Mutex<HashMap<String, usize>>,
@@ -191,12 +202,16 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
             .get(&key)
             .cloned()
             .ok_or(EcstoreError::FileNotFound)?;
-        let revision = *self.revisions.lock().await.entry(key).or_insert(1);
+        let data_len = i64::try_from(data.len()).expect("memory test object length should fit in i64");
+        let revision = *self.revisions.lock().await.entry(key.clone()).or_insert(1);
+        let is_dir = self.non_regular_objects.lock().await.contains(&key);
 
         Ok(GetObjectReader {
             stream: Box::new(Cursor::new(data)),
             object_info: ObjectInfo {
                 etag: Some(format!("memory-{revision}")),
+                size: data_len,
+                is_dir,
                 ..Default::default()
             },
             buffered_body: None,
@@ -797,6 +812,10 @@ fn scanner_cycle_state_decodes_legacy_and_fenced_formats() {
     let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode");
     assert_eq!(fenced_cycle.next, 13);
     assert_eq!(fenced_epoch, 7);
+
+    let mut trailing = fenced;
+    trailing.push(0);
+    assert!(decode_scanner_cycle_state(&trailing).is_err());
 }
 
 #[test]
@@ -823,6 +842,840 @@ fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() {
     assert!(encode_scanner_cycle_state(&exhausted, 7).is_err());
 }
 
+#[tokio::test]
+async fn corrupt_cycle_state_is_quarantined_once() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key.clone(), 7);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let marker_data = store
+        .objects
+        .lock()
+        .await
+        .get(&marker_key)
+        .cloned()
+        .expect("corrupt state must leave a durable recovery marker");
+    let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should be valid JSON");
+    assert_eq!(marker.primary_revision, "memory-7");
+    assert_eq!(marker.path, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    assert_eq!(marker.quarantine_path, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    assert_eq!(marker.classification, "corrupt");
+
+    // A second startup sees the matching marker before consuming the poison body.
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+
+    // Replacing the primary object advances its revision; the stale marker must
+    // not quarantine the newer, valid state.
+    let cycle = CurrentCycle {
+        next: 9,
+        ..Default::default()
+    };
+    let encoded = encode_scanner_cycle_state(&cycle, 3).expect("valid state should encode");
+    store.objects.lock().await.insert(state_key.clone(), encoded);
+    store.revisions.lock().await.insert(state_key, 8);
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Ready {
+            cycle: CurrentCycle { next: 9, .. },
+            leader_epoch: 3,
+            ..
+        }
+    ));
+}
+
+#[tokio::test]
+async fn empty_cycle_state_object_is_quarantined_as_corrupt() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), Vec::new());
+    store.revisions.lock().await.insert(state_key, 6);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt"));
+    assert!(
+        scanner_cycle_recovery_status()
+            .reason
+            .as_deref()
+            .is_some_and(|reason| reason.contains("empty"))
+    );
+}
+
+#[tokio::test]
+async fn future_cycle_state_schema_is_recovery_required() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let mut future = 17_u64.to_le_bytes().to_vec();
+    future.extend_from_slice(b"RSCYC999");
+    future.extend_from_slice(&4_u64.to_le_bytes());
+    future.extend_from_slice(&[0x90]);
+    store.objects.lock().await.insert(state_key.clone(), future);
+    store.revisions.lock().await.insert(state_key, 13);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("future_schema"));
+}
+
+#[tokio::test]
+async fn concurrent_leaders_cannot_quarantine_newer_cycle_state() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key, 4);
+
+    let (first, second) = tokio::join!(
+        load_scanner_cycle_state_for_startup(store.clone()),
+        load_scanner_cycle_state_for_startup(store.clone()),
+    );
+    assert!(matches!(first, ScannerCycleStateStartup::Blocked));
+    assert!(matches!(second, ScannerCycleStateStartup::Blocked));
+
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let marker_data = store
+        .objects
+        .lock()
+        .await
+        .get(&marker_key)
+        .cloned()
+        .expect("one contender must publish the recovery marker");
+    let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should decode");
+    assert_eq!(marker.primary_revision, "memory-4");
+}
+
+#[tokio::test]
+async fn cleanup_pending_marker_blocks_a_rewritten_primary_after_restart() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    let encoded = encode_scanner_cycle_state(
+        &CurrentCycle {
+            next: 12,
+            ..Default::default()
+        },
+        8,
+    )
+    .expect("valid state should encode");
+    store.objects.lock().await.insert(state_key.clone(), encoded);
+    store.revisions.lock().await.insert(state_key, 22);
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-21".to_string(),
+        generation: 11,
+        leader_epoch: 7,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    store
+        .objects
+        .lock()
+        .await
+        .insert(marker_key.clone(), serde_json::to_vec(&marker).expect("marker should encode"));
+    store.revisions.lock().await.insert(marker_key, 3);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().state, "cleanup-pending");
+}
+
+#[test]
+fn full_rescan_reset_accepts_unknown_marker_fields_without_trusting_cursor() {
+    let marker = br#"{
+        "schema_version": 99,
+        "primary_revision": "memory-7",
+        "generation": 9000,
+        "leader_epoch": 9000,
+        "classification": "new-future-classification",
+        "first_detected_at_unix_secs": 1,
+        "last_attempt_at_unix_secs": 2,
+        "retry_count": 9,
+        "reason": "future marker",
+        "path": "buckets/.bloomcycle.bin",
+        "quarantine_path": "buckets/.bloomcycle.bin.recovery-required.json",
+        "future_field": {"cursor": "untrusted"}
+    }"#;
+    let decoded =
+        super::cycle_state::decode_recovery_marker_for_reset(marker, &DataUsageCacheRevision::Etag("memory-3".to_string()))
+            .expect("full-rescan compatibility decoder should accept additive fields");
+    assert_eq!(decoded.primary_revision, "memory-7");
+    assert_eq!(decoded.classification, "future_schema");
+    assert_eq!(decoded.generation, 0);
+    assert_eq!(decoded.leader_epoch, 0);
+    assert_eq!(decoded.state, "blocked");
+
+    let malformed =
+        super::cycle_state::decode_recovery_marker_for_reset(b"{not-json", &DataUsageCacheRevision::Etag("memory-4".to_string()))
+            .expect("a full-rescan reset must recover even when the marker is malformed");
+    assert!(malformed.primary_revision.is_empty());
+    assert_eq!(malformed.classification, "future_schema");
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_after_malformed_marker_without_trusting_cursor() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover malformed marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0, "reset must use the verified usage floor, not marker cursor");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_ignores_epoch_from_malformed_future_primary() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let mut future_primary = vec![0; 24];
+    future_primary[8..16].copy_from_slice(b"RSCY9999");
+    future_primary[16..24].copy_from_slice(&u64::MAX.to_le_bytes());
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), future_primary)
+        .await
+        .expect("future cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover malformed future state");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1, "invalid persisted bytes must not raise the recovery epoch");
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn ecstore_exact_recovery_marker_delete_honors_etag() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v1".to_vec())
+        .await
+        .expect("initial recovery marker should be persisted");
+    let (_, stale_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("initial marker revision should load");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v2".to_vec())
+        .await
+        .expect("replacement recovery marker should be persisted");
+
+    let delete_result = store
+        .delete_config_object(
+            RUSTFS_META_BUCKET,
+            DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+            ObjectOptions {
+                http_preconditions: Some(stale_revision.preconditions()),
+                ..Default::default()
+            },
+        )
+        .await;
+    assert!(matches!(delete_result, Err(EcstoreError::PreconditionFailed)));
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("replacement marker should remain durable"),
+        b"marker-v2"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_corrupt_primary_under_stale_blocked_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let corrupt_primary = vec![0xff, 0x00, 0x01];
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), corrupt_primary.clone())
+        .await
+        .expect("corrupt cycle state should be persisted");
+    let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary revision should load");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-stale".to_string(),
+        generation: 1,
+        leader_epoch: 1,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "blocked primary changed".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "blocked".to_string(),
+    };
+    let marker_data = serde_json::to_vec(&marker).expect("blocked marker should encode");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), marker_data.clone())
+        .await
+        .expect("blocked marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "a strict marker must fail closed when its primary revision changed"
+    );
+    assert_eq!(
+        read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+            .await
+            .expect("primary should remain readable"),
+        corrupt_primary
+    );
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("blocked marker should remain durable"),
+        marker_data
+    );
+    assert!(!matches!(primary_revision, DataUsageCacheRevision::Missing));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    let old_primary_data = encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode");
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), old_primary_data.clone())
+        .await
+        .expect("valid cycle state should be persisted");
+    let (_, old_primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary state revision should load");
+    let old_usage = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(41),
+        ..Default::default()
+    };
+    let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
+    save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
+        .await
+        .expect("usage snapshot should be persisted");
+    let (_, old_usage_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage snapshot revision should load");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("reset should clear a stale malformed marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("valid primary should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("primary cycle state should decode");
+    assert_eq!(cycle.next, 42, "reset must not regress an independently fenced primary");
+    assert_eq!(leader_epoch, 8, "reset must advance the preserved primary epoch");
+    let stale_primary_save = save_config_with_preconditions(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        old_primary_data,
+        old_primary_revision.preconditions(),
+    )
+    .await;
+    assert!(matches!(stale_primary_save, Err(EcstoreError::PreconditionFailed)));
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage epoch fence should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<DataUsageInfo>(&usage)
+            .expect("fenced usage should decode")
+            .scanner_epoch,
+        Some(8)
+    );
+    let stale_save = save_config_with_preconditions(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        old_usage_data,
+        old_usage_revision.preconditions(),
+    )
+    .await;
+    assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed)));
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let completed_at = Utc::now();
+    let primary = CurrentCycle {
+        current: 3,
+        next: 42,
+        cycle_completed: vec![completed_at],
+        started: completed_at,
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    let usage = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(41),
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+    )
+    .await
+    .expect("usage snapshot should be persisted");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-old".to_string(),
+        generation: 41,
+        leader_epoch: 7,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("marker should encode"),
+    )
+    .await
+    .expect("cleanup marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("reset should resume a cleanup-pending preserved primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("preserved cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("cycle state should decode");
+    assert_eq!(cycle.current, 3, "cleanup retry must preserve the in-progress cursor");
+    assert_eq!(cycle.next, 42);
+    assert_eq!(cycle.cycle_completed, vec![completed_at]);
+    assert_eq!(cycle.started, completed_at);
+    assert_eq!(leader_epoch, 8);
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("usage epoch fence should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<DataUsageInfo>(&usage)
+            .expect("usage should decode")
+            .scanner_epoch,
+        Some(8)
+    );
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_oversized_regular_primary_with_malformed_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1])
+        .await
+        .expect("oversized cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("explicit full-rescan reset should replace an oversized regular primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_oversized_primary_after_cleanup_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1])
+        .await
+        .expect("oversized cycle state should be persisted");
+    let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("primary revision should load");
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: match primary_revision {
+            DataUsageCacheRevision::Etag(etag) => etag,
+            DataUsageCacheRevision::Missing => panic!("primary revision should be present"),
+        },
+        generation: 1,
+        leader_epoch: 1,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 1,
+        reason: "reset in progress".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "cleanup-pending".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("cleanup marker should encode"),
+    )
+    .await
+    .expect("cleanup marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("cleanup retry should rebuild an oversized primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_with_oversized_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), vec![b'x'; 64 * 1024 + 1])
+        .await
+        .expect("oversized recovery marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover an oversized marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_with_empty_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), Vec::new())
+        .await
+        .expect("empty recovery marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recover an empty marker");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_keeps_cleanup_marker_when_preserved_epoch_is_exhausted() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, u64::MAX).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err()
+    );
+
+    let marker = read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("cleanup marker should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker)
+            .expect("cleanup marker should decode")
+            .state,
+        "cleanup-pending"
+    );
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_preserved_epoch_that_would_be_terminal() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let primary = CurrentCycle {
+        next: 42,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&primary, u64::MAX - 1).expect("valid cycle state should encode"),
+    )
+    .await
+    .expect("valid cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "reset must not persist the terminal leader epoch"
+    );
+
+    let marker = read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+        .await
+        .expect("cleanup marker should remain durable");
+    assert_eq!(
+        serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker)
+            .expect("cleanup marker should decode")
+            .state,
+        "cleanup-pending"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01])
+        .await
+        .expect("corrupt cycle state should be persisted");
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&DataUsageInfo {
+            scanner_epoch: Some(u64::MAX - 1),
+            ..Default::default()
+        })
+        .expect("usage floor should encode"),
+    )
+    .await
+    .expect("usage floor should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    assert!(
+        reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+            .await
+            .is_err(),
+        "reset must not persist the terminal leader epoch"
+    );
+    assert_eq!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
+            .await
+            .expect("recovery marker should remain durable"),
+        b"{not-json"
+    );
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_empty_primary_with_malformed_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), Vec::new())
+        .await
+        .expect("empty cycle state should be persisted");
+    save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("explicit full-rescan reset should replace an empty primary");
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("rebuilt cycle state should remain durable");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn full_rescan_reset_rebuilds_when_primary_cycle_state_is_missing() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let marker = ScannerCycleRecoveryMarker {
+        schema_version: 1,
+        primary_revision: "memory-missing".to_string(),
+        generation: u64::MAX,
+        leader_epoch: u64::MAX,
+        classification: "corrupt".to_string(),
+        first_detected_at_unix_secs: 1,
+        last_attempt_at_unix_secs: 2,
+        retry_count: 0,
+        reason: "missing primary".to_string(),
+        path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
+        quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
+        state: "blocked".to_string(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("marker should encode"),
+    )
+    .await
+    .expect("marker should be persisted");
+
+    reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
+        .await
+        .expect("full-rescan reset should recreate missing primary");
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("missing primary should be rebuilt");
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode");
+    assert_eq!(cycle.next, 0);
+    assert_eq!(leader_epoch, 1);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn corrupt_cycle_state_rename_or_marker_failure_stays_recovery_required() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    store.objects.lock().await.insert(state_key.clone(), vec![1]);
+    store.revisions.lock().await.insert(state_key, 9);
+    store.fail_put_number.lock().await.insert(marker_key, 1);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Transient(_)
+    ));
+    let status = scanner_cycle_recovery_status();
+    assert_eq!(status.state, "recovery-required");
+    assert!(status.retryable);
+    assert!(
+        store
+            .objects
+            .lock()
+            .await
+            .contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()))
+    );
+}
+
+#[tokio::test]
+async fn oversized_or_symlinked_cycle_state_is_rejected() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.objects.lock().await.insert(key.clone(), vec![0; 1024 * 1024 + 1]);
+    store.revisions.lock().await.insert(key.clone(), 11);
+
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store.clone()).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+    assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt"));
+    assert!(
+        scanner_cycle_recovery_status()
+            .reason
+            .as_deref()
+            .is_some_and(|reason| reason.contains("oversized"))
+    );
+
+    let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
+    store.objects.lock().await.remove(&marker_key);
+    store.objects.lock().await.insert(key.clone(), vec![1]);
+    store.revisions.lock().await.insert(key.clone(), 12);
+    store.non_regular_objects.lock().await.insert(key);
+    // The object contract exposes a non-regular object as `is_dir`; local
+    // backends reject symlink/reparse entries before they become an object.
+    assert!(matches!(
+        load_scanner_cycle_state_for_startup(store).await,
+        ScannerCycleStateStartup::Blocked
+    ));
+}
+
 #[tokio::test]
 async fn scanner_startup_uses_primary_and_backup_usage_floor() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -855,6 +1708,31 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
     assert_eq!(epoch, 11);
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&DataUsageInfo {
+                scanner_epoch: Some(epoch),
+                scanner_cycle: Some(cycle),
+                ..Default::default()
+            })
+            .expect("usage snapshot should encode"),
+        );
+    }
+
+    assert_eq!(
+        persisted_usage_floor(store).await.expect("usage floor should load"),
+        PersistedUsageFloor {
+            next_cycle: 101,
+            leader_epoch: 8,
+        }
+    );
+}
+
 #[test]
 fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
     let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
@@ -987,6 +1865,15 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state()
 
     assert!(persisted_usage_floor(store.clone()).await.is_err());
 
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        br#"{}"#.to_vec(),
+    );
+    assert!(
+        persisted_usage_floor(store.clone()).await.is_err(),
+        "a structurally incomplete usage snapshot must not be treated as an empty floor"
+    );
+
     store.objects.lock().await.insert(
         memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
         serde_json::to_vec(&DataUsageInfo {
@@ -1244,6 +2131,22 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
     assert_eq!(store.put_counts.lock().await.get(&key), Some(&3));
 }
 
+#[tokio::test]
+async fn test_leadership_claim_rejects_terminal_epoch() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let ctx = CancellationToken::new();
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle {
+        next: 12,
+        ..Default::default()
+    };
+    let mut persisted_epoch = u64::MAX - 1;
+
+    assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
+    assert_eq!(persisted_epoch, u64::MAX - 1);
+    assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
+}
+
 #[tokio::test]
 async fn test_leadership_claim_confirms_commit_after_returned_error() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2975,6 +3878,24 @@ fn superseded_retry_backoff_grows_from_the_default_cycle() {
     }
 }
 
+#[tokio::test(start_paused = true)]
+async fn corrupt_cycle_state_backoff_uses_virtual_clock() {
+    let mut backoff = ScannerRetryBackoff::default();
+    backoff.record_retryable_cycle(true);
+    let first_delay = backoff
+        .retry_interval(Duration::from_secs(60))
+        .expect("the first recovery retry should be scheduled");
+    assert_eq!(first_delay, Duration::from_secs(5));
+
+    let deadline = Instant::now() + first_delay;
+    assert!(Instant::now() < deadline);
+    tokio::time::advance(first_delay).await;
+    assert!(Instant::now() >= deadline);
+
+    backoff.record_retryable_cycle(true);
+    assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(10)));
+}
+
 #[test]
 fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
     let runtime_config = ScannerRuntimeConfig {

rustfs/src/admin/handlers/mod.rs

@@ -126,6 +126,7 @@ mod tests {
         let _list_remote_target_handler = replication::ListRemoteTargetHandler {};
         let _remove_remote_target_handler = replication::RemoveRemoteTargetHandler {};
         let _scanner_status_handler = scanner::ScannerStatusHandler {};
+        let _scanner_cycle_state_reset_handler = scanner::ScannerCycleStateResetHandler {};
         let _ilm_expiry_status_handler = scanner::IlmExpiryStatusHandler {};
         let _manual_transition_handler = ilm_transition::ManualTransitionRunHandler {};
         let _manual_transition_status_handler = ilm_transition::ManualTransitionJobStatusHandler {};

rustfs/src/admin/handlers/scanner.rs

@@ -13,8 +13,11 @@
 // limitations under the License.
 
 use crate::admin::auth::authorize_admin_request;
+use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
-use crate::admin::runtime_sources::current_scanner_metrics_report;
+use crate::admin::runtime_sources::{
+    app_context_from_req, current_object_store_handle_for_context, current_scanner_metrics_report,
+};
 use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env};
 use crate::server::ADMIN_PREFIX;
 use chrono::Utc;
@@ -22,11 +25,13 @@ use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
 use matchit::Params;
 use rustfs_common::metrics::{ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport};
+use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_credentials::Credentials;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use s3s::header::CONTENT_TYPE;
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use tokio_util::sync::CancellationToken;
 
 const JSON_CONTENT_TYPE: &str = "application/json";
 
@@ -38,6 +43,13 @@ struct ScannerStatusResponse {
     metrics: ScannerMetricsReport,
     cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus,
     runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
+    cycle_recovery: rustfs_scanner::ScannerCycleRecoveryStatus,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ScannerCycleResetRequest {
+    mode: String,
 }
 
 #[derive(Debug, Serialize)]
@@ -117,6 +129,7 @@ fn scanner_status_response(
         metrics,
         cycle_schedule,
         runtime_config,
+        cycle_recovery: rustfs_scanner::scanner::scanner_cycle_recovery_status(),
     }
 }
 
@@ -144,6 +157,11 @@ pub fn register_scanner_route(r: &mut S3Router<AdminOperation>) -> std::io::Resu
         format!("{ADMIN_PREFIX}/v3/scanner/status").as_str(),
         AdminOperation(&ScannerStatusHandler {}),
     )?;
+    r.insert(
+        Method::POST,
+        format!("{ADMIN_PREFIX}/v3/scanner/cycle-state/reset").as_str(),
+        AdminOperation(&ScannerCycleStateResetHandler {}),
+    )?;
     r.insert(
         Method::GET,
         format!("{ADMIN_PREFIX}/v3/ilm/expiry/status").as_str(),
@@ -163,6 +181,13 @@ async fn validate_scanner_status_request(req: &S3Request<Body>) -> S3Result<Cred
     authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await
 }
 
+async fn validate_scanner_reset_request(req: &S3Request<Body>) -> S3Result<Credentials> {
+    if req.credentials.is_none() {
+        return Err(s3_error!(InvalidRequest, "missing credentials"));
+    }
+    authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)]).await
+}
+
 fn json_response(body: Vec<u8>) -> S3Result<S3Response<(StatusCode, Body)>> {
     let mut headers = HeaderMap::new();
     let content_type = HeaderValue::from_str(JSON_CONTENT_TYPE)
@@ -192,6 +217,37 @@ impl Operation for ScannerStatusHandler {
 
 pub struct IlmExpiryStatusHandler {}
 
+pub struct ScannerCycleStateResetHandler {}
+
+#[async_trait::async_trait]
+impl Operation for ScannerCycleStateResetHandler {
+    async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let _cred = validate_scanner_reset_request(&req).await?;
+        let body = req
+            .input
+            .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
+            .await
+            .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?;
+        let reset = serde_json::from_slice::<ScannerCycleResetRequest>(&body)
+            .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?;
+        if reset.mode != "full-rescan" {
+            return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "reset mode must be full-rescan"));
+        }
+        let context = app_context_from_req(&req)
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?;
+        let store = current_object_store_handle_for_context(Some(context.as_ref()))
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?;
+        supervise_admin_mutation("scanner cycle state reset", async move {
+            rustfs_scanner::scanner::reset_scanner_cycle_recovery(CancellationToken::new(), store)
+                .await
+                .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
+            Ok::<_, S3Error>(())
+        })
+        .await?;
+        json_response(br#"{"status":"reset","mode":"full-rescan"}"#.to_vec())
+    }
+}
+
 #[async_trait::async_trait]
 impl Operation for IlmExpiryStatusHandler {
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
@@ -237,6 +293,38 @@ mod tests {
         assert_eq!(err.message(), Some("missing credentials"));
     }
 
+    #[tokio::test]
+    async fn scanner_reset_gate_rejects_missing_credentials() {
+        let req = S3Request {
+            input: Body::from(String::new()),
+            method: Method::POST,
+            uri: http::Uri::from_static("/rustfs/admin/v3/scanner/cycle-state/reset"),
+            headers: HeaderMap::new(),
+            extensions: http::Extensions::new(),
+            credentials: None,
+            region: None,
+            service: None,
+            trailing_headers: None,
+        };
+
+        let err = validate_scanner_reset_request(&req)
+            .await
+            .expect_err("a reset request without credentials must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+        assert_eq!(err.message(), Some("missing credentials"));
+    }
+
+    #[test]
+    fn admin_reset_requires_full_rescan_or_verified_cursor() {
+        let full_rescan: ScannerCycleResetRequest =
+            serde_json::from_str(r#"{"mode":"full-rescan"}"#).expect("full rescan must be accepted");
+        assert_eq!(full_rescan.mode, "full-rescan");
+        let cursor: ScannerCycleResetRequest =
+            serde_json::from_str(r#"{"mode":"cursor"}"#).expect("mode validation belongs to the handler");
+        assert_ne!(cursor.mode, "full-rescan");
+        assert!(serde_json::from_str::<ScannerCycleResetRequest>(r#"{"mode":"full-rescan","cursor":"untrusted"}"#).is_err());
+    }
+
     #[test]
     fn scanner_disabled_reason_reports_startup_env_key() {
         assert_eq!(scanner_disabled_reason(true), None);
@@ -304,6 +392,11 @@ mod tests {
         assert_eq!(encoded["cycle_schedule"]["effective_interval_seconds"], 0);
         assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_enabled"], false);
         assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_multiplier"], 1);
+        assert_eq!(encoded["cycle_recovery"]["state"], "healthy");
+        assert_eq!(
+            encoded["cycle_recovery"]["quarantine_path"],
+            rustfs_scanner::DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()
+        );
     }
 
     #[test]

rustfs/src/admin/route_policy.rs

@@ -428,6 +428,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
     admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High),
     admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High),
     admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive),
+    admin(
+        HttpMethod::Post,
+        "/rustfs/admin/v3/scanner/cycle-state/reset",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/v3/ilm/expiry/status",
@@ -2020,6 +2026,12 @@ mod tests {
         assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/expiry/status", SET_TIER);
     }
 
+    #[test]
+    fn route_policy_requires_config_update_for_scanner_cycle_reset() {
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", CONFIG_UPDATE);
+        assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", SERVER_INFO);
+    }
+
     #[test]
     fn route_policy_uses_tier_actions_for_transition_routes() {
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);

rustfs/src/admin/route_registration_test.rs

@@ -243,6 +243,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::GET, "/v3/config"),
         admin_route(Method::PUT, "/v3/config"),
         admin_route(Method::GET, "/v3/scanner/status"),
+        admin_route(Method::POST, "/v3/scanner/cycle-state/reset"),
         admin_route(Method::GET, "/v3/audit/target/list"),
         admin_route_sample(
             Method::PUT,
@@ -879,6 +880,7 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/config"));
     assert_route(&router, Method::PUT, &admin_path("/v3/config"));
     assert_route(&router, Method::GET, &admin_path("/v3/scanner/status"));
+    assert_route(&router, Method::POST, &admin_path("/v3/scanner/cycle-state/reset"));
     assert_route(&router, Method::GET, &admin_path("/v3/ilm/expiry/status"));
     assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/run"));
     assert_route(
@@ -1367,6 +1369,7 @@ fn test_admin_alias_paths_match_existing_admin_routes() {
         (Method::GET, compat_admin_alias_path("/v3/config")),
         (Method::PUT, compat_admin_alias_path("/v3/config")),
         (Method::GET, compat_admin_alias_path("/v3/scanner/status")),
+        (Method::POST, compat_admin_alias_path("/v3/scanner/cycle-state/reset")),
         (Method::GET, compat_admin_alias_path("/v3/ilm/expiry/status")),
     ] {
         assert!(

scripts/check_test_wiring.py

@@ -15,20 +15,6 @@ from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[1]
-SCHEDULED_ALERT_WORKFLOWS = (
-    ".github/workflows/audit.yml",
-    ".github/workflows/build.yml",
-    ".github/workflows/ci.yml",
-    ".github/workflows/coverage.yml",
-    ".github/workflows/e2e-replication-nightly.yml",
-    ".github/workflows/e2e-s3tests.yml",
-    ".github/workflows/fuzz.yml",
-    ".github/workflows/mint.yml",
-    ".github/workflows/minio-interop.yml",
-    ".github/workflows/nightly-gnu.yml",
-    ".github/workflows/performance-ab.yml",
-    ".github/workflows/runner-hygiene.yml",
-)
 
 
 def words(value: str) -> set[str]:
@@ -266,72 +252,6 @@ def check_profile_definitions(root: Path) -> list[str]:
     return errors
 
 
-def check_scheduled_alerts(root: Path) -> list[str]:
-    errors: list[str] = []
-    for relative in SCHEDULED_ALERT_WORKFLOWS:
-        path = root / relative
-        try:
-            lines = path.read_text().splitlines()
-        except FileNotFoundError:
-            errors.append(f"{relative}: missing scheduled validation workflow")
-            continue
-
-        try:
-            start = lines.index("  alert-on-failure:") + 1
-        except ValueError:
-            errors.append(f"{relative}: missing alert-on-failure job")
-            continue
-        end = next(
-            (index for index in range(start, len(lines)) if re.fullmatch(r"  [A-Za-z0-9_-]+:", lines[index])),
-            len(lines),
-        )
-        job = "\n".join(line.split("#", 1)[0] for line in lines[start:end])
-        required = (
-            "always()",
-            "github.event_name == 'schedule'",
-            "contains(needs.*.result, 'failure')",
-            "issues: write",
-            "uses: ./.github/actions/schedule-failure-issue",
-            "github-token: ${{ secrets.GITHUB_TOKEN }}",
-        )
-        missing = [token for token in required if token not in job]
-        if missing:
-            errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}")
-
-    watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml"
-    try:
-        watchdog = "\n".join(
-            line.split("#", 1)[0] for line in watchdog_path.read_text().splitlines()
-        )
-    except FileNotFoundError:
-        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog")
-        return errors
-    for relative in SCHEDULED_ALERT_WORKFLOWS:
-        path = root / relative
-        if not path.is_file():
-            continue
-        source = path.read_text()
-        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
-        if not match:
-            errors.append(f"{relative}: missing workflow name")
-        elif f'- "{match.group(1).strip()}"' not in watchdog:
-            errors.append(f"{relative}: missing from scheduled completion watchdog")
-    required = (
-        "github.event.workflow_run.event == 'schedule'",
-        "github.event.workflow_run.conclusion != 'success'",
-        "github.event.workflow_run.conclusion != 'failure'",
-        "workflow-name: ${{ github.event.workflow_run.name }}",
-        "source-run-id: ${{ github.event.workflow_run.id }}",
-        "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
-    )
-    missing = [token for token in required if token not in watchdog]
-    if missing:
-        errors.append(
-            ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing)
-        )
-    return errors
-
-
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
     try:
         expected_digest = profile_selection(root, profile)
@@ -361,7 +281,6 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_runner_selection(root))
     errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
-    errors.extend(check_scheduled_alerts(root))
     return errors
 
 
@@ -444,7 +363,6 @@ class SelfTests(unittest.TestCase):
                 mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
                 mock.patch(__name__ + ".check_runner_selection", return_value=[]),
                 mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
-                mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
             ):
                 self.assertEqual(len(validate(root)), 1)
 
@@ -495,54 +413,6 @@ class SelfTests(unittest.TestCase):
             with mock.patch.object(sys, "platform", "linux"):
                 self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
 
-    def test_scheduled_alerts_require_completion_watchdog(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            alert = (
-                "  alert-on-failure:\n"
-                "    if: always() && github.event_name == 'schedule' && "
-                "contains(needs.*.result, 'failure')\n"
-                "    permissions:\n"
-                "      issues: write\n"
-                "    steps:\n"
-                "      - uses: ./.github/actions/schedule-failure-issue\n"
-                "        with:\n"
-                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
-            )
-            names: list[str] = []
-            for relative in SCHEDULED_ALERT_WORKFLOWS:
-                path = root / relative
-                path.parent.mkdir(parents=True, exist_ok=True)
-                names.append(path.stem)
-                path.write_text(f'name: "{path.stem}"\n{alert}')
-            watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
-            watchdog.write_text(
-                "\n".join(f'- "{name}"' for name in names)
-                + "\ngithub.event.workflow_run.event == 'schedule'\n"
-                + "github.event.workflow_run.conclusion != 'success'\n"
-                + "github.event.workflow_run.conclusion != 'failure'\n"
-                + "workflow-name: ${{ github.event.workflow_run.name }}\n"
-                + "source-run-id: ${{ github.event.workflow_run.id }}\n"
-                + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n"
-            )
-            self.assertEqual(check_scheduled_alerts(root), [])
-
-            first = root / SCHEDULED_ALERT_WORKFLOWS[0]
-            mutations = (
-                ("contains(needs.*.result, 'failure')", "false"),
-                ("issues: write", "issues: read"),
-                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
-                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
-            )
-            for required, replacement in mutations:
-                original = first.read_text()
-                first.write_text(original.replace(required, replacement))
-                self.assertEqual(len(check_scheduled_alerts(root)), 1)
-                first.write_text(original)
-
-            watchdog.write_text(watchdog.read_text().replace(f'- "{names[0]}"\n', ""))
-            self.assertEqual(len(check_scheduled_alerts(root)), 1)
-
 def main() -> int:
     if sys.argv[1:] == ["--self-test"]:
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
@@ -566,7 +436,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
     return 0
 
 

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!