fix(scanner): retain usage across transient peer failures (#6859)

* test(scanner): cover bucket drive guard lifecycle Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): recover usage floor from fenced backups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): retry transient activity probes Retry one failed scanner activity probe after a bounded reconnect when the failure is transport-like or timed out. Keep protocol and response validation failures fail-closed. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): retain post-scan observations Preserve a complete scanner walk as a non-converged observation when the final activity probe is unavailable. Advance the cycle as partial without acknowledging dirty usage.\n\nCo-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): classify publication lease deferrals Distinguish persistence budget and lease deadline deferrals from unavailable activity baselines, and ensure lease-gate deferrals update usage metrics. Keep the fixed lease gate fail-closed while storage-owned commit scope work remains pending. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): recover usage floor from fenced backups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): preserve publication lease defer reasons Keep lease expiry and release failures distinct from activity baseline failures so scanner freshness metrics and cycle outcomes identify the publication barrier that blocked progress. Preserve fail-closed behavior. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): reuse recovered usage baseline for publication Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): fence legacy usage floor fallback Co-Authored-By: heihutu <heihutu@gmail.com> * fix(scanner): use typed activity timeout error --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-09-01 09:48:20 +00:00 · 2026-08-30 02:54:20 +08:00
parent 64cca79fbb
commit c235f7c05d
11 changed files with 783 additions and 60 deletions
@@ -286,6 +286,16 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
 mod tests {
    use super::*;
    use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
+    use tokio::sync::oneshot;
+
+    fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
+        global_metrics()
+            .scanner_runtime_details_report()
+            .active_bucket_drive_scans
+            .into_iter()
+            .find(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
+            .map_or(0, |active| active.count)
+    }

    #[test]
    fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
@@ -305,4 +315,85 @@ mod tests {
                .any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
        );
    }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guard_retires_active_scan_after_cancellation() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_cancel_lifecycle_test__";
+        let drive = "/__guard_cancel_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let cancellation = CancellationToken::new();
+        let worker_cancellation = cancellation.clone();
+        let worker = tokio::spawn(async move {
+            let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            worker_cancellation.cancelled().await;
+            guard.mark_not_failed();
+        });
+
+        cancellation.cancel();
+        worker.await.expect("cancelled scanner worker should finish");
+
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guard_retires_active_scan_when_worker_is_aborted() {
+        let source = ScannerWorkSource::Bitrot;
+        let bucket = "__guard_abort_lifecycle_test__";
+        let drive = "/__guard_abort_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let (started_sender, started_receiver) = oneshot::channel();
+        let worker = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            started_sender.send(()).expect("test should observe worker start");
+            std::future::pending::<()>().await;
+        });
+        started_receiver.await.expect("scanner worker should start");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
+
+        worker.abort();
+        worker.await.expect_err("aborted scanner worker should report cancellation");
+
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guards_track_overlapping_scans_independently() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_overlap_lifecycle_test__";
+        let drive = "/__guard_overlap_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let (first_release_sender, first_release_receiver) = oneshot::channel();
+        let (second_release_sender, second_release_receiver) = oneshot::channel();
+        let (first_started_sender, first_started_receiver) = oneshot::channel();
+        let (second_started_sender, second_started_receiver) = oneshot::channel();
+        let first = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            first_started_sender.send(()).expect("test should observe first worker start");
+            first_release_receiver.await.expect("first worker should be released");
+        });
+        let second = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            second_started_sender
+                .send(())
+                .expect("test should observe second worker start");
+            second_release_receiver.await.expect("second worker should be released");
+        });
+
+        first_started_receiver.await.expect("first scanner worker should start");
+        second_started_receiver.await.expect("second scanner worker should start");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 2);
+
+        first_release_sender.send(()).expect("first worker should be released");
+        first.await.expect("first scanner worker should finish");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
+
+        second_release_sender.send(()).expect("second worker should be released");
+        second.await.expect("second scanner worker should finish");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
 }