fix(scanner): harden data scanner integrity handling (#4012)

* fix(scanner): heal metadata scan failures * fix(scanner): preserve dirty buckets after scan failures * feat(scanner): record leader lock liveness * fix(scanner): preserve failed deep scan state * fix(scanner): reject untimestamped stale usage * fix(scanner): report topology-derived admission limit * fix(scanner): accumulate tier usage stats * fix(scanner): guard data usage cache recursion * feat(scanner): expose startup enabled status * fix(scanner): continue after heal admission rejection * fix(scanner): avoid cyclic root usage double count * fix(scanner): preserve dirty markers on cache save failure * feat(scanner): expose leader liveness status * fix(scanner): avoid heal escalation for transient metadata reads * feat(scanner): persist rejected heal retry candidates * test(scanner): satisfy freshness status clippy * test(app): avoid global store reinit in context test * test(app): gate global store helper to tests
2026-08-21 03:46:37 +00:00 · 2026-06-29 08:19:58 +08:00
parent cf7af17b43
commit 0ff261864a
10 changed files with 1312 additions and 105 deletions
@@ -17,6 +17,8 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::current_scanner_metrics_report;
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
+use crate::startup_background::{ENV_SCANNER_ENABLED, scanner_enabled_from_env};
+use chrono::Utc;
 use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
 use matchit::Params;
@@ -31,10 +33,63 @@ const JSON_CONTENT_TYPE: &str = "application/json";

 #[derive(Debug, Serialize)]
 struct ScannerStatusResponse {
+    enabled: bool,
+    disabled_reason: Option<String>,
+    freshness: ScannerFreshnessStatus,
    metrics: ScannerMetricsReport,
    runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
 }

+#[derive(Debug, Serialize)]
+struct ScannerFreshnessStatus {
+    state: &'static str,
+    last_cycle_end_unix_secs: u64,
+    max_expected_age_seconds: u64,
+    reason: Option<&'static str>,
+}
+
+fn scanner_disabled_reason(enabled: bool) -> Option<String> {
+    (!enabled).then(|| format!("disabled by {ENV_SCANNER_ENABLED}"))
+}
+
+fn scanner_freshness_status(
+    metrics: &ScannerMetricsReport,
+    runtime_config: &rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus,
+) -> ScannerFreshnessStatus {
+    const FRESHNESS_MULTIPLIER: u64 = 2;
+
+    let max_expected_age_seconds = runtime_config
+        .cycle_interval_seconds
+        .value
+        .saturating_mul(FRESHNESS_MULTIPLIER);
+    if metrics.last_cycle_end_unix_secs == 0 {
+        return ScannerFreshnessStatus {
+            state: "unknown",
+            last_cycle_end_unix_secs: 0,
+            max_expected_age_seconds,
+            reason: Some("no completed cycle recorded"),
+        };
+    }
+
+    let now = Utc::now().timestamp().max(0) as u64;
+    let age = now.saturating_sub(metrics.last_cycle_end_unix_secs);
+    if max_expected_age_seconds > 0 && age > max_expected_age_seconds {
+        return ScannerFreshnessStatus {
+            state: "stale",
+            last_cycle_end_unix_secs: metrics.last_cycle_end_unix_secs,
+            max_expected_age_seconds,
+            reason: Some("last cycle is older than freshness window"),
+        };
+    }
+
+    ScannerFreshnessStatus {
+        state: "fresh",
+        last_cycle_end_unix_secs: metrics.last_cycle_end_unix_secs,
+        max_expected_age_seconds,
+        reason: None,
+    }
+}
+
 pub fn register_scanner_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
    r.insert(
        Method::GET,
@@ -84,9 +139,16 @@ pub struct ScannerStatusHandler {}
 impl Operation for ScannerStatusHandler {
    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
        let _cred = validate_scanner_status_request(&req).await?;
+        let enabled = scanner_enabled_from_env();
+        let metrics = current_scanner_metrics_report().await;
+        let runtime_config = rustfs_scanner::scanner_runtime_config_status();
+        let freshness = scanner_freshness_status(&metrics, &runtime_config);
        let response = ScannerStatusResponse {
-            metrics: current_scanner_metrics_report().await,
-            runtime_config: rustfs_scanner::scanner_runtime_config_status(),
+            enabled,
+            disabled_reason: scanner_disabled_reason(enabled),
+            freshness,
+            metrics,
+            runtime_config,
        };
        let body = serde_json::to_vec(&response).map_err(|err| {
            S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode scanner status: {err}"))
@@ -95,3 +157,44 @@ impl Operation for ScannerStatusHandler {
        json_response(body)
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn scanner_disabled_reason_reports_startup_env_key() {
+        assert_eq!(scanner_disabled_reason(true), None);
+        assert_eq!(scanner_disabled_reason(false), Some(format!("disabled by {ENV_SCANNER_ENABLED}")));
+    }
+
+    #[test]
+    fn scanner_freshness_reports_unknown_without_cycle_end() {
+        let metrics = ScannerMetricsReport::default();
+        let mut runtime_config = rustfs_scanner::scanner_runtime_config_status();
+        runtime_config.cycle_interval_seconds.value = 60;
+
+        let freshness = scanner_freshness_status(&metrics, &runtime_config);
+
+        assert_eq!(freshness.state, "unknown");
+        assert_eq!(freshness.last_cycle_end_unix_secs, 0);
+        assert_eq!(freshness.max_expected_age_seconds, 120);
+        assert_eq!(freshness.reason, Some("no completed cycle recorded"));
+    }
+
+    #[test]
+    fn scanner_freshness_reports_stale_after_window() {
+        let metrics = ScannerMetricsReport {
+            last_cycle_end_unix_secs: Utc::now().timestamp().max(0) as u64 - 121,
+            ..Default::default()
+        };
+        let mut runtime_config = rustfs_scanner::scanner_runtime_config_status();
+        runtime_config.cycle_interval_seconds.value = 60;
+
+        let freshness = scanner_freshness_status(&metrics, &runtime_config);
+
+        assert_eq!(freshness.state, "stale");
+        assert_eq!(freshness.max_expected_age_seconds, 120);
+        assert_eq!(freshness.reason, Some("last cycle is older than freshness window"));
+    }
+}
@@ -845,6 +845,10 @@ mod tests {
        };
        let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);

+        if let Some(store) = crate::storage::storage_api::ecstore_global::new_object_layer_fn() {
+            return (temp_dir, store, endpoint_pools);
+        }
+
        init_local_disks(endpoint_pools.clone()).await.expect("test local disks");
        let store = ECStore::new(
            "127.0.0.1:0".parse().expect("test addr"),
@@ -22,18 +22,22 @@ use rustfs_utils::get_env_bool_with_aliases;
 use std::{io::Result, sync::Arc};
 use tracing::{debug, info};

-const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
-const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
+pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
+pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
 const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
 const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
 const LOG_COMPONENT_MAIN: &str = "main";
 const LOG_SUBSYSTEM_STARTUP: &str = "startup";
 const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured";

+pub(crate) fn scanner_enabled_from_env() -> bool {
+    get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true)
+}
+
 pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Result<bool> {
    let _ = create_ahm_services_cancel_token();

-    let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true);
+    let enable_scanner = scanner_enabled_from_env();
    let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true);

    info!(
@@ -375,6 +375,8 @@ pub(crate) mod ecstore_event {
 }

 pub(crate) mod ecstore_global {
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::global::new_object_layer_fn;
    pub(crate) use rustfs_ecstore::api::global::{
        GLOBAL_BOOT_TIME, GLOBAL_TierConfigMgr, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt,
        get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, global_rustfs_port,
@@ -27,7 +27,6 @@ const REPAIR_QUEUE_BACKLOG_PRESENT: &str = "repair queue has pending work";
 const REPLICATION_RUNTIME_NOT_INITIALIZED: &str = "replication runtime not initialized";
 const REPLICATION_QUEUE_BACKLOG_PRESENT: &str = "replication queue has pending work";
 const REPLICATION_QUEUE_STATS_UNAVAILABLE: &str = "replication queue stats unavailable";
-const SCANNER_ADMISSION_DISABLED: &str = "scanner admission disabled because max concurrent set scans is zero";
 const SCANNER_ADMISSION_SATURATED: &str = "scanner active work reached configured set-scan limit";
 const SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED: &str = "scanner activity idle or not initialized";
 const STORAGE_CONCURRENCY_PROVIDER_MISSING_FOREGROUND_READ: &str =
@@ -114,9 +113,8 @@ pub fn scanner_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
 }

 fn scanner_workload_admission_snapshot_from_activity(active: u64, limit: usize) -> WorkloadAdmissionSnapshot {
-    let state = if limit == 0 {
-        AdmissionState::Disabled
-    } else if usize::try_from(active).ok().is_some_and(|active| active >= limit) {
+    let effective_limit = if limit == 0 { None } else { Some(limit) };
+    let state = if effective_limit.is_some_and(|limit| usize::try_from(active).ok().is_some_and(|active| active >= limit)) {
        AdmissionState::Saturated
    } else if active > 0 {
        AdmissionState::Open
@@ -127,11 +125,10 @@ fn scanner_workload_admission_snapshot_from_activity(active: u64, limit: usize)
    let snapshot = WorkloadAdmissionSnapshot::new(WorkloadClass::Scanner, state).with_counts(
        Some(u64_to_usize_saturated(active)),
        None,
-        Some(limit),
+        effective_limit,
    );

    match state {
-        AdmissionState::Disabled => snapshot.with_reason(SCANNER_ADMISSION_DISABLED),
        AdmissionState::Saturated => snapshot.with_reason(SCANNER_ADMISSION_SATURATED),
        AdmissionState::Unknown => snapshot.with_reason(SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED),
        _ => snapshot,
@@ -302,13 +299,24 @@ mod tests {
    }

    #[test]
-    fn scanner_snapshot_reports_disabled_when_set_scan_limit_is_zero() {
+    fn scanner_snapshot_treats_zero_set_scan_limit_as_topology_derived() {
        let snapshot = scanner_workload_admission_snapshot_from_activity(0, 0);

        assert_eq!(snapshot.class, WorkloadClass::Scanner);
-        assert_eq!(snapshot.state, AdmissionState::Disabled);
-        assert_eq!(snapshot.limit, Some(0));
-        assert_eq!(snapshot.reason.as_deref(), Some(SCANNER_ADMISSION_DISABLED));
+        assert_eq!(snapshot.state, AdmissionState::Unknown);
+        assert_eq!(snapshot.limit, None);
+        assert_eq!(snapshot.reason.as_deref(), Some(SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED));
+    }
+
+    #[test]
+    fn scanner_snapshot_with_topology_derived_limit_reports_active_work_open() {
+        let snapshot = scanner_workload_admission_snapshot_from_activity(4, 0);
+
+        assert_eq!(snapshot.class, WorkloadClass::Scanner);
+        assert_eq!(snapshot.state, AdmissionState::Open);
+        assert_eq!(snapshot.active, Some(4));
+        assert_eq!(snapshot.limit, None);
+        assert_eq!(snapshot.reason, None);
    }

    #[test]