fix(scanner): make distributed usage convergence authoritative (#5151)

* fix(scanner): make distributed usage cycles authoritative * fix(scanner): close distributed refresh races * fix(config): align scanner reload integration * fix(admin): scope config test helpers * fix(scanner): harden distributed usage convergence * fix(scanner): preserve rolling activity compatibility * fix(admin): expose non-secret optional config values * fix(scanner): acknowledge distributed dirty usage * fix(ecstore): make bucket mutations cancellation safe * fix(scanner): preserve pending dirty acknowledgements * test(obs): account for superseded scanner metric * fix(api): reject excess detached bucket mutations * test: close scanner convergence coverage gaps * fix(scanner): make path tracking cleanup one-shot --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-07-26 08:18:18 +00:00 · 2026-07-25 18:45:16 +08:00
parent 0364523dad
commit a63b79004c
69 changed files with 18073 additions and 1585 deletions
@@ -44,8 +44,9 @@ use crate::metrics::schema::scanner::{
    SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
    SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
    SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
-    SCANNER_PARTIAL_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD,
-    SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
+    SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
+    SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
+    SCANNER_YIELD_EVERY_N_OBJECTS_MD,
 };

 /// Scanner statistics.
@@ -139,7 +140,7 @@ pub struct ScannerStats {
    pub current_cycle_usage_saves: u64,
    /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
    pub current_scan_mode: u64,
-    /// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial
+    /// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded
    pub last_cycle_result: u64,
    /// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
    pub last_cycle_partial_reason: u64,
@@ -177,6 +178,8 @@ pub struct ScannerStats {
    pub last_cycle_usage_saves: u64,
    /// Number of scanner cycles that failed since server start
    pub failed_cycles: u64,
+    /// Number of scanner cycles superseded by concurrent namespace activity
+    pub superseded_cycles: u64,
    /// Number of scanner cycles stopped by runtime budget since server start
    pub partial_cycles: u64,
    /// Number of scanner cycles stopped by an unknown budget reason
@@ -318,6 +321,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
        PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_SUPERSEDED_CYCLES_MD, stats.superseded_cycles as f64),
        PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_MD, stats.partial_cycles as f64),
        PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_unknown as f64)
            .with_label("reason", "unknown"),
@@ -405,6 +409,7 @@ mod tests {
            last_cycle_replication_checks: 12,
            last_cycle_usage_saves: 9,
            failed_cycles: 3,
+            superseded_cycles: 5,
            partial_cycles: 10,
            partial_cycles_unknown: 1,
            partial_cycles_runtime: 2,
@@ -415,7 +420,7 @@ mod tests {
        let metrics = collect_scanner_metrics(&stats);
        report_metrics(&metrics);

-        assert_eq!(metrics.len(), 68);
+        assert_eq!(metrics.len(), 69);

        let objects = metrics.iter().find(|m| m.value == 1000000.0);
        assert!(objects.is_some());
@@ -708,6 +713,11 @@ mod tests {
            .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
        assert_eq!(failed_cycles.map(|m| m.value), Some(3.0));

+        let superseded_cycles = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_SUPERSEDED_CYCLES_MD.get_full_metric_name());
+        assert_eq!(superseded_cycles.map(|m| m.value), Some(5.0));
+
        let partial_cycles = metrics
            .iter()
            .find(|m| m.name == SCANNER_PARTIAL_CYCLES_MD.get_full_metric_name());
@@ -743,7 +753,7 @@ mod tests {
        let stats = ScannerStats::default();
        let metrics = collect_scanner_metrics(&stats);

-        assert_eq!(metrics.len(), 68);
+        assert_eq!(metrics.len(), 69);
        for metric in &metrics {
            assert_eq!(metric.value, 0.0);
            if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() {
@@ -337,6 +337,7 @@ pub enum MetricName {
    ScannerLastCycleReplicationChecks,
    ScannerLastCycleUsageSaves,
    ScannerFailedCycles,
+    ScannerSupersededCycles,
    ScannerPartialCycles,
    ScannerPartialCyclesByReason,

@@ -745,6 +746,7 @@ impl MetricName {
            Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
            Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
            Self::ScannerFailedCycles => "failed_cycles".to_string(),
+            Self::ScannerSupersededCycles => "superseded_cycles".to_string(),
            Self::ScannerPartialCycles => "partial_cycles".to_string(),
            Self::ScannerPartialCyclesByReason => "partial_cycles_by_reason".to_string(),

@@ -416,7 +416,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::
 pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::ScannerLastCycleResult,
-        "Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial.",
+        "Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.",
        &[],
        subsystems::SCANNER,
    )
@@ -584,6 +584,15 @@ pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
    )
 });

+pub static SCANNER_SUPERSEDED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerSupersededCycles,
+        "Total number of clean scanner cycles superseded by concurrent cluster, configuration, topology, or namespace activity since server start.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
 pub static SCANNER_PARTIAL_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_counter_md(
        MetricName::ScannerPartialCycles,
@@ -1135,6 +1135,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
        last_cycle_replication_checks: metrics.last_cycle_replication_checks,
        last_cycle_usage_saves: metrics.last_cycle_usage_saves,
        failed_cycles: metrics.failed_cycles,
+        superseded_cycles: metrics.superseded_cycles,
        partial_cycles: metrics.partial_cycles,
        partial_cycles_unknown: metrics.partial_cycles_unknown,
        partial_cycles_runtime: metrics.partial_cycles_runtime,