feat(scanner): add source-aware maintenance controls (#3461)

* feat(scanner): add source-aware maintenance controls * fix(scanner): preserve source control between cycles --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-08-09 06:39:25 +00:00 · 2026-06-15 12:12:05 +08:00
parent f68967de7a
commit 46fb4bdc2f
5 changed files with 572 additions and 14 deletions
@@ -158,6 +158,108 @@ pub struct ScannerSourceWorkSnapshot {
    pub missed: u64,
 }

+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScannerMaintenanceSourceSnapshot {
+    #[serde(rename = "source", default)]
+    pub source: String,
+    #[serde(rename = "state", default)]
+    pub state: String,
+    #[serde(rename = "reason", default)]
+    pub reason: String,
+    #[serde(rename = "backlog", default)]
+    pub backlog: u64,
+    #[serde(rename = "current_checked", default)]
+    pub current_checked: u64,
+    #[serde(rename = "current_queued", default)]
+    pub current_queued: u64,
+    #[serde(rename = "current_missed", default)]
+    pub current_missed: u64,
+    #[serde(rename = "lifetime_missed", default)]
+    pub lifetime_missed: u64,
+    #[serde(rename = "partial_cycles", default)]
+    pub partial_cycles: u64,
+}
+
+impl ScannerMaintenanceSourceSnapshot {
+    fn merge(&mut self, other: &Self) {
+        if scanner_maintenance_state_priority(&other.state) > scanner_maintenance_state_priority(&self.state) {
+            self.state = other.state.clone();
+            self.reason = other.reason.clone();
+        } else if self.reason.is_empty() || self.reason == SCANNER_MAINTENANCE_REASON_IDLE {
+            self.reason = other.reason.clone();
+        }
+
+        self.backlog = self.backlog.saturating_add(other.backlog);
+        self.current_checked = self.current_checked.saturating_add(other.current_checked);
+        self.current_queued = self.current_queued.saturating_add(other.current_queued);
+        self.current_missed = self.current_missed.saturating_add(other.current_missed);
+        self.lifetime_missed = self.lifetime_missed.saturating_add(other.lifetime_missed);
+        self.partial_cycles = self.partial_cycles.saturating_add(other.partial_cycles);
+    }
+}
+
+const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none";
+const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle";
+
+fn scanner_maintenance_state_priority(state: &str) -> u8 {
+    match state {
+        "blocked" => 3,
+        "deferred" => 2,
+        "active" => 1,
+        _ => 0,
+    }
+}
+
+fn scanner_maintenance_control_priority(control: &str) -> u8 {
+    match control {
+        "blocked_source" => 4,
+        "deferred_source" => 3,
+        "active_source" => 2,
+        "pacing_pressure" => 1,
+        _ => 0,
+    }
+}
+
+fn default_scanner_maintenance_control() -> String {
+    SCANNER_MAINTENANCE_CONTROL_NONE.to_string()
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScannerMaintenanceControlSnapshot {
+    #[serde(rename = "primary_control", default = "default_scanner_maintenance_control")]
+    pub primary_control: String,
+    #[serde(rename = "sources", default)]
+    pub sources: Vec<ScannerMaintenanceSourceSnapshot>,
+}
+
+impl Default for ScannerMaintenanceControlSnapshot {
+    fn default() -> Self {
+        Self {
+            primary_control: default_scanner_maintenance_control(),
+            sources: Vec::new(),
+        }
+    }
+}
+
+impl ScannerMaintenanceControlSnapshot {
+    fn merge(&mut self, other: &Self) {
+        if scanner_maintenance_control_priority(&other.primary_control)
+            > scanner_maintenance_control_priority(&self.primary_control)
+        {
+            self.primary_control = other.primary_control.clone();
+        }
+
+        for source in other.sources.iter() {
+            if let Some(existing) = self.sources.iter_mut().find(|existing| existing.source == source.source) {
+                existing.merge(source);
+            } else {
+                self.sources.push(source.clone());
+            }
+        }
+        self.sources.sort_by(|left, right| left.source.cmp(&right.source));
+    }
+}
+
 impl ScannerSourceWorkSnapshot {
    fn merge(&mut self, other: &Self) {
        self.checked = self.checked.saturating_add(other.checked);
@@ -424,6 +526,8 @@ pub struct ScannerMetrics {
    pub pacing_pressure: ScannerPacingPressureSnapshot,
    #[serde(rename = "lifecycle_transition", default)]
    pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
+    #[serde(rename = "maintenance_control", default)]
+    pub maintenance_control: ScannerMaintenanceControlSnapshot,
    #[serde(rename = "throttle_idle_mode_enabled", default)]
    pub throttle_idle_mode_enabled: bool,
    #[serde(rename = "throttle_sleep_factor", default)]
@@ -496,6 +600,7 @@ impl ScannerMetrics {
        self.oldest_active_path_age_seconds = self.oldest_active_path_age_seconds.max(other.oldest_active_path_age_seconds);
        self.pacing_pressure.merge(&other.pacing_pressure);
        self.lifecycle_transition.merge(&other.lifecycle_transition);
+        self.maintenance_control.merge(&other.maintenance_control);
        self.current_set_scan_concurrency_limit = self
            .current_set_scan_concurrency_limit
            .saturating_add(other.current_set_scan_concurrency_limit);
@@ -1307,6 +1412,87 @@ mod tests {
        assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 26);
    }

+    #[test]
+    fn scanner_metrics_merge_aggregates_maintenance_control_status() {
+        let collected_at = Utc::now();
+        let mut scanner = ScannerMetrics {
+            collected_at,
+            maintenance_control: ScannerMaintenanceControlSnapshot {
+                primary_control: "active_source".to_string(),
+                sources: vec![ScannerMaintenanceSourceSnapshot {
+                    source: "usage".to_string(),
+                    state: "active".to_string(),
+                    reason: "active_work".to_string(),
+                    backlog: 1,
+                    current_checked: 2,
+                    current_queued: 0,
+                    current_missed: 0,
+                    lifetime_missed: 0,
+                    partial_cycles: 0,
+                }],
+            },
+            ..Default::default()
+        };
+
+        scanner.merge(&ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            maintenance_control: ScannerMaintenanceControlSnapshot {
+                primary_control: "blocked_source".to_string(),
+                sources: vec![
+                    ScannerMaintenanceSourceSnapshot {
+                        source: "usage".to_string(),
+                        state: "deferred".to_string(),
+                        reason: "partial_cycle".to_string(),
+                        backlog: 3,
+                        current_checked: 5,
+                        current_queued: 1,
+                        current_missed: 0,
+                        lifetime_missed: 0,
+                        partial_cycles: 2,
+                    },
+                    ScannerMaintenanceSourceSnapshot {
+                        source: "lifecycle".to_string(),
+                        state: "blocked".to_string(),
+                        reason: "missed_work".to_string(),
+                        backlog: 4,
+                        current_checked: 0,
+                        current_queued: 0,
+                        current_missed: 4,
+                        lifetime_missed: 9,
+                        partial_cycles: 1,
+                    },
+                ],
+            },
+            ..Default::default()
+        });
+
+        assert_eq!(scanner.maintenance_control.primary_control, "blocked_source");
+        let lifecycle = scanner
+            .maintenance_control
+            .sources
+            .iter()
+            .find(|source| source.source == "lifecycle")
+            .expect("lifecycle maintenance control should be present");
+        assert_eq!(lifecycle.state, "blocked");
+        assert_eq!(lifecycle.reason, "missed_work");
+        assert_eq!(lifecycle.backlog, 4);
+        assert_eq!(lifecycle.current_missed, 4);
+        assert_eq!(lifecycle.lifetime_missed, 9);
+
+        let usage = scanner
+            .maintenance_control
+            .sources
+            .iter()
+            .find(|source| source.source == "usage")
+            .expect("usage maintenance control should be present");
+        assert_eq!(usage.state, "deferred");
+        assert_eq!(usage.reason, "partial_cycle");
+        assert_eq!(usage.backlog, 4);
+        assert_eq!(usage.current_checked, 7);
+        assert_eq!(usage.current_queued, 1);
+        assert_eq!(usage.partial_cycles, 2);
+    }
+
    #[test]
    fn scanner_metrics_merge_preserves_distributed_status_fields() {
        let collected_at = Utc::now();