diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 32c7c724d..5af6d5396 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -762,6 +762,34 @@ pub struct ScannerSourceWorkSnapshot { pub missed: u64, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerMaintenanceSourceSnapshot { + pub source: String, + pub state: String, + pub reason: String, + pub backlog: u64, + pub current_checked: u64, + pub current_queued: u64, + pub current_missed: u64, + pub lifetime_missed: u64, + pub partial_cycles: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerMaintenanceControlSnapshot { + pub primary_control: String, + pub sources: Vec, +} + +impl Default for ScannerMaintenanceControlSnapshot { + fn default() -> Self { + Self { + primary_control: SCANNER_MAINTENANCE_CONTROL_NONE.to_string(), + sources: Vec::new(), + } + } +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ScannerSourceCycleSnapshot { pub source: String, @@ -940,6 +968,8 @@ pub struct ScannerMetricsReport { #[serde(default)] pub lifecycle_transition: ScannerLifecycleTransitionSnapshot, #[serde(default)] + pub maintenance_control: ScannerMaintenanceControlSnapshot, + #[serde(default)] pub throttle_idle_mode_enabled: bool, #[serde(default)] pub throttle_sleep_factor: f64, @@ -1090,6 +1120,164 @@ fn scanner_pacing_pressure(metrics: &ScannerMetricsReport) -> ScannerPacingPress } } +const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none"; +const SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE: &str = "blocked_source"; +const SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE: &str = "deferred_source"; +const SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE: &str = "active_source"; +const SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE: &str = "pacing_pressure"; + +const SCANNER_MAINTENANCE_STATE_IDLE: &str = "idle"; +const SCANNER_MAINTENANCE_STATE_ACTIVE: &str = "active"; +const SCANNER_MAINTENANCE_STATE_DEFERRED: &str = "deferred"; +const SCANNER_MAINTENANCE_STATE_BLOCKED: &str = "blocked"; + +const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle"; +const SCANNER_MAINTENANCE_REASON_ACTIVE_WORK: &str = "active_work"; +const SCANNER_MAINTENANCE_REASON_QUEUED_WORK: &str = "queued_work"; +const SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE: &str = "partial_cycle"; +const SCANNER_MAINTENANCE_REASON_MISSED_WORK: &str = "missed_work"; +const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG: &str = "transition_queue_backlog"; +const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL: &str = "transition_queue_full"; + +fn scanner_source_work_snapshot(work: &[ScannerSourceWorkSnapshot], source: ScannerWorkSource) -> ScannerSourceWorkSnapshot { + work.iter() + .find(|snapshot| snapshot.source == source.as_str()) + .cloned() + .unwrap_or_else(|| ScannerSourceWorkSnapshot { + source: source.as_str().to_string(), + ..Default::default() + }) +} + +fn scanner_source_partial_cycles(metrics: &ScannerMetricsReport, source: ScannerWorkSource) -> u64 { + metrics + .partial_cycles_by_source + .iter() + .find(|snapshot| snapshot.source == source.as_str()) + .map(|snapshot| snapshot.cycles) + .unwrap_or_default() +} + +fn scanner_source_has_current_work(work: &ScannerSourceWorkSnapshot) -> bool { + work.checked > 0 || work.queued > 0 || work.executed > 0 || work.failed > 0 || work.skipped > 0 || work.missed > 0 +} + +fn scanner_lifecycle_transition_backlog(metrics: &ScannerMetricsReport) -> u64 { + metrics + .lifecycle_transition + .current_queued + .saturating_add(metrics.lifecycle_transition.current_active) +} + +fn scanner_source_maintenance_state( + source: ScannerWorkSource, + current: &ScannerSourceWorkSnapshot, + metrics: &ScannerMetricsReport, +) -> (&'static str, &'static str, u64) { + if current.missed > 0 { + return (SCANNER_MAINTENANCE_STATE_BLOCKED, SCANNER_MAINTENANCE_REASON_MISSED_WORK, current.missed); + } + + if source == ScannerWorkSource::Lifecycle { + let transition_backlog = scanner_lifecycle_transition_backlog(metrics); + if metrics.lifecycle_transition.current_queue_capacity > 0 + && metrics.lifecycle_transition.current_queued >= metrics.lifecycle_transition.current_queue_capacity + { + return ( + SCANNER_MAINTENANCE_STATE_BLOCKED, + SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL, + transition_backlog, + ); + } + } + + if metrics.last_cycle_partial_source == source.as_str() && metrics.pacing_pressure.last_cycle_budget_limited { + return (SCANNER_MAINTENANCE_STATE_DEFERRED, SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE, 0); + } + + if current.queued > 0 { + return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_QUEUED_WORK, current.queued); + } + + if source == ScannerWorkSource::Lifecycle { + let transition_backlog = scanner_lifecycle_transition_backlog(metrics); + if transition_backlog > 0 { + return ( + SCANNER_MAINTENANCE_STATE_ACTIVE, + SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG, + transition_backlog, + ); + } + } + + if scanner_source_has_current_work(current) { + return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_ACTIVE_WORK, 0); + } + + (SCANNER_MAINTENANCE_STATE_IDLE, SCANNER_MAINTENANCE_REASON_IDLE, 0) +} + +fn scanner_maintenance_source_work(metrics: &ScannerMetricsReport) -> &[ScannerSourceWorkSnapshot] { + if metrics.current_cycle_source_work.is_empty() { + &metrics.last_cycle_source_work + } else { + &metrics.current_cycle_source_work + } +} + +fn scanner_maintenance_control(metrics: &ScannerMetricsReport) -> ScannerMaintenanceControlSnapshot { + let mut has_blocked = false; + let mut has_deferred = false; + let mut has_active = false; + let current_source_work = scanner_maintenance_source_work(metrics); + + let sources = ScannerWorkSource::all() + .iter() + .map(|source| { + let current = scanner_source_work_snapshot(current_source_work, *source); + let lifetime = scanner_source_work_snapshot(&metrics.source_work, *source); + let partial_cycles = scanner_source_partial_cycles(metrics, *source); + let (state, reason, backlog) = scanner_source_maintenance_state(*source, ¤t, metrics); + + match state { + SCANNER_MAINTENANCE_STATE_BLOCKED => has_blocked = true, + SCANNER_MAINTENANCE_STATE_DEFERRED => has_deferred = true, + SCANNER_MAINTENANCE_STATE_ACTIVE => has_active = true, + _ => {} + } + + ScannerMaintenanceSourceSnapshot { + source: source.as_str().to_string(), + state: state.to_string(), + reason: reason.to_string(), + backlog, + current_checked: current.checked, + current_queued: current.queued, + current_missed: current.missed, + lifetime_missed: lifetime.missed, + partial_cycles, + } + }) + .collect(); + + let primary_control = if has_blocked { + SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE + } else if has_deferred { + SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE + } else if has_active { + SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE + } else if metrics.pacing_pressure.primary_pressure != SCANNER_MAINTENANCE_CONTROL_NONE { + SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE + } else { + SCANNER_MAINTENANCE_CONTROL_NONE + }; + + ScannerMaintenanceControlSnapshot { + primary_control: primary_control.to_string(), + sources, + } +} + fn duration_millis_saturated(duration: Duration) -> u64 { duration.as_millis().min(u64::MAX as u128) as u64 } @@ -2163,6 +2351,7 @@ impl Metrics { } m.pacing_pressure = scanner_pacing_pressure(&m); + m.maintenance_control = scanner_maintenance_control(&m); m } @@ -2479,6 +2668,86 @@ mod tests { assert_eq!(report.lifecycle_transition.failed, 1); } + #[tokio::test] + async fn report_derives_scanner_maintenance_control_status() { + let metrics = Metrics::new(); + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_checked(ScannerWorkSource::Usage, 3); + metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2); + metrics.record_scanner_source_queued(ScannerWorkSource::BucketReplication, 1); + metrics.record_scan_cycle_partial_with_source( + Duration::from_secs(5), + ScanCyclePartialReason::Objects, + Some(ScannerWorkSource::Usage), + ); + metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate { + queue_capacity: 4, + queued: 2, + queue_full: 1, + ..Default::default() + }); + + let report = metrics.report().await; + + assert_eq!(report.maintenance_control.primary_control, "blocked_source"); + let usage = report + .maintenance_control + .sources + .iter() + .find(|source| source.source == ScannerWorkSource::Usage.as_str()) + .expect("usage source control should be visible"); + assert_eq!(usage.state, "deferred"); + assert_eq!(usage.reason, "partial_cycle"); + assert_eq!(usage.current_checked, 3); + assert_eq!(usage.partial_cycles, 1); + + let lifecycle = report + .maintenance_control + .sources + .iter() + .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str()) + .expect("lifecycle source control should be visible"); + assert_eq!(lifecycle.state, "blocked"); + assert_eq!(lifecycle.reason, "missed_work"); + assert_eq!(lifecycle.current_missed, 2); + assert_eq!(lifecycle.backlog, 2); + + let bucket_replication = report + .maintenance_control + .sources + .iter() + .find(|source| source.source == ScannerWorkSource::BucketReplication.as_str()) + .expect("bucket replication source control should be visible"); + assert_eq!(bucket_replication.state, "active"); + assert_eq!(bucket_replication.reason, "queued_work"); + assert_eq!(bucket_replication.current_queued, 1); + + metrics.finish_scan_cycle_work(start); + } + + #[tokio::test] + async fn report_uses_last_cycle_source_work_for_maintenance_control_between_cycles() { + let metrics = Metrics::new(); + let start = metrics.start_scan_cycle_work(); + metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2); + + metrics.finish_scan_cycle_work(start); + let report = metrics.report().await; + + assert!(report.current_cycle_source_work.is_empty()); + let lifecycle = report + .maintenance_control + .sources + .iter() + .find(|source| source.source == ScannerWorkSource::Lifecycle.as_str()) + .expect("lifecycle source control should be visible"); + assert_eq!(report.maintenance_control.primary_control, "blocked_source"); + assert_eq!(lifecycle.state, "blocked"); + assert_eq!(lifecycle.reason, "missed_work"); + assert_eq!(lifecycle.current_missed, 2); + assert_eq!(lifecycle.backlog, 2); + } + #[tokio::test] async fn scanner_metrics_update_source_work() { let metrics = Metrics::new(); diff --git a/crates/ecstore/src/metrics_realtime.rs b/crates/ecstore/src/metrics_realtime.rs index c60036ddf..c91b150f3 100644 --- a/crates/ecstore/src/metrics_realtime.rs +++ b/crates/ecstore/src/metrics_realtime.rs @@ -19,7 +19,9 @@ use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_madmin::metrics::{ DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics, ScannerCheckpointReport as MadminScannerCheckpointReport, - ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot, ScannerMetrics as MadminScannerMetrics, + ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot, + ScannerMaintenanceControlSnapshot as MadminScannerMaintenanceControlSnapshot, + ScannerMaintenanceSourceSnapshot as MadminScannerMaintenanceSourceSnapshot, ScannerMetrics as MadminScannerMetrics, ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot, ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot, TimedAction as MadminTimedAction, @@ -177,6 +179,25 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo completed: metrics.lifecycle_transition.completed, failed: metrics.lifecycle_transition.failed, }, + maintenance_control: MadminScannerMaintenanceControlSnapshot { + primary_control: metrics.maintenance_control.primary_control, + sources: metrics + .maintenance_control + .sources + .into_iter() + .map(|source| MadminScannerMaintenanceSourceSnapshot { + source: source.source, + state: source.state, + reason: source.reason, + backlog: source.backlog, + current_checked: source.current_checked, + current_queued: source.current_queued, + current_missed: source.current_missed, + lifetime_missed: source.lifetime_missed, + partial_cycles: source.partial_cycles, + }) + .collect(), + }, partial_cycles_by_source: metrics .partial_cycles_by_source .into_iter() @@ -580,6 +601,43 @@ mod test { assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 7); } + #[test] + fn scanner_metrics_mapping_preserves_maintenance_control_status() { + let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport { + maintenance_control: rustfs_common::metrics::ScannerMaintenanceControlSnapshot { + primary_control: "blocked_source".to_string(), + sources: vec![rustfs_common::metrics::ScannerMaintenanceSourceSnapshot { + source: "lifecycle".to_string(), + state: "blocked".to_string(), + reason: "missed_work".to_string(), + backlog: 4, + current_checked: 2, + current_queued: 1, + current_missed: 3, + lifetime_missed: 8, + partial_cycles: 5, + }], + }, + ..Default::default() + }); + + assert_eq!(scanner.maintenance_control.primary_control, "blocked_source"); + let lifecycle = scanner + .maintenance_control + .sources + .iter() + .find(|source| source.source == "lifecycle") + .expect("lifecycle maintenance control should be mapped"); + assert_eq!(lifecycle.state, "blocked"); + assert_eq!(lifecycle.reason, "missed_work"); + assert_eq!(lifecycle.backlog, 4); + assert_eq!(lifecycle.current_checked, 2); + assert_eq!(lifecycle.current_queued, 1); + assert_eq!(lifecycle.current_missed, 3); + assert_eq!(lifecycle.lifetime_missed, 8); + assert_eq!(lifecycle.partial_cycles, 5); + } + #[test] fn scanner_metrics_mapping_preserves_distributed_status_fields() { let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport { diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index b5932f8fc..afdc586eb 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -158,6 +158,108 @@ pub struct ScannerSourceWorkSnapshot { pub missed: u64, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerMaintenanceSourceSnapshot { + #[serde(rename = "source", default)] + pub source: String, + #[serde(rename = "state", default)] + pub state: String, + #[serde(rename = "reason", default)] + pub reason: String, + #[serde(rename = "backlog", default)] + pub backlog: u64, + #[serde(rename = "current_checked", default)] + pub current_checked: u64, + #[serde(rename = "current_queued", default)] + pub current_queued: u64, + #[serde(rename = "current_missed", default)] + pub current_missed: u64, + #[serde(rename = "lifetime_missed", default)] + pub lifetime_missed: u64, + #[serde(rename = "partial_cycles", default)] + pub partial_cycles: u64, +} + +impl ScannerMaintenanceSourceSnapshot { + fn merge(&mut self, other: &Self) { + if scanner_maintenance_state_priority(&other.state) > scanner_maintenance_state_priority(&self.state) { + self.state = other.state.clone(); + self.reason = other.reason.clone(); + } else if self.reason.is_empty() || self.reason == SCANNER_MAINTENANCE_REASON_IDLE { + self.reason = other.reason.clone(); + } + + self.backlog = self.backlog.saturating_add(other.backlog); + self.current_checked = self.current_checked.saturating_add(other.current_checked); + self.current_queued = self.current_queued.saturating_add(other.current_queued); + self.current_missed = self.current_missed.saturating_add(other.current_missed); + self.lifetime_missed = self.lifetime_missed.saturating_add(other.lifetime_missed); + self.partial_cycles = self.partial_cycles.saturating_add(other.partial_cycles); + } +} + +const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none"; +const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle"; + +fn scanner_maintenance_state_priority(state: &str) -> u8 { + match state { + "blocked" => 3, + "deferred" => 2, + "active" => 1, + _ => 0, + } +} + +fn scanner_maintenance_control_priority(control: &str) -> u8 { + match control { + "blocked_source" => 4, + "deferred_source" => 3, + "active_source" => 2, + "pacing_pressure" => 1, + _ => 0, + } +} + +fn default_scanner_maintenance_control() -> String { + SCANNER_MAINTENANCE_CONTROL_NONE.to_string() +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerMaintenanceControlSnapshot { + #[serde(rename = "primary_control", default = "default_scanner_maintenance_control")] + pub primary_control: String, + #[serde(rename = "sources", default)] + pub sources: Vec, +} + +impl Default for ScannerMaintenanceControlSnapshot { + fn default() -> Self { + Self { + primary_control: default_scanner_maintenance_control(), + sources: Vec::new(), + } + } +} + +impl ScannerMaintenanceControlSnapshot { + fn merge(&mut self, other: &Self) { + if scanner_maintenance_control_priority(&other.primary_control) + > scanner_maintenance_control_priority(&self.primary_control) + { + self.primary_control = other.primary_control.clone(); + } + + for source in other.sources.iter() { + if let Some(existing) = self.sources.iter_mut().find(|existing| existing.source == source.source) { + existing.merge(source); + } else { + self.sources.push(source.clone()); + } + } + self.sources.sort_by(|left, right| left.source.cmp(&right.source)); + } +} + impl ScannerSourceWorkSnapshot { fn merge(&mut self, other: &Self) { self.checked = self.checked.saturating_add(other.checked); @@ -424,6 +526,8 @@ pub struct ScannerMetrics { pub pacing_pressure: ScannerPacingPressureSnapshot, #[serde(rename = "lifecycle_transition", default)] pub lifecycle_transition: ScannerLifecycleTransitionSnapshot, + #[serde(rename = "maintenance_control", default)] + pub maintenance_control: ScannerMaintenanceControlSnapshot, #[serde(rename = "throttle_idle_mode_enabled", default)] pub throttle_idle_mode_enabled: bool, #[serde(rename = "throttle_sleep_factor", default)] @@ -496,6 +600,7 @@ impl ScannerMetrics { self.oldest_active_path_age_seconds = self.oldest_active_path_age_seconds.max(other.oldest_active_path_age_seconds); self.pacing_pressure.merge(&other.pacing_pressure); self.lifecycle_transition.merge(&other.lifecycle_transition); + self.maintenance_control.merge(&other.maintenance_control); self.current_set_scan_concurrency_limit = self .current_set_scan_concurrency_limit .saturating_add(other.current_set_scan_concurrency_limit); @@ -1307,6 +1412,87 @@ mod tests { assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 26); } + #[test] + fn scanner_metrics_merge_aggregates_maintenance_control_status() { + let collected_at = Utc::now(); + let mut scanner = ScannerMetrics { + collected_at, + maintenance_control: ScannerMaintenanceControlSnapshot { + primary_control: "active_source".to_string(), + sources: vec![ScannerMaintenanceSourceSnapshot { + source: "usage".to_string(), + state: "active".to_string(), + reason: "active_work".to_string(), + backlog: 1, + current_checked: 2, + current_queued: 0, + current_missed: 0, + lifetime_missed: 0, + partial_cycles: 0, + }], + }, + ..Default::default() + }; + + scanner.merge(&ScannerMetrics { + collected_at: collected_at + chrono::Duration::seconds(1), + maintenance_control: ScannerMaintenanceControlSnapshot { + primary_control: "blocked_source".to_string(), + sources: vec![ + ScannerMaintenanceSourceSnapshot { + source: "usage".to_string(), + state: "deferred".to_string(), + reason: "partial_cycle".to_string(), + backlog: 3, + current_checked: 5, + current_queued: 1, + current_missed: 0, + lifetime_missed: 0, + partial_cycles: 2, + }, + ScannerMaintenanceSourceSnapshot { + source: "lifecycle".to_string(), + state: "blocked".to_string(), + reason: "missed_work".to_string(), + backlog: 4, + current_checked: 0, + current_queued: 0, + current_missed: 4, + lifetime_missed: 9, + partial_cycles: 1, + }, + ], + }, + ..Default::default() + }); + + assert_eq!(scanner.maintenance_control.primary_control, "blocked_source"); + let lifecycle = scanner + .maintenance_control + .sources + .iter() + .find(|source| source.source == "lifecycle") + .expect("lifecycle maintenance control should be present"); + assert_eq!(lifecycle.state, "blocked"); + assert_eq!(lifecycle.reason, "missed_work"); + assert_eq!(lifecycle.backlog, 4); + assert_eq!(lifecycle.current_missed, 4); + assert_eq!(lifecycle.lifetime_missed, 9); + + let usage = scanner + .maintenance_control + .sources + .iter() + .find(|source| source.source == "usage") + .expect("usage maintenance control should be present"); + assert_eq!(usage.state, "deferred"); + assert_eq!(usage.reason, "partial_cycle"); + assert_eq!(usage.backlog, 4); + assert_eq!(usage.current_checked, 7); + assert_eq!(usage.current_queued, 1); + assert_eq!(usage.partial_cycles, 2); + } + #[test] fn scanner_metrics_merge_preserves_distributed_status_fields() { let collected_at = Utc::now(); diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 91459916b..04ddf0343 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -282,6 +282,8 @@ Compare these fields between baseline and tuned runs: | `runtime_config.*.value` and `runtime_config.*.source` | Confirms the tested settings actually took effect. | | `metrics.pacing_pressure.primary_pressure` | Shows whether pressure is from queues, budgets, pause activity, active scans, or no scanner pressure. | | `metrics.pacing_pressure.last_cycle_total_pause_ratio` | Shows how much of the last cycle was cooperative scanner pause time. | +| `metrics.maintenance_control.primary_control` | Shows whether source-level maintenance is blocked, deferred, active, only pacing-limited, or idle. | +| `metrics.maintenance_control.sources` | Shows the source, state, reason, backlog, current or last-cycle missed work, and partial-cycle count for each scanner maintenance source. | | `metrics.current_cycle_objects_scanned` | Confirms object scan progress during the current cycle. | | `metrics.current_cycle_directories_scanned` | Confirms directory walk progress during the current cycle. | | `metrics.last_cycle_result` | Confirms whether the previous cycle completed, stopped partially, or failed. | diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index f65eafe5e..bbb3d01c5 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -97,6 +97,7 @@ metrics.pacing_pressure.primary_pressure metrics.pacing_pressure.last_cycle_budget_limited metrics.lifecycle_transition.current_queued metrics.lifecycle_transition.scanner_missed +metrics.maintenance_control.primary_control metrics.source_work metrics.scan_checkpoint ``` @@ -146,6 +147,45 @@ Use these counters to decide whether scan progress is limited by scanner pacing or by a downstream subsystem such as lifecycle transition, replication repair, or heal admission. +## Reading Maintenance Control + +`metrics.maintenance_control` derives a source-level control snapshot from +scanner pacing, partial-cycle state, source work, and lifecycle transition +queue state. It does not change scanner scheduling by itself; it explains why a +source is moving, deferred, or blocked. When no scan cycle is currently active, +source-work controls use the last completed cycle so recently missed work stays +visible between scanner passes. + +`metrics.maintenance_control.primary_control` summarizes the highest-priority +source state: + +| Value | Meaning | +|---|---| +| `blocked_source` | At least one maintenance source found work that could not be admitted or is blocked by a downstream queue. | +| `deferred_source` | At least one source was deferred by a partial scanner cycle or budget-limited pass. | +| `active_source` | At least one source has current-cycle work or queued downstream work. | +| `pacing_pressure` | No source-specific state dominated, but scanner pacing pressure is still visible. | +| `none` | No source-level maintenance control pressure was observed. | + +Each `metrics.maintenance_control.sources[]` entry has: + +| Field | Meaning | +|---|---| +| `source` | Scanner source such as `usage`, `lifecycle`, `bucket_replication`, `site_replication`, `heal`, `bitrot`, or `alerts`. | +| `state` | `idle`, `active`, `deferred`, or `blocked`. | +| `reason` | Derived reason such as `active_work`, `queued_work`, `partial_cycle`, `missed_work`, `transition_queue_backlog`, or `transition_queue_full`. | +| `backlog` | Current source-level backlog estimate from queued or missed work. | +| `current_checked` | Current-cycle checked work for this source, or the last completed cycle when no scan cycle is active. | +| `current_queued` | Current-cycle queued work for this source, or the last completed cycle when no scan cycle is active. | +| `current_missed` | Current-cycle work that could not be admitted, or the last completed cycle when no scan cycle is active. | +| `lifetime_missed` | Lifetime missed work counter for context. | +| `partial_cycles` | Partial cycles attributed to this source. | + +Use this snapshot before changing scanner controls. For example, +`blocked_source` with `lifecycle/missed_work` points at downstream lifecycle +admission, while `deferred_source` with `usage/partial_cycle` points at scanner +cycle budgets. + ## Reading Distributed Metrics `/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the @@ -169,11 +209,12 @@ done ``` The `aggregated.scanner` payload preserves the same scanner progress, -checkpoint, pacing, source work, and lifecycle transition fields used by the -local scanner status, but only for the node that returned the response. The -`by_host.*.scanner` payload keeps that node's host view. Compare the per-node -artifacts externally to find old active paths, partial checkpoints, pacing -pressure, or downstream queue admission problems across the deployment. +checkpoint, pacing, source work, maintenance control, and lifecycle transition +fields used by the local scanner status, but only for the node that returned +the response. The `by_host.*.scanner` payload keeps that node's host view. +Compare the per-node artifacts externally to find old active paths, partial +checkpoints, pacing pressure, source-level control pressure, or downstream +queue admission problems across the deployment. ## Reading Lifecycle Transition Status @@ -206,21 +247,23 @@ sustained CPU usage while the scanner is enabled: 1. Read `/v3/scanner/status`. 2. Check `metrics.pacing_pressure.primary_pressure`. -3. Check `runtime_config.delay`, `runtime_config.max_wait_seconds`, and +3. Check `metrics.maintenance_control.primary_control` and source entries + before changing runtime controls. +4. Check `runtime_config.delay`, `runtime_config.max_wait_seconds`, and `runtime_config.cycle_interval_seconds` to confirm the active values and their sources. -4. Check `metrics.current_cycle_objects_scanned`, +5. Check `metrics.current_cycle_objects_scanned`, `metrics.current_cycle_directories_scanned`, and active paths to confirm the scanner is the active work. -5. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise +6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise `scanner.delay` first. -6. If individual sleeps are too short, raise `scanner.max_wait`. -7. If each scan cycle finishes but starts too often, raise `scanner.cycle`. -8. If scans must be broken into bounded chunks, set one of the cycle budgets: +7. If individual sleeps are too short, raise `scanner.max_wait`. +8. If each scan cycle finishes but starts too often, raise `scanner.cycle`. +9. If scans must be broken into bounded chunks, set one of the cycle budgets: `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`. -9. Recheck `pacing_pressure`, source work, and lifecycle transition status after - one or more scanner cycles. +10. Recheck `pacing_pressure`, `maintenance_control`, source work, and + lifecycle transition status after one or more scanner cycles. Do not rely only on a longer cycle interval if lifecycle, replication, heal, or bitrot work must keep moving. Use source work and transition status to confirm