feat(scanner): add source-aware maintenance controls (#3461)

* feat(scanner): add source-aware maintenance controls

* fix(scanner): preserve source control between cycles

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-15 12:12:05 +08:00
committed by GitHub
parent f68967de7a
commit 46fb4bdc2f
5 changed files with 572 additions and 14 deletions
+269
View File
@@ -762,6 +762,34 @@ pub struct ScannerSourceWorkSnapshot {
pub missed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceSourceSnapshot {
pub source: String,
pub state: String,
pub reason: String,
pub backlog: u64,
pub current_checked: u64,
pub current_queued: u64,
pub current_missed: u64,
pub lifetime_missed: u64,
pub partial_cycles: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceControlSnapshot {
pub primary_control: String,
pub sources: Vec<ScannerMaintenanceSourceSnapshot>,
}
impl Default for ScannerMaintenanceControlSnapshot {
fn default() -> Self {
Self {
primary_control: SCANNER_MAINTENANCE_CONTROL_NONE.to_string(),
sources: Vec::new(),
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerSourceCycleSnapshot {
pub source: String,
@@ -940,6 +968,8 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub maintenance_control: ScannerMaintenanceControlSnapshot,
#[serde(default)]
pub throttle_idle_mode_enabled: bool,
#[serde(default)]
pub throttle_sleep_factor: f64,
@@ -1090,6 +1120,164 @@ fn scanner_pacing_pressure(metrics: &ScannerMetricsReport) -> ScannerPacingPress
}
}
const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none";
const SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE: &str = "blocked_source";
const SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE: &str = "deferred_source";
const SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE: &str = "active_source";
const SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE: &str = "pacing_pressure";
const SCANNER_MAINTENANCE_STATE_IDLE: &str = "idle";
const SCANNER_MAINTENANCE_STATE_ACTIVE: &str = "active";
const SCANNER_MAINTENANCE_STATE_DEFERRED: &str = "deferred";
const SCANNER_MAINTENANCE_STATE_BLOCKED: &str = "blocked";
const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle";
const SCANNER_MAINTENANCE_REASON_ACTIVE_WORK: &str = "active_work";
const SCANNER_MAINTENANCE_REASON_QUEUED_WORK: &str = "queued_work";
const SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE: &str = "partial_cycle";
const SCANNER_MAINTENANCE_REASON_MISSED_WORK: &str = "missed_work";
const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG: &str = "transition_queue_backlog";
const SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL: &str = "transition_queue_full";
fn scanner_source_work_snapshot(work: &[ScannerSourceWorkSnapshot], source: ScannerWorkSource) -> ScannerSourceWorkSnapshot {
work.iter()
.find(|snapshot| snapshot.source == source.as_str())
.cloned()
.unwrap_or_else(|| ScannerSourceWorkSnapshot {
source: source.as_str().to_string(),
..Default::default()
})
}
fn scanner_source_partial_cycles(metrics: &ScannerMetricsReport, source: ScannerWorkSource) -> u64 {
metrics
.partial_cycles_by_source
.iter()
.find(|snapshot| snapshot.source == source.as_str())
.map(|snapshot| snapshot.cycles)
.unwrap_or_default()
}
fn scanner_source_has_current_work(work: &ScannerSourceWorkSnapshot) -> bool {
work.checked > 0 || work.queued > 0 || work.executed > 0 || work.failed > 0 || work.skipped > 0 || work.missed > 0
}
fn scanner_lifecycle_transition_backlog(metrics: &ScannerMetricsReport) -> u64 {
metrics
.lifecycle_transition
.current_queued
.saturating_add(metrics.lifecycle_transition.current_active)
}
fn scanner_source_maintenance_state(
source: ScannerWorkSource,
current: &ScannerSourceWorkSnapshot,
metrics: &ScannerMetricsReport,
) -> (&'static str, &'static str, u64) {
if current.missed > 0 {
return (SCANNER_MAINTENANCE_STATE_BLOCKED, SCANNER_MAINTENANCE_REASON_MISSED_WORK, current.missed);
}
if source == ScannerWorkSource::Lifecycle {
let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
if metrics.lifecycle_transition.current_queue_capacity > 0
&& metrics.lifecycle_transition.current_queued >= metrics.lifecycle_transition.current_queue_capacity
{
return (
SCANNER_MAINTENANCE_STATE_BLOCKED,
SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_FULL,
transition_backlog,
);
}
}
if metrics.last_cycle_partial_source == source.as_str() && metrics.pacing_pressure.last_cycle_budget_limited {
return (SCANNER_MAINTENANCE_STATE_DEFERRED, SCANNER_MAINTENANCE_REASON_PARTIAL_CYCLE, 0);
}
if current.queued > 0 {
return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_QUEUED_WORK, current.queued);
}
if source == ScannerWorkSource::Lifecycle {
let transition_backlog = scanner_lifecycle_transition_backlog(metrics);
if transition_backlog > 0 {
return (
SCANNER_MAINTENANCE_STATE_ACTIVE,
SCANNER_MAINTENANCE_REASON_TRANSITION_QUEUE_BACKLOG,
transition_backlog,
);
}
}
if scanner_source_has_current_work(current) {
return (SCANNER_MAINTENANCE_STATE_ACTIVE, SCANNER_MAINTENANCE_REASON_ACTIVE_WORK, 0);
}
(SCANNER_MAINTENANCE_STATE_IDLE, SCANNER_MAINTENANCE_REASON_IDLE, 0)
}
fn scanner_maintenance_source_work(metrics: &ScannerMetricsReport) -> &[ScannerSourceWorkSnapshot] {
if metrics.current_cycle_source_work.is_empty() {
&metrics.last_cycle_source_work
} else {
&metrics.current_cycle_source_work
}
}
fn scanner_maintenance_control(metrics: &ScannerMetricsReport) -> ScannerMaintenanceControlSnapshot {
let mut has_blocked = false;
let mut has_deferred = false;
let mut has_active = false;
let current_source_work = scanner_maintenance_source_work(metrics);
let sources = ScannerWorkSource::all()
.iter()
.map(|source| {
let current = scanner_source_work_snapshot(current_source_work, *source);
let lifetime = scanner_source_work_snapshot(&metrics.source_work, *source);
let partial_cycles = scanner_source_partial_cycles(metrics, *source);
let (state, reason, backlog) = scanner_source_maintenance_state(*source, &current, metrics);
match state {
SCANNER_MAINTENANCE_STATE_BLOCKED => has_blocked = true,
SCANNER_MAINTENANCE_STATE_DEFERRED => has_deferred = true,
SCANNER_MAINTENANCE_STATE_ACTIVE => has_active = true,
_ => {}
}
ScannerMaintenanceSourceSnapshot {
source: source.as_str().to_string(),
state: state.to_string(),
reason: reason.to_string(),
backlog,
current_checked: current.checked,
current_queued: current.queued,
current_missed: current.missed,
lifetime_missed: lifetime.missed,
partial_cycles,
}
})
.collect();
let primary_control = if has_blocked {
SCANNER_MAINTENANCE_CONTROL_BLOCKED_SOURCE
} else if has_deferred {
SCANNER_MAINTENANCE_CONTROL_DEFERRED_SOURCE
} else if has_active {
SCANNER_MAINTENANCE_CONTROL_ACTIVE_SOURCE
} else if metrics.pacing_pressure.primary_pressure != SCANNER_MAINTENANCE_CONTROL_NONE {
SCANNER_MAINTENANCE_CONTROL_PACING_PRESSURE
} else {
SCANNER_MAINTENANCE_CONTROL_NONE
};
ScannerMaintenanceControlSnapshot {
primary_control: primary_control.to_string(),
sources,
}
}
fn duration_millis_saturated(duration: Duration) -> u64 {
duration.as_millis().min(u64::MAX as u128) as u64
}
@@ -2163,6 +2351,7 @@ impl Metrics {
}
m.pacing_pressure = scanner_pacing_pressure(&m);
m.maintenance_control = scanner_maintenance_control(&m);
m
}
@@ -2479,6 +2668,86 @@ mod tests {
assert_eq!(report.lifecycle_transition.failed, 1);
}
#[tokio::test]
async fn report_derives_scanner_maintenance_control_status() {
let metrics = Metrics::new();
let start = metrics.start_scan_cycle_work();
metrics.record_scanner_source_checked(ScannerWorkSource::Usage, 3);
metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
metrics.record_scanner_source_queued(ScannerWorkSource::BucketReplication, 1);
metrics.record_scan_cycle_partial_with_source(
Duration::from_secs(5),
ScanCyclePartialReason::Objects,
Some(ScannerWorkSource::Usage),
);
metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
queue_capacity: 4,
queued: 2,
queue_full: 1,
..Default::default()
});
let report = metrics.report().await;
assert_eq!(report.maintenance_control.primary_control, "blocked_source");
let usage = report
.maintenance_control
.sources
.iter()
.find(|source| source.source == ScannerWorkSource::Usage.as_str())
.expect("usage source control should be visible");
assert_eq!(usage.state, "deferred");
assert_eq!(usage.reason, "partial_cycle");
assert_eq!(usage.current_checked, 3);
assert_eq!(usage.partial_cycles, 1);
let lifecycle = report
.maintenance_control
.sources
.iter()
.find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source control should be visible");
assert_eq!(lifecycle.state, "blocked");
assert_eq!(lifecycle.reason, "missed_work");
assert_eq!(lifecycle.current_missed, 2);
assert_eq!(lifecycle.backlog, 2);
let bucket_replication = report
.maintenance_control
.sources
.iter()
.find(|source| source.source == ScannerWorkSource::BucketReplication.as_str())
.expect("bucket replication source control should be visible");
assert_eq!(bucket_replication.state, "active");
assert_eq!(bucket_replication.reason, "queued_work");
assert_eq!(bucket_replication.current_queued, 1);
metrics.finish_scan_cycle_work(start);
}
#[tokio::test]
async fn report_uses_last_cycle_source_work_for_maintenance_control_between_cycles() {
let metrics = Metrics::new();
let start = metrics.start_scan_cycle_work();
metrics.record_scanner_source_missed(ScannerWorkSource::Lifecycle, 2);
metrics.finish_scan_cycle_work(start);
let report = metrics.report().await;
assert!(report.current_cycle_source_work.is_empty());
let lifecycle = report
.maintenance_control
.sources
.iter()
.find(|source| source.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source control should be visible");
assert_eq!(report.maintenance_control.primary_control, "blocked_source");
assert_eq!(lifecycle.state, "blocked");
assert_eq!(lifecycle.reason, "missed_work");
assert_eq!(lifecycle.current_missed, 2);
assert_eq!(lifecycle.backlog, 2);
}
#[tokio::test]
async fn scanner_metrics_update_source_work() {
let metrics = Metrics::new();
+59 -1
View File
@@ -19,7 +19,9 @@ use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_madmin::metrics::{
DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
ScannerCheckpointReport as MadminScannerCheckpointReport,
ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot, ScannerMetrics as MadminScannerMetrics,
ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot,
ScannerMaintenanceControlSnapshot as MadminScannerMaintenanceControlSnapshot,
ScannerMaintenanceSourceSnapshot as MadminScannerMaintenanceSourceSnapshot, ScannerMetrics as MadminScannerMetrics,
ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
TimedAction as MadminTimedAction,
@@ -177,6 +179,25 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
completed: metrics.lifecycle_transition.completed,
failed: metrics.lifecycle_transition.failed,
},
maintenance_control: MadminScannerMaintenanceControlSnapshot {
primary_control: metrics.maintenance_control.primary_control,
sources: metrics
.maintenance_control
.sources
.into_iter()
.map(|source| MadminScannerMaintenanceSourceSnapshot {
source: source.source,
state: source.state,
reason: source.reason,
backlog: source.backlog,
current_checked: source.current_checked,
current_queued: source.current_queued,
current_missed: source.current_missed,
lifetime_missed: source.lifetime_missed,
partial_cycles: source.partial_cycles,
})
.collect(),
},
partial_cycles_by_source: metrics
.partial_cycles_by_source
.into_iter()
@@ -580,6 +601,43 @@ mod test {
assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 7);
}
#[test]
fn scanner_metrics_mapping_preserves_maintenance_control_status() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
maintenance_control: rustfs_common::metrics::ScannerMaintenanceControlSnapshot {
primary_control: "blocked_source".to_string(),
sources: vec![rustfs_common::metrics::ScannerMaintenanceSourceSnapshot {
source: "lifecycle".to_string(),
state: "blocked".to_string(),
reason: "missed_work".to_string(),
backlog: 4,
current_checked: 2,
current_queued: 1,
current_missed: 3,
lifetime_missed: 8,
partial_cycles: 5,
}],
},
..Default::default()
});
assert_eq!(scanner.maintenance_control.primary_control, "blocked_source");
let lifecycle = scanner
.maintenance_control
.sources
.iter()
.find(|source| source.source == "lifecycle")
.expect("lifecycle maintenance control should be mapped");
assert_eq!(lifecycle.state, "blocked");
assert_eq!(lifecycle.reason, "missed_work");
assert_eq!(lifecycle.backlog, 4);
assert_eq!(lifecycle.current_checked, 2);
assert_eq!(lifecycle.current_queued, 1);
assert_eq!(lifecycle.current_missed, 3);
assert_eq!(lifecycle.lifetime_missed, 8);
assert_eq!(lifecycle.partial_cycles, 5);
}
#[test]
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
+186
View File
@@ -158,6 +158,108 @@ pub struct ScannerSourceWorkSnapshot {
pub missed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceSourceSnapshot {
#[serde(rename = "source", default)]
pub source: String,
#[serde(rename = "state", default)]
pub state: String,
#[serde(rename = "reason", default)]
pub reason: String,
#[serde(rename = "backlog", default)]
pub backlog: u64,
#[serde(rename = "current_checked", default)]
pub current_checked: u64,
#[serde(rename = "current_queued", default)]
pub current_queued: u64,
#[serde(rename = "current_missed", default)]
pub current_missed: u64,
#[serde(rename = "lifetime_missed", default)]
pub lifetime_missed: u64,
#[serde(rename = "partial_cycles", default)]
pub partial_cycles: u64,
}
impl ScannerMaintenanceSourceSnapshot {
fn merge(&mut self, other: &Self) {
if scanner_maintenance_state_priority(&other.state) > scanner_maintenance_state_priority(&self.state) {
self.state = other.state.clone();
self.reason = other.reason.clone();
} else if self.reason.is_empty() || self.reason == SCANNER_MAINTENANCE_REASON_IDLE {
self.reason = other.reason.clone();
}
self.backlog = self.backlog.saturating_add(other.backlog);
self.current_checked = self.current_checked.saturating_add(other.current_checked);
self.current_queued = self.current_queued.saturating_add(other.current_queued);
self.current_missed = self.current_missed.saturating_add(other.current_missed);
self.lifetime_missed = self.lifetime_missed.saturating_add(other.lifetime_missed);
self.partial_cycles = self.partial_cycles.saturating_add(other.partial_cycles);
}
}
const SCANNER_MAINTENANCE_CONTROL_NONE: &str = "none";
const SCANNER_MAINTENANCE_REASON_IDLE: &str = "idle";
fn scanner_maintenance_state_priority(state: &str) -> u8 {
match state {
"blocked" => 3,
"deferred" => 2,
"active" => 1,
_ => 0,
}
}
fn scanner_maintenance_control_priority(control: &str) -> u8 {
match control {
"blocked_source" => 4,
"deferred_source" => 3,
"active_source" => 2,
"pacing_pressure" => 1,
_ => 0,
}
}
fn default_scanner_maintenance_control() -> String {
SCANNER_MAINTENANCE_CONTROL_NONE.to_string()
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceControlSnapshot {
#[serde(rename = "primary_control", default = "default_scanner_maintenance_control")]
pub primary_control: String,
#[serde(rename = "sources", default)]
pub sources: Vec<ScannerMaintenanceSourceSnapshot>,
}
impl Default for ScannerMaintenanceControlSnapshot {
fn default() -> Self {
Self {
primary_control: default_scanner_maintenance_control(),
sources: Vec::new(),
}
}
}
impl ScannerMaintenanceControlSnapshot {
fn merge(&mut self, other: &Self) {
if scanner_maintenance_control_priority(&other.primary_control)
> scanner_maintenance_control_priority(&self.primary_control)
{
self.primary_control = other.primary_control.clone();
}
for source in other.sources.iter() {
if let Some(existing) = self.sources.iter_mut().find(|existing| existing.source == source.source) {
existing.merge(source);
} else {
self.sources.push(source.clone());
}
}
self.sources.sort_by(|left, right| left.source.cmp(&right.source));
}
}
impl ScannerSourceWorkSnapshot {
fn merge(&mut self, other: &Self) {
self.checked = self.checked.saturating_add(other.checked);
@@ -424,6 +526,8 @@ pub struct ScannerMetrics {
pub pacing_pressure: ScannerPacingPressureSnapshot,
#[serde(rename = "lifecycle_transition", default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(rename = "maintenance_control", default)]
pub maintenance_control: ScannerMaintenanceControlSnapshot,
#[serde(rename = "throttle_idle_mode_enabled", default)]
pub throttle_idle_mode_enabled: bool,
#[serde(rename = "throttle_sleep_factor", default)]
@@ -496,6 +600,7 @@ impl ScannerMetrics {
self.oldest_active_path_age_seconds = self.oldest_active_path_age_seconds.max(other.oldest_active_path_age_seconds);
self.pacing_pressure.merge(&other.pacing_pressure);
self.lifecycle_transition.merge(&other.lifecycle_transition);
self.maintenance_control.merge(&other.maintenance_control);
self.current_set_scan_concurrency_limit = self
.current_set_scan_concurrency_limit
.saturating_add(other.current_set_scan_concurrency_limit);
@@ -1307,6 +1412,87 @@ mod tests {
assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 26);
}
#[test]
fn scanner_metrics_merge_aggregates_maintenance_control_status() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
maintenance_control: ScannerMaintenanceControlSnapshot {
primary_control: "active_source".to_string(),
sources: vec![ScannerMaintenanceSourceSnapshot {
source: "usage".to_string(),
state: "active".to_string(),
reason: "active_work".to_string(),
backlog: 1,
current_checked: 2,
current_queued: 0,
current_missed: 0,
lifetime_missed: 0,
partial_cycles: 0,
}],
},
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
maintenance_control: ScannerMaintenanceControlSnapshot {
primary_control: "blocked_source".to_string(),
sources: vec![
ScannerMaintenanceSourceSnapshot {
source: "usage".to_string(),
state: "deferred".to_string(),
reason: "partial_cycle".to_string(),
backlog: 3,
current_checked: 5,
current_queued: 1,
current_missed: 0,
lifetime_missed: 0,
partial_cycles: 2,
},
ScannerMaintenanceSourceSnapshot {
source: "lifecycle".to_string(),
state: "blocked".to_string(),
reason: "missed_work".to_string(),
backlog: 4,
current_checked: 0,
current_queued: 0,
current_missed: 4,
lifetime_missed: 9,
partial_cycles: 1,
},
],
},
..Default::default()
});
assert_eq!(scanner.maintenance_control.primary_control, "blocked_source");
let lifecycle = scanner
.maintenance_control
.sources
.iter()
.find(|source| source.source == "lifecycle")
.expect("lifecycle maintenance control should be present");
assert_eq!(lifecycle.state, "blocked");
assert_eq!(lifecycle.reason, "missed_work");
assert_eq!(lifecycle.backlog, 4);
assert_eq!(lifecycle.current_missed, 4);
assert_eq!(lifecycle.lifetime_missed, 9);
let usage = scanner
.maintenance_control
.sources
.iter()
.find(|source| source.source == "usage")
.expect("usage maintenance control should be present");
assert_eq!(usage.state, "deferred");
assert_eq!(usage.reason, "partial_cycle");
assert_eq!(usage.backlog, 4);
assert_eq!(usage.current_checked, 7);
assert_eq!(usage.current_queued, 1);
assert_eq!(usage.partial_cycles, 2);
}
#[test]
fn scanner_metrics_merge_preserves_distributed_status_fields() {
let collected_at = Utc::now();