feat(scanner): expose replication repair kind metrics (#3476)

This commit is contained in:
Henry Guo
2026-06-15 16:52:39 +08:00
committed by GitHub
parent 5063524b36
commit d28aedfbb3
6 changed files with 597 additions and 13 deletions
+251
View File
@@ -309,6 +309,62 @@ impl ScannerWorkSource {
} }
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScannerReplicationRepairKind {
BucketObject,
BucketDeleteMarker,
BucketVersionPurge,
BucketExistingObject,
SitePassiveRequeue,
SiteActiveResync,
}
impl ScannerReplicationRepairKind {
const ALL: [Self; 6] = [
Self::BucketObject,
Self::BucketDeleteMarker,
Self::BucketVersionPurge,
Self::BucketExistingObject,
Self::SitePassiveRequeue,
Self::SiteActiveResync,
];
pub fn source(self) -> ScannerWorkSource {
match self {
Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
ScannerWorkSource::BucketReplication
}
Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerWorkSource::SiteReplication,
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::BucketObject => "object",
Self::BucketDeleteMarker => "delete_marker",
Self::BucketVersionPurge => "version_purge",
Self::BucketExistingObject => "existing_object",
Self::SitePassiveRequeue => "passive_requeue",
Self::SiteActiveResync => "active_resync",
}
}
fn all() -> &'static [Self] {
&Self::ALL
}
fn index(self) -> usize {
match self {
Self::BucketObject => 0,
Self::BucketDeleteMarker => 1,
Self::BucketVersionPurge => 2,
Self::BucketExistingObject => 3,
Self::SitePassiveRequeue => 4,
Self::SiteActiveResync => 5,
}
}
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct ScannerSourceWorkCounters { struct ScannerSourceWorkCounters {
checked: AtomicU64, checked: AtomicU64,
@@ -454,6 +510,19 @@ impl ScannerSourceWorkValues {
missed: self.missed, missed: self.missed,
} }
} }
fn replication_repair_snapshot(self, kind: ScannerReplicationRepairKind) -> ScannerReplicationRepairSnapshot {
ScannerReplicationRepairSnapshot {
source: kind.source().as_str().to_string(),
kind: kind.as_str().to_string(),
checked: self.checked,
queued: self.queued,
executed: self.executed,
failed: self.failed,
skipped: self.skipped,
missed: self.missed,
}
}
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -687,6 +756,9 @@ pub struct Metrics {
scanner_source_work: Vec<ScannerSourceWorkCounters>, scanner_source_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>, current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>, last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
scanner_replication_repair_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_replication_repair_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_replication_repair_work: Vec<ScannerSourceWorkCounters>,
partial_scan_cycles: AtomicU64, partial_scan_cycles: AtomicU64,
} }
@@ -781,6 +853,19 @@ pub struct ScannerSourceWorkSnapshot {
pub missed: u64, pub missed: u64,
} }
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerReplicationRepairSnapshot {
pub source: String,
pub kind: String,
pub checked: u64,
pub queued: u64,
pub executed: u64,
pub failed: u64,
pub skipped: u64,
#[serde(default)]
pub missed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceSourceSnapshot { pub struct ScannerMaintenanceSourceSnapshot {
pub source: String, pub source: String,
@@ -1050,6 +1135,12 @@ pub struct ScannerMetricsReport {
#[serde(default)] #[serde(default)]
pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>, pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
#[serde(default)] #[serde(default)]
pub replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(default)]
pub current_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(default)]
pub last_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(default)]
pub partial_cycles: u64, pub partial_cycles: u64,
} }
@@ -1549,6 +1640,18 @@ impl Metrics {
.iter() .iter()
.map(|_| ScannerSourceWorkCounters::default()) .map(|_| ScannerSourceWorkCounters::default())
.collect(), .collect(),
scanner_replication_repair_work: ScannerReplicationRepairKind::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
.collect(),
current_scan_cycle_replication_repair_work_start: ScannerReplicationRepairKind::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
.collect(),
last_scan_cycle_replication_repair_work: ScannerReplicationRepairKind::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
.collect(),
partial_scan_cycles: AtomicU64::new(0), partial_scan_cycles: AtomicU64::new(0),
} }
} }
@@ -1812,6 +1915,12 @@ impl Metrics {
} }
} }
pub fn record_scanner_replication_repair_work(&self, kind: ScannerReplicationRepairKind, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.scanner_replication_repair_work.get(kind.index()) {
counters.add(work);
}
}
fn record_last_cycle_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) { fn record_last_cycle_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.last_scan_cycle_source_work.get(source.index()) { if let Some(counters) = self.last_scan_cycle_source_work.get(source.index()) {
counters.add(work); counters.add(work);
@@ -2060,6 +2169,7 @@ impl Metrics {
pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot { pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
let snapshot = self.scan_cycle_work_snapshot(); let snapshot = self.scan_cycle_work_snapshot();
let source_snapshot = self.scanner_source_work_values(); let source_snapshot = self.scanner_source_work_values();
let replication_repair_snapshot = self.scanner_replication_repair_work_values();
self.current_scan_cycle_objects_start self.current_scan_cycle_objects_start
.store(snapshot.objects_scanned, Ordering::Relaxed); .store(snapshot.objects_scanned, Ordering::Relaxed);
self.current_scan_cycle_directories_start self.current_scan_cycle_directories_start
@@ -2089,6 +2199,10 @@ impl Metrics {
self.current_scan_cycle_usage_saves_start self.current_scan_cycle_usage_saves_start
.store(snapshot.usage_saves, Ordering::Relaxed); .store(snapshot.usage_saves, Ordering::Relaxed);
self.store_scanner_source_work_values(&self.current_scan_cycle_source_work_start, &source_snapshot); self.store_scanner_source_work_values(&self.current_scan_cycle_source_work_start, &source_snapshot);
self.store_scanner_replication_repair_work_values(
&self.current_scan_cycle_replication_repair_work_start,
&replication_repair_snapshot,
);
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed); self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
snapshot snapshot
} }
@@ -2096,8 +2210,11 @@ impl Metrics {
pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) { pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
let work = self.scan_cycle_work_since(start); let work = self.scan_cycle_work_since(start);
let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values()); let source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
self.record_scan_cycle_work(work); self.record_scan_cycle_work(work);
self.record_scan_cycle_source_work(&source_work); self.record_scan_cycle_source_work(&source_work);
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed); self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
} }
@@ -2213,6 +2330,64 @@ impl Metrics {
.collect() .collect()
} }
fn scanner_replication_repair_work_values(&self) -> Vec<ScannerSourceWorkValues> {
ScannerReplicationRepairKind::all()
.iter()
.filter_map(|kind| {
self.scanner_replication_repair_work
.get(kind.index())
.map(ScannerSourceWorkCounters::values)
})
.collect()
}
fn current_scan_cycle_replication_repair_work_start_values(&self) -> Vec<ScannerSourceWorkValues> {
ScannerReplicationRepairKind::all()
.iter()
.filter_map(|kind| {
self.current_scan_cycle_replication_repair_work_start
.get(kind.index())
.map(ScannerSourceWorkCounters::values)
})
.collect()
}
fn scanner_replication_repair_work_since(&self, start: &[ScannerSourceWorkValues]) -> Vec<ScannerSourceWorkValues> {
self.scanner_replication_repair_work_values()
.into_iter()
.enumerate()
.map(|(index, current)| current.saturating_sub(start.get(index).copied().unwrap_or_default()))
.collect()
}
fn scanner_replication_repair_work_snapshots(
&self,
values: &[ScannerSourceWorkValues],
) -> Vec<ScannerReplicationRepairSnapshot> {
ScannerReplicationRepairKind::all()
.iter()
.filter_map(|kind| {
values
.get(kind.index())
.map(|values| values.replication_repair_snapshot(*kind))
})
.collect()
}
fn scanner_replication_repair_work_counter_snapshots(
&self,
counters: &[ScannerSourceWorkCounters],
) -> Vec<ScannerReplicationRepairSnapshot> {
ScannerReplicationRepairKind::all()
.iter()
.filter_map(|kind| {
counters
.get(kind.index())
.map(|counters| counters.values().replication_repair_snapshot(*kind))
})
.collect()
}
fn store_scanner_source_work_values(&self, counters: &[ScannerSourceWorkCounters], values: &[ScannerSourceWorkValues]) { fn store_scanner_source_work_values(&self, counters: &[ScannerSourceWorkCounters], values: &[ScannerSourceWorkValues]) {
for source in ScannerWorkSource::all() { for source in ScannerWorkSource::all() {
if let (Some(counter), Some(values)) = (counters.get(source.index()), values.get(source.index())) { if let (Some(counter), Some(values)) = (counters.get(source.index()), values.get(source.index())) {
@@ -2221,6 +2396,18 @@ impl Metrics {
} }
} }
fn store_scanner_replication_repair_work_values(
&self,
counters: &[ScannerSourceWorkCounters],
values: &[ScannerSourceWorkValues],
) {
for kind in ScannerReplicationRepairKind::all() {
if let (Some(counter), Some(values)) = (counters.get(kind.index()), values.get(kind.index())) {
counter.store(*values);
}
}
}
pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) { pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
// Telemetry-only gauges: readers may observe a transient mixed snapshot // Telemetry-only gauges: readers may observe a transient mixed snapshot
// while these independent atomic fields are updated. // while these independent atomic fields are updated.
@@ -2254,6 +2441,10 @@ impl Metrics {
self.store_scanner_source_work_values(&self.last_scan_cycle_source_work, work); self.store_scanner_source_work_values(&self.last_scan_cycle_source_work, work);
} }
fn record_scan_cycle_replication_repair_work(&self, work: &[ScannerSourceWorkValues]) {
self.store_scanner_replication_repair_work_values(&self.last_scan_cycle_replication_repair_work, work);
}
/// Snapshot of every path currently being scanned. /// Snapshot of every path currently being scanned.
pub async fn get_current_paths(&self) -> Vec<String> { pub async fn get_current_paths(&self) -> Vec<String> {
self.current_path_snapshots() self.current_path_snapshots()
@@ -2326,6 +2517,8 @@ impl Metrics {
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) { if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start()); let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values()); let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
m.current_cycle_objects_scanned = current_work.objects_scanned; m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned; m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans; m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
@@ -2341,6 +2534,7 @@ impl Metrics {
m.current_cycle_replication_checks = current_work.replication_checks; m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves; m.current_cycle_usage_saves = current_work.usage_saves;
m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work); m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
} }
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed); let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string(); m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
@@ -2371,6 +2565,8 @@ impl Metrics {
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed); m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed); m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
m.last_cycle_source_work = self.scanner_source_work_counter_snapshots(&self.last_scan_cycle_source_work); m.last_cycle_source_work = self.scanner_source_work_counter_snapshots(&self.last_scan_cycle_source_work);
m.last_cycle_replication_repair =
self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work);
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed); m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed); m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed);
m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed); m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed);
@@ -2430,6 +2626,7 @@ impl Metrics {
m.scan_checkpoint_ignored = self.scanner_checkpoint_ignored.load(Ordering::Relaxed); m.scan_checkpoint_ignored = self.scanner_checkpoint_ignored.load(Ordering::Relaxed);
m.scan_checkpoint_stale = self.scanner_checkpoint_stale.load(Ordering::Relaxed); m.scan_checkpoint_stale = self.scanner_checkpoint_stale.load(Ordering::Relaxed);
m.source_work = self.scanner_source_work_counter_snapshots(&self.scanner_source_work); m.source_work = self.scanner_source_work_counter_snapshots(&self.scanner_source_work);
m.replication_repair = self.scanner_replication_repair_work_counter_snapshots(&self.scanner_replication_repair_work);
m.partial_cycles = self.partial_scan_cycles.load(Ordering::Relaxed); m.partial_cycles = self.partial_scan_cycles.load(Ordering::Relaxed);
// Lifetime operation counts // Lifetime operation counts
@@ -3069,6 +3266,60 @@ mod tests {
assert_eq!(heal.executed, 0); assert_eq!(heal.executed, 0);
} }
#[tokio::test]
async fn report_includes_scanner_replication_repair_work() {
let metrics = Metrics::new();
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::BucketObject,
ScannerSourceWorkUpdate::queued(2),
);
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::BucketDeleteMarker,
ScannerSourceWorkUpdate::missed(1),
);
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::BucketVersionPurge,
ScannerSourceWorkUpdate {
skipped: 3,
..Default::default()
},
);
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::BucketExistingObject,
ScannerSourceWorkUpdate::queued(4),
);
let report = metrics.report().await;
let object = report
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
.expect("bucket object repair work should be visible");
assert_eq!(object.queued, 2);
let delete_marker = report
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
.expect("bucket delete-marker repair work should be visible");
assert_eq!(delete_marker.missed, 1);
let version_purge = report
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "version_purge")
.expect("bucket version purge repair work should be visible");
assert_eq!(version_purge.skipped, 3);
let existing_object = report
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "existing_object")
.expect("bucket existing object repair work should be visible");
assert_eq!(existing_object.queued, 4);
}
#[tokio::test] #[tokio::test]
async fn report_includes_scan_cycle_source_work() { async fn report_includes_scan_cycle_source_work() {
let metrics = Metrics::new(); let metrics = Metrics::new();
+83 -2
View File
@@ -24,6 +24,7 @@ use rustfs_madmin::metrics::{
ScannerMaintenanceControlSnapshot as MadminScannerMaintenanceControlSnapshot, ScannerMaintenanceControlSnapshot as MadminScannerMaintenanceControlSnapshot,
ScannerMaintenanceSourceSnapshot as MadminScannerMaintenanceSourceSnapshot, ScannerMetrics as MadminScannerMetrics, ScannerMaintenanceSourceSnapshot as MadminScannerMaintenanceSourceSnapshot, ScannerMetrics as MadminScannerMetrics,
ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot, ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
ScannerReplicationRepairSnapshot as MadminScannerReplicationRepairSnapshot,
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot, ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
TimedAction as MadminTimedAction, TimedAction as MadminTimedAction,
}; };
@@ -276,6 +277,48 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
missed: work.missed, missed: work.missed,
}) })
.collect(), .collect(),
replication_repair: metrics
.replication_repair
.into_iter()
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
failed: repair.failed,
skipped: repair.skipped,
missed: repair.missed,
})
.collect(),
current_cycle_replication_repair: metrics
.current_cycle_replication_repair
.into_iter()
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
failed: repair.failed,
skipped: repair.skipped,
missed: repair.missed,
})
.collect(),
last_cycle_replication_repair: metrics
.last_cycle_replication_repair
.into_iter()
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
failed: repair.failed,
skipped: repair.skipped,
missed: repair.missed,
})
.collect(),
partial_cycles: metrics.partial_cycles, partial_cycles: metrics.partial_cycles,
} }
} }
@@ -761,7 +804,37 @@ mod test {
skipped: 60, skipped: 60,
missed: 61, missed: 61,
}], }],
partial_cycles: 62, replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
checked: 62,
queued: 63,
executed: 64,
failed: 65,
skipped: 66,
missed: 67,
}],
current_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
checked: 68,
queued: 69,
executed: 70,
failed: 71,
skipped: 72,
missed: 73,
}],
last_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
checked: 74,
queued: 75,
executed: 76,
failed: 77,
skipped: 78,
missed: 79,
}],
partial_cycles: 80,
..Default::default() ..Default::default()
}); });
@@ -831,6 +904,14 @@ mod test {
assert_eq!(scanner.current_cycle_source_work[0].queued, 51); assert_eq!(scanner.current_cycle_source_work[0].queued, 51);
assert_eq!(scanner.last_cycle_source_work[0].source, "heal"); assert_eq!(scanner.last_cycle_source_work[0].source, "heal");
assert_eq!(scanner.last_cycle_source_work[0].skipped, 60); assert_eq!(scanner.last_cycle_source_work[0].skipped, 60);
assert_eq!(scanner.partial_cycles, 62); assert_eq!(scanner.replication_repair[0].source, "bucket_replication");
assert_eq!(scanner.replication_repair[0].kind, "object");
assert_eq!(scanner.replication_repair[0].missed, 67);
assert_eq!(scanner.current_cycle_replication_repair[0].kind, "delete_marker");
assert_eq!(scanner.current_cycle_replication_repair[0].queued, 69);
assert_eq!(scanner.last_cycle_replication_repair[0].source, "site_replication");
assert_eq!(scanner.last_cycle_replication_repair[0].kind, "active_resync");
assert_eq!(scanner.last_cycle_replication_repair[0].skipped, 78);
assert_eq!(scanner.partial_cycles, 80);
} }
} }
+119
View File
@@ -158,6 +158,26 @@ pub struct ScannerSourceWorkSnapshot {
pub missed: u64, pub missed: u64,
} }
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerReplicationRepairSnapshot {
#[serde(rename = "source", default)]
pub source: String,
#[serde(rename = "kind", default)]
pub kind: String,
#[serde(rename = "checked", default)]
pub checked: u64,
#[serde(rename = "queued", default)]
pub queued: u64,
#[serde(rename = "executed", default)]
pub executed: u64,
#[serde(rename = "failed", default)]
pub failed: u64,
#[serde(rename = "skipped", default)]
pub skipped: u64,
#[serde(rename = "missed", default)]
pub missed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerMaintenanceSourceSnapshot { pub struct ScannerMaintenanceSourceSnapshot {
#[serde(rename = "source", default)] #[serde(rename = "source", default)]
@@ -271,6 +291,17 @@ impl ScannerSourceWorkSnapshot {
} }
} }
impl ScannerReplicationRepairSnapshot {
fn merge(&mut self, other: &Self) {
self.checked = self.checked.saturating_add(other.checked);
self.queued = self.queued.saturating_add(other.queued);
self.executed = self.executed.saturating_add(other.executed);
self.failed = self.failed.saturating_add(other.failed);
self.skipped = self.skipped.saturating_add(other.skipped);
self.missed = self.missed.saturating_add(other.missed);
}
}
fn merge_source_work_snapshots(target: &mut Vec<ScannerSourceWorkSnapshot>, source: &[ScannerSourceWorkSnapshot]) { fn merge_source_work_snapshots(target: &mut Vec<ScannerSourceWorkSnapshot>, source: &[ScannerSourceWorkSnapshot]) {
for work in source { for work in source {
if let Some(existing) = target.iter_mut().find(|existing| existing.source == work.source) { if let Some(existing) = target.iter_mut().find(|existing| existing.source == work.source) {
@@ -282,6 +313,23 @@ fn merge_source_work_snapshots(target: &mut Vec<ScannerSourceWorkSnapshot>, sour
target.sort_by(|left, right| left.source.cmp(&right.source)); target.sort_by(|left, right| left.source.cmp(&right.source));
} }
fn merge_replication_repair_snapshots(
target: &mut Vec<ScannerReplicationRepairSnapshot>,
source: &[ScannerReplicationRepairSnapshot],
) {
for repair in source {
if let Some(existing) = target
.iter_mut()
.find(|existing| existing.source == repair.source && existing.kind == repair.kind)
{
existing.merge(repair);
} else {
target.push(repair.clone());
}
}
target.sort_by(|left, right| left.source.cmp(&right.source).then_with(|| left.kind.cmp(&right.kind)));
}
const SCANNER_PRIMARY_PRESSURE_NONE: &str = "none"; const SCANNER_PRIMARY_PRESSURE_NONE: &str = "none";
fn default_scanner_primary_pressure() -> String { fn default_scanner_primary_pressure() -> String {
@@ -599,6 +647,12 @@ pub struct ScannerMetrics {
pub current_cycle_source_work: Vec<ScannerSourceWorkSnapshot>, pub current_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
#[serde(rename = "last_cycle_source_work", default)] #[serde(rename = "last_cycle_source_work", default)]
pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>, pub last_cycle_source_work: Vec<ScannerSourceWorkSnapshot>,
#[serde(rename = "replication_repair", default)]
pub replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(rename = "current_cycle_replication_repair", default)]
pub current_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(rename = "last_cycle_replication_repair", default)]
pub last_cycle_replication_repair: Vec<ScannerReplicationRepairSnapshot>,
#[serde(rename = "partial_cycles", default)] #[serde(rename = "partial_cycles", default)]
pub partial_cycles: u64, pub partial_cycles: u64,
} }
@@ -730,6 +784,9 @@ impl ScannerMetrics {
merge_source_work_snapshots(&mut self.source_work, &other.source_work); merge_source_work_snapshots(&mut self.source_work, &other.source_work);
merge_source_work_snapshots(&mut self.current_cycle_source_work, &other.current_cycle_source_work); merge_source_work_snapshots(&mut self.current_cycle_source_work, &other.current_cycle_source_work);
merge_source_work_snapshots(&mut self.last_cycle_source_work, &other.last_cycle_source_work); merge_source_work_snapshots(&mut self.last_cycle_source_work, &other.last_cycle_source_work);
merge_replication_repair_snapshots(&mut self.replication_repair, &other.replication_repair);
merge_replication_repair_snapshots(&mut self.current_cycle_replication_repair, &other.current_cycle_replication_repair);
merge_replication_repair_snapshots(&mut self.last_cycle_replication_repair, &other.last_cycle_replication_repair);
if self.ongoing_buckets < other.ongoing_buckets { if self.ongoing_buckets < other.ongoing_buckets {
self.ongoing_buckets = other.ongoing_buckets; self.ongoing_buckets = other.ongoing_buckets;
@@ -1557,6 +1614,68 @@ mod tests {
assert_eq!(usage.partial_cycles, 2); assert_eq!(usage.partial_cycles, 2);
} }
#[test]
fn scanner_metrics_merge_aggregates_replication_repair_status() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
queued: 2,
..Default::default()
}],
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
replication_repair: vec![
ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
missed: 3,
..Default::default()
},
ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
skipped: 4,
..Default::default()
},
ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
queued: 5,
..Default::default()
},
],
..Default::default()
});
let object = scanner
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
.expect("bucket object repair snapshot should be merged");
assert_eq!(object.queued, 2);
assert_eq!(object.missed, 3);
let delete_marker = scanner
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
.expect("bucket delete-marker repair snapshot should be merged");
assert_eq!(delete_marker.skipped, 4);
let site_resync = scanner
.replication_repair
.iter()
.find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
.expect("site active resync snapshot should remain separate");
assert_eq!(site_resync.queued, 5);
}
#[test] #[test]
fn scanner_metrics_merge_preserves_distributed_status_fields() { fn scanner_metrics_merge_preserves_distributed_status_fields() {
let collected_at = Utc::now(); let collected_at = Utc::now();
+118 -11
View File
@@ -36,8 +36,8 @@ use rustfs_common::heal_channel::{
send_heal_request_with_admission, send_heal_request_with_admission,
}; };
use rustfs_common::metrics::{ use rustfs_common::metrics::{
IlmAction, Metric, Metrics, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, current_path_updater, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn,
global_metrics, current_path_updater, global_metrics,
}; };
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule}; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
@@ -59,7 +59,9 @@ use rustfs_ecstore::global::is_erasure;
use rustfs_ecstore::pools::{path2_bucket_object, path2_bucket_object_with_base_path}; use rustfs_ecstore::pools::{path2_bucket_object, path2_bucket_object_with_base_path};
use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete}; use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket; use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicationStatusType}; use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicateObjectInfo, ReplicationStatusType, ReplicationType,
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
use time::OffsetDateTime; use time::OffsetDateTime;
@@ -205,16 +207,39 @@ fn record_scanner_ilm_action_if_queued(metrics: &Metrics, action: IlmAction, cou
queued queued
} }
fn record_scanner_replication_admission(metrics: &Metrics, admission: ReplicationQueueAdmission) { fn scanner_replication_work_update(admission: ReplicationQueueAdmission) -> ScannerSourceWorkUpdate {
let work = match admission { match admission {
ReplicationQueueAdmission::Queued => ScannerSourceWorkUpdate::queued(1), ReplicationQueueAdmission::Queued => ScannerSourceWorkUpdate::queued(1),
ReplicationQueueAdmission::Missed => ScannerSourceWorkUpdate::missed(1), ReplicationQueueAdmission::Missed => ScannerSourceWorkUpdate::missed(1),
ReplicationQueueAdmission::Skipped => ScannerSourceWorkUpdate { ReplicationQueueAdmission::Skipped => ScannerSourceWorkUpdate {
skipped: 1, skipped: 1,
..Default::default() ..Default::default()
}, },
}; }
}
fn scanner_replication_repair_kind(roi: &ReplicateObjectInfo) -> Option<ScannerReplicationRepairKind> {
if roi.bucket.is_empty() && roi.name.is_empty() {
return None;
}
if roi.op_type == ReplicationType::ExistingObject || roi.existing_obj_resync.must_resync() {
Some(ScannerReplicationRepairKind::BucketExistingObject)
} else if !roi.version_purge_status.is_empty() {
Some(ScannerReplicationRepairKind::BucketVersionPurge)
} else if roi.delete_marker {
Some(ScannerReplicationRepairKind::BucketDeleteMarker)
} else {
Some(ScannerReplicationRepairKind::BucketObject)
}
}
fn record_scanner_replication_admission(metrics: &Metrics, roi: &ReplicateObjectInfo, admission: ReplicationQueueAdmission) {
let work = scanner_replication_work_update(admission);
metrics.record_scanner_source_work(ScannerWorkSource::BucketReplication, work); metrics.record_scanner_source_work(ScannerWorkSource::BucketReplication, work);
if let Some(kind) = scanner_replication_repair_kind(roi) {
metrics.record_scanner_replication_repair_work(kind, work);
}
} }
fn scanner_heal_source(scan_mode: HealScanMode) -> ScannerWorkSource { fn scanner_heal_source(scan_mode: HealScanMode) -> ScannerWorkSource {
@@ -920,8 +945,8 @@ impl ScannerItem {
let done_replication = Metrics::time(Metric::CheckReplication); let done_replication = Metrics::time(Metric::CheckReplication);
let replication_result = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await; let replication_result = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
done_replication(); done_replication();
record_scanner_replication_admission(global_metrics(), replication_result.admission);
let roi = replication_result.object_info; let roi = replication_result.object_info;
record_scanner_replication_admission(global_metrics(), &roi, replication_result.admission);
if !Self::should_account_replication_stats(oi) { if !Self::should_account_replication_stats(oi) {
return; return;
} }
@@ -2409,7 +2434,7 @@ mod tests {
use super::*; use super::*;
use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk}; use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
use rustfs_filemeta::VersionPurgeStatusType; use rustfs_filemeta::{ReplicateObjectInfo, ReplicationType, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType};
use serial_test::serial; use serial_test::serial;
#[cfg(unix)] #[cfg(unix)]
use std::os::unix::fs::{PermissionsExt, symlink}; use std::os::unix::fs::{PermissionsExt, symlink};
@@ -2605,10 +2630,15 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_scanner_replication_admission_accounting_maps_source_work() { async fn test_scanner_replication_admission_accounting_maps_source_work() {
let metrics = Metrics::new(); let metrics = Metrics::new();
let object = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "object-a".to_string(),
..Default::default()
};
record_scanner_replication_admission(&metrics, ReplicationQueueAdmission::Skipped); record_scanner_replication_admission(&metrics, &object, ReplicationQueueAdmission::Skipped);
record_scanner_replication_admission(&metrics, ReplicationQueueAdmission::Queued); record_scanner_replication_admission(&metrics, &object, ReplicationQueueAdmission::Queued);
record_scanner_replication_admission(&metrics, ReplicationQueueAdmission::Missed); record_scanner_replication_admission(&metrics, &object, ReplicationQueueAdmission::Missed);
let report = metrics.report().await; let report = metrics.report().await;
let replication = report let replication = report
@@ -2620,6 +2650,83 @@ mod tests {
assert_eq!(replication.skipped, 1); assert_eq!(replication.skipped, 1);
assert_eq!(replication.queued, 1); assert_eq!(replication.queued, 1);
assert_eq!(replication.missed, 1); assert_eq!(replication.missed, 1);
let object_repair = report
.replication_repair
.iter()
.find(|repair| repair.source == ScannerWorkSource::BucketReplication.as_str() && repair.kind == "object")
.expect("bucket object repair work should be visible");
assert_eq!(object_repair.skipped, 1);
assert_eq!(object_repair.queued, 1);
assert_eq!(object_repair.missed, 1);
}
#[test]
fn test_scanner_replication_repair_kind_maps_bucket_variants() {
let object = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "object-a".to_string(),
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
assert_eq!(scanner_replication_repair_kind(&object), Some(ScannerReplicationRepairKind::BucketObject));
let delete_marker = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "delete-marker-a".to_string(),
delete_marker: true,
replication_status: ReplicationStatusType::Failed,
..Default::default()
};
assert_eq!(
scanner_replication_repair_kind(&delete_marker),
Some(ScannerReplicationRepairKind::BucketDeleteMarker)
);
let version_purge = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "version-purge-a".to_string(),
version_purge_status: VersionPurgeStatusType::Pending,
..Default::default()
};
assert_eq!(
scanner_replication_repair_kind(&version_purge),
Some(ScannerReplicationRepairKind::BucketVersionPurge)
);
let existing_object = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "existing-object-a".to_string(),
op_type: ReplicationType::ExistingObject,
..Default::default()
};
assert_eq!(
scanner_replication_repair_kind(&existing_object),
Some(ScannerReplicationRepairKind::BucketExistingObject)
);
let mut existing_obj_resync = ResyncDecision::new();
existing_obj_resync.targets.insert(
"arn:minio:replication:::target".to_string(),
ResyncTargetDecision {
replicate: true,
..Default::default()
},
);
let existing_delete_marker = ReplicateObjectInfo {
bucket: "bucket-a".to_string(),
name: "existing-delete-marker-a".to_string(),
delete_marker: true,
replication_status: ReplicationStatusType::Completed,
existing_obj_resync,
..Default::default()
};
assert_eq!(
scanner_replication_repair_kind(&existing_delete_marker),
Some(ScannerReplicationRepairKind::BucketExistingObject)
);
assert_eq!(scanner_replication_repair_kind(&ReplicateObjectInfo::default()), None);
} }
#[tokio::test] #[tokio::test]
@@ -292,6 +292,9 @@ Compare these fields between baseline and tuned runs:
| `metrics.source_work` | Shows cumulative work found, queued, skipped, missed, executed, and failed by source. | | `metrics.source_work` | Shows cumulative work found, queued, skipped, missed, executed, and failed by source. |
| `metrics.current_cycle_source_work` | Shows which source is consuming the current scan cycle. | | `metrics.current_cycle_source_work` | Shows which source is consuming the current scan cycle. |
| `metrics.last_cycle_source_work` | Shows which source consumed the previous scan cycle. | | `metrics.last_cycle_source_work` | Shows which source consumed the previous scan cycle. |
| `metrics.replication_repair` | Splits scanner-discovered replication repair by source and kind, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. |
| `metrics.current_cycle_replication_repair` | Shows which replication repair kind is being discovered or admitted in the current cycle. |
| `metrics.last_cycle_replication_repair` | Shows which replication repair kind consumed the previous cycle. |
| `metrics.lifecycle_expiry.current_queued` | Shows scanner-driven expiry/delete work waiting in the expiry worker queue. | | `metrics.lifecycle_expiry.current_queued` | Shows scanner-driven expiry/delete work waiting in the expiry worker queue. |
| `metrics.lifecycle_expiry.current_active` | Shows scanner-driven expiry/delete work currently running in expiry workers. | | `metrics.lifecycle_expiry.current_active` | Shows scanner-driven expiry/delete work currently running in expiry workers. |
| `metrics.lifecycle_expiry.queue_missed` | Shows expiry/delete queue admission failures outside the scanner walk itself. | | `metrics.lifecycle_expiry.queue_missed` | Shows expiry/delete queue admission failures outside the scanner walk itself. |
@@ -99,6 +99,7 @@ metrics.lifecycle_transition.current_queued
metrics.lifecycle_transition.scanner_missed metrics.lifecycle_transition.scanner_missed
metrics.maintenance_control.primary_control metrics.maintenance_control.primary_control
metrics.source_work metrics.source_work
metrics.replication_repair
metrics.scan_checkpoint metrics.scan_checkpoint
``` ```
@@ -147,6 +148,28 @@ Use these counters to decide whether scan progress is limited by scanner pacing
or by a downstream subsystem such as lifecycle transition, replication repair, or by a downstream subsystem such as lifecycle transition, replication repair,
or heal admission. or heal admission.
## Reading Replication Repair
`metrics.replication_repair`, `metrics.current_cycle_replication_repair`, and
`metrics.last_cycle_replication_repair` split scanner-discovered replication
repair work by source and repair kind.
Each entry has the same `checked`, `queued`, `executed`, `failed`, `skipped`,
and `missed` counters used by `source_work`, plus:
| Field | Meaning |
|---|---|
| `source` | `bucket_replication` for bucket replication repair, or `site_replication` for site replication boundary signals. |
| `kind` | Bucket repair kinds are `object`, `delete_marker`, `version_purge`, and `existing_object`. Site replication boundary kinds are `passive_requeue` and `active_resync`. |
For bucket replication, `queued` means scanner-discovered repair was admitted
to the replication queue, `missed` means the queue or worker path could not
accept it, and `skipped` means the object did not require a new repair task.
The site replication kinds keep passive scanner discovery separate from active
resync. Scanner status may report site replication boundary counters, but the
scanner should not be treated as the active site replication resync controller.
## Reading Maintenance Control ## Reading Maintenance Control
`metrics.maintenance_control` derives a source-level control snapshot from `metrics.maintenance_control` derives a source-level control snapshot from