From 7bdb25ae9d2d4781bc1d33ec8316b67d71e652cc Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Tue, 23 Jun 2026 21:35:36 +0800 Subject: [PATCH] feat(scanner): define replication boundary contract (#3630) * feat(scanner): define replication boundary contract * fix(scanner): propagate replication boundary metrics * fix(scanner): preserve repair metadata on merge --------- Co-authored-by: Henry Guo Co-authored-by: houseme --- crates/common/src/metrics.rs | 130 +++++++++++++++++++ crates/ecstore/src/metrics_realtime.rs | 18 +++ crates/madmin/src/metrics.rs | 107 +++++++++++++++ crates/scanner/src/scanner_folder.rs | 2 + docs/operations/scanner-benchmark-runbook.md | 11 +- docs/operations/scanner-runtime-controls.md | 16 +++ 6 files changed, 282 insertions(+), 2 deletions(-) diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 89c18f6d9..a718ecd04 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -319,6 +319,36 @@ pub enum ScannerReplicationRepairKind { SiteActiveResync, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScannerReplicationRepairRole { + RepairAdmission, + BoundarySignal, +} + +impl ScannerReplicationRepairRole { + pub fn as_str(self) -> &'static str { + match self { + Self::RepairAdmission => "repair_admission", + Self::BoundarySignal => "boundary_signal", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScannerReplicationExecutionOwner { + BucketReplicationQueue, + SiteReplicationRuntime, +} + +impl ScannerReplicationExecutionOwner { + pub fn as_str(self) -> &'static str { + match self { + Self::BucketReplicationQueue => "bucket_replication_queue", + Self::SiteReplicationRuntime => "site_replication_runtime", + } + } +} + impl ScannerReplicationRepairKind { const ALL: [Self; 6] = [ Self::BucketObject, @@ -338,6 +368,24 @@ impl ScannerReplicationRepairKind { } } + pub fn scanner_role(self) -> ScannerReplicationRepairRole { + match self { + Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => { + ScannerReplicationRepairRole::RepairAdmission + } + Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationRepairRole::BoundarySignal, + } + } + + pub fn execution_owner(self) -> ScannerReplicationExecutionOwner { + match self { + Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => { + ScannerReplicationExecutionOwner::BucketReplicationQueue + } + Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationExecutionOwner::SiteReplicationRuntime, + } + } + pub fn as_str(self) -> &'static str { match self { Self::BucketObject => "object", @@ -515,6 +563,8 @@ impl ScannerSourceWorkValues { ScannerReplicationRepairSnapshot { source: kind.source().as_str().to_string(), kind: kind.as_str().to_string(), + scanner_role: kind.scanner_role().as_str().to_string(), + execution_owner: kind.execution_owner().as_str().to_string(), checked: self.checked, queued: self.queued, executed: self.executed, @@ -896,6 +946,10 @@ pub struct ScannerSourceWorkSnapshot { pub struct ScannerReplicationRepairSnapshot { pub source: String, pub kind: String, + #[serde(default)] + pub scanner_role: String, + #[serde(default)] + pub execution_owner: String, pub checked: u64, pub queued: u64, pub executed: u64, @@ -3407,6 +3461,14 @@ mod tests { ScannerReplicationRepairKind::BucketExistingObject, ScannerSourceWorkUpdate::queued(4), ); + metrics.record_scanner_replication_repair_work( + ScannerReplicationRepairKind::SitePassiveRequeue, + ScannerSourceWorkUpdate::missed(5), + ); + metrics.record_scanner_replication_repair_work( + ScannerReplicationRepairKind::SiteActiveResync, + ScannerSourceWorkUpdate::queued(6), + ); let report = metrics.report().await; let object = report @@ -3414,6 +3476,8 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "object") .expect("bucket object repair work should be visible"); + assert_eq!(object.scanner_role, "repair_admission"); + assert_eq!(object.execution_owner, "bucket_replication_queue"); assert_eq!(object.queued, 2); let delete_marker = report @@ -3421,6 +3485,8 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker") .expect("bucket delete-marker repair work should be visible"); + assert_eq!(delete_marker.scanner_role, "repair_admission"); + assert_eq!(delete_marker.execution_owner, "bucket_replication_queue"); assert_eq!(delete_marker.missed, 1); let version_purge = report @@ -3428,6 +3494,8 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "version_purge") .expect("bucket version purge repair work should be visible"); + assert_eq!(version_purge.scanner_role, "repair_admission"); + assert_eq!(version_purge.execution_owner, "bucket_replication_queue"); assert_eq!(version_purge.skipped, 3); let existing_object = report @@ -3435,7 +3503,69 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "existing_object") .expect("bucket existing object repair work should be visible"); + assert_eq!(existing_object.scanner_role, "repair_admission"); + assert_eq!(existing_object.execution_owner, "bucket_replication_queue"); assert_eq!(existing_object.queued, 4); + + let passive_requeue = report + .replication_repair + .iter() + .find(|repair| repair.source == "site_replication" && repair.kind == "passive_requeue") + .expect("site passive requeue boundary work should be visible"); + assert_eq!(passive_requeue.scanner_role, "boundary_signal"); + assert_eq!(passive_requeue.execution_owner, "site_replication_runtime"); + assert_eq!(passive_requeue.missed, 5); + + let active_resync = report + .replication_repair + .iter() + .find(|repair| repair.source == "site_replication" && repair.kind == "active_resync") + .expect("site active resync boundary work should be visible"); + assert_eq!(active_resync.scanner_role, "boundary_signal"); + assert_eq!(active_resync.execution_owner, "site_replication_runtime"); + assert_eq!(active_resync.queued, 6); + } + + #[test] + fn scanner_replication_repair_kind_sources_are_stable() { + let bucket_kinds = [ + ScannerReplicationRepairKind::BucketObject, + ScannerReplicationRepairKind::BucketDeleteMarker, + ScannerReplicationRepairKind::BucketVersionPurge, + ScannerReplicationRepairKind::BucketExistingObject, + ]; + for kind in bucket_kinds { + assert_eq!(kind.source(), ScannerWorkSource::BucketReplication); + assert_eq!(kind.scanner_role(), ScannerReplicationRepairRole::RepairAdmission); + assert_eq!(kind.execution_owner(), ScannerReplicationExecutionOwner::BucketReplicationQueue); + } + + assert_eq!( + ScannerReplicationRepairKind::SitePassiveRequeue.source(), + ScannerWorkSource::SiteReplication + ); + assert_eq!( + ScannerReplicationRepairKind::SiteActiveResync.source(), + ScannerWorkSource::SiteReplication + ); + assert_eq!( + ScannerReplicationRepairKind::SitePassiveRequeue.scanner_role(), + ScannerReplicationRepairRole::BoundarySignal + ); + assert_eq!( + ScannerReplicationRepairKind::SiteActiveResync.scanner_role(), + ScannerReplicationRepairRole::BoundarySignal + ); + assert_eq!( + ScannerReplicationRepairKind::SitePassiveRequeue.execution_owner(), + ScannerReplicationExecutionOwner::SiteReplicationRuntime + ); + assert_eq!( + ScannerReplicationRepairKind::SiteActiveResync.execution_owner(), + ScannerReplicationExecutionOwner::SiteReplicationRuntime + ); + assert_eq!(ScannerReplicationRepairKind::SitePassiveRequeue.as_str(), "passive_requeue"); + assert_eq!(ScannerReplicationRepairKind::SiteActiveResync.as_str(), "active_resync"); } #[tokio::test] diff --git a/crates/ecstore/src/metrics_realtime.rs b/crates/ecstore/src/metrics_realtime.rs index 82d517748..e496c6ae1 100644 --- a/crates/ecstore/src/metrics_realtime.rs +++ b/crates/ecstore/src/metrics_realtime.rs @@ -294,6 +294,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo .map(|repair| MadminScannerReplicationRepairSnapshot { source: repair.source, kind: repair.kind, + scanner_role: repair.scanner_role, + execution_owner: repair.execution_owner, checked: repair.checked, queued: repair.queued, executed: repair.executed, @@ -308,6 +310,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo .map(|repair| MadminScannerReplicationRepairSnapshot { source: repair.source, kind: repair.kind, + scanner_role: repair.scanner_role, + execution_owner: repair.execution_owner, checked: repair.checked, queued: repair.queued, executed: repair.executed, @@ -322,6 +326,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo .map(|repair| MadminScannerReplicationRepairSnapshot { source: repair.source, kind: repair.kind, + scanner_role: repair.scanner_role, + execution_owner: repair.execution_owner, checked: repair.checked, queued: repair.queued, executed: repair.executed, @@ -844,6 +850,8 @@ mod test { replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "object".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), checked: 62, queued: 63, executed: 64, @@ -854,6 +862,8 @@ mod test { current_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "delete_marker".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), checked: 68, queued: 69, executed: 70, @@ -864,6 +874,8 @@ mod test { last_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot { source: "site_replication".to_string(), kind: "active_resync".to_string(), + scanner_role: "boundary_signal".to_string(), + execution_owner: "site_replication_runtime".to_string(), checked: 74, queued: 75, executed: 76, @@ -943,11 +955,17 @@ mod test { assert_eq!(scanner.last_cycle_source_work[0].skipped, 60); assert_eq!(scanner.replication_repair[0].source, "bucket_replication"); assert_eq!(scanner.replication_repair[0].kind, "object"); + assert_eq!(scanner.replication_repair[0].scanner_role, "repair_admission"); + assert_eq!(scanner.replication_repair[0].execution_owner, "bucket_replication_queue"); assert_eq!(scanner.replication_repair[0].missed, 67); assert_eq!(scanner.current_cycle_replication_repair[0].kind, "delete_marker"); + assert_eq!(scanner.current_cycle_replication_repair[0].scanner_role, "repair_admission"); + assert_eq!(scanner.current_cycle_replication_repair[0].execution_owner, "bucket_replication_queue"); assert_eq!(scanner.current_cycle_replication_repair[0].queued, 69); assert_eq!(scanner.last_cycle_replication_repair[0].source, "site_replication"); assert_eq!(scanner.last_cycle_replication_repair[0].kind, "active_resync"); + assert_eq!(scanner.last_cycle_replication_repair[0].scanner_role, "boundary_signal"); + assert_eq!(scanner.last_cycle_replication_repair[0].execution_owner, "site_replication_runtime"); assert_eq!(scanner.last_cycle_replication_repair[0].skipped, 78); assert_eq!(scanner.partial_cycles, 80); } diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 54cb8cdfe..4322ea21a 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -164,6 +164,10 @@ pub struct ScannerReplicationRepairSnapshot { pub source: String, #[serde(rename = "kind", default)] pub kind: String, + #[serde(rename = "scanner_role", default)] + pub scanner_role: String, + #[serde(rename = "execution_owner", default)] + pub execution_owner: String, #[serde(rename = "checked", default)] pub checked: u64, #[serde(rename = "queued", default)] @@ -330,6 +334,12 @@ impl ScannerSourceWorkSnapshot { impl ScannerReplicationRepairSnapshot { fn merge(&mut self, other: &Self) { + if self.scanner_role.is_empty() && !other.scanner_role.is_empty() { + self.scanner_role = other.scanner_role.clone(); + } + if self.execution_owner.is_empty() && !other.execution_owner.is_empty() { + self.execution_owner = other.execution_owner.clone(); + } self.checked = self.checked.saturating_add(other.checked); self.queued = self.queued.saturating_add(other.queued); self.executed = self.executed.saturating_add(other.executed); @@ -1662,6 +1672,8 @@ mod tests { replication_repair: vec![ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "object".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), queued: 2, ..Default::default() }], @@ -1674,18 +1686,24 @@ mod tests { ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "object".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), missed: 3, ..Default::default() }, ScannerReplicationRepairSnapshot { source: "bucket_replication".to_string(), kind: "delete_marker".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), skipped: 4, ..Default::default() }, ScannerReplicationRepairSnapshot { source: "site_replication".to_string(), kind: "active_resync".to_string(), + scanner_role: "boundary_signal".to_string(), + execution_owner: "site_replication_runtime".to_string(), queued: 5, ..Default::default() }, @@ -1698,6 +1716,8 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "object") .expect("bucket object repair snapshot should be merged"); + assert_eq!(object.scanner_role, "repair_admission"); + assert_eq!(object.execution_owner, "bucket_replication_queue"); assert_eq!(object.queued, 2); assert_eq!(object.missed, 3); @@ -1706,6 +1726,8 @@ mod tests { .iter() .find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker") .expect("bucket delete-marker repair snapshot should be merged"); + assert_eq!(delete_marker.scanner_role, "repair_admission"); + assert_eq!(delete_marker.execution_owner, "bucket_replication_queue"); assert_eq!(delete_marker.skipped, 4); let site_resync = scanner @@ -1713,9 +1735,94 @@ mod tests { .iter() .find(|repair| repair.source == "site_replication" && repair.kind == "active_resync") .expect("site active resync snapshot should remain separate"); + assert_eq!(site_resync.scanner_role, "boundary_signal"); + assert_eq!(site_resync.execution_owner, "site_replication_runtime"); assert_eq!(site_resync.queued, 5); } + #[test] + fn scanner_metrics_merge_preserves_replication_repair_metadata_from_newer_nodes() { + let collected_at = Utc::now(); + let mut scanner = ScannerMetrics { + collected_at, + replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "bucket_replication".to_string(), + kind: "object".to_string(), + checked: 2, + ..Default::default() + }], + current_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "bucket_replication".to_string(), + kind: "delete_marker".to_string(), + queued: 3, + ..Default::default() + }], + last_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "site_replication".to_string(), + kind: "active_resync".to_string(), + missed: 4, + ..Default::default() + }], + ..Default::default() + }; + + scanner.merge(&ScannerMetrics { + collected_at: collected_at + chrono::Duration::seconds(1), + replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "bucket_replication".to_string(), + kind: "object".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), + checked: 5, + ..Default::default() + }], + current_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "bucket_replication".to_string(), + kind: "delete_marker".to_string(), + scanner_role: "repair_admission".to_string(), + execution_owner: "bucket_replication_queue".to_string(), + queued: 7, + ..Default::default() + }], + last_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot { + source: "site_replication".to_string(), + kind: "active_resync".to_string(), + scanner_role: "boundary_signal".to_string(), + execution_owner: "site_replication_runtime".to_string(), + missed: 11, + ..Default::default() + }], + ..Default::default() + }); + + let object = scanner + .replication_repair + .iter() + .find(|repair| repair.source == "bucket_replication" && repair.kind == "object") + .expect("mixed-version object repair snapshot should be merged"); + assert_eq!(object.scanner_role, "repair_admission"); + assert_eq!(object.execution_owner, "bucket_replication_queue"); + assert_eq!(object.checked, 7); + + let delete_marker = scanner + .current_cycle_replication_repair + .iter() + .find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker") + .expect("mixed-version current-cycle repair snapshot should be merged"); + assert_eq!(delete_marker.scanner_role, "repair_admission"); + assert_eq!(delete_marker.execution_owner, "bucket_replication_queue"); + assert_eq!(delete_marker.queued, 10); + + let active_resync = scanner + .last_cycle_replication_repair + .iter() + .find(|repair| repair.source == "site_replication" && repair.kind == "active_resync") + .expect("mixed-version last-cycle repair snapshot should be merged"); + assert_eq!(active_resync.scanner_role, "boundary_signal"); + assert_eq!(active_resync.execution_owner, "site_replication_runtime"); + assert_eq!(active_resync.missed, 15); + } + #[test] fn scanner_metrics_merge_preserves_distributed_status_fields() { let collected_at = Utc::now(); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 64be8dd1a..530b0ae74 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -2643,6 +2643,8 @@ mod tests { .iter() .find(|repair| repair.source == ScannerWorkSource::BucketReplication.as_str() && repair.kind == "object") .expect("bucket object repair work should be visible"); + assert_eq!(object_repair.scanner_role, "repair_admission"); + assert_eq!(object_repair.execution_owner, "bucket_replication_queue"); assert_eq!(object_repair.skipped, 1); assert_eq!(object_repair.queued, 1); assert_eq!(object_repair.missed, 1); diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 3d4811fcf..3bc9e85a9 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -335,7 +335,7 @@ Compare these fields between baseline and tuned runs: | `metrics.source_work` | Shows cumulative work found, queued, skipped, missed, executed, and failed by source. | | `metrics.current_cycle_source_work` | Shows which source is consuming the current scan cycle. | | `metrics.last_cycle_source_work` | Shows which source consumed the previous scan cycle. | -| `metrics.replication_repair` | Splits scanner-discovered replication repair by source and kind, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. | +| `metrics.replication_repair` | Splits scanner-discovered replication repair by source, kind, scanner role, and execution owner, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. | | `metrics.current_cycle_replication_repair` | Shows which replication repair kind is being discovered or admitted in the current cycle. | | `metrics.last_cycle_replication_repair` | Shows which replication repair kind consumed the previous cycle. | | `metrics.lifecycle_expiry.current_queued` | Shows scanner-driven expiry/delete work waiting in the expiry worker queue. | @@ -425,6 +425,13 @@ Treat these as failure signals: reduce backlog; - bucket metrics show zero usage after post-start uploads while dirty usage remains pending and `life_time_save_usage` does not advance; +- `bucket_replication` missed work with `scanner_role=repair_admission` grows + while replication worker queues or target failures are also growing; treat + this as downstream replication pressure, not only scanner pacing pressure; +- `site_replication` `active_resync` grows and is interpreted as scanner-owned + repair execution; `scanner_role=boundary_signal` and + `execution_owner=site_replication_runtime` mean active site resync remains + owned by the site replication runtime and admin resync path; - heal or bitrot work moves from `queued` to `missed` after a scanner pacing change. @@ -458,7 +465,7 @@ at least these runs: | Single-node post-start bucket metrics freshness | Empty data path startup, post-start bucket creation/upload, bucket metrics snapshots, `scanner-summary.csv` usage freshness columns, and evidence that `DataUsageInfo` save work occurred before accepting bucket usage metrics. | | Single-node erasure or multi-disk | Checkpoint movement, active path age, set/disk scan pressure, data usage freshness, and before/after scanner config. | | Distributed lifecycle backlog | `maintenance_control`, lifecycle expiry/transition queue fields, source work missed/failed counts, and by-host admin metrics. | -| Distributed replication backlog | Bucket replication repair kind counters, site replication passive/active boundary counters, source work queued/skipped/missed counts, and by-host admin metrics. | +| Distributed replication backlog | Bucket replication repair kind counters, `scanner_role`, `execution_owner`, site replication passive/active boundary counters, source work queued/skipped/missed counts, and by-host admin metrics. | | Heal or bitrot pressure | Background heal `healOperations` queued/active source and priority counts, scanner source work for heal/bitrot, and by-host admin metrics. | The expected conclusion is MinIO-style scanner behavior at the operational diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 1d63ddf27..26b8bc80c 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -188,6 +188,8 @@ and `missed` counters used by `source_work`, plus: |---|---| | `source` | `bucket_replication` for bucket replication repair, or `site_replication` for site replication boundary signals. | | `kind` | Bucket repair kinds are `object`, `delete_marker`, `version_purge`, and `existing_object`. Site replication boundary kinds are `passive_requeue` and `active_resync`. | +| `scanner_role` | `repair_admission` means scanner found work and attempted to admit it to a worker queue. `boundary_signal` means scanner is reporting state owned by another runtime. | +| `execution_owner` | `bucket_replication_queue` for bucket replication repair execution, or `site_replication_runtime` for site replication resync execution. | For bucket replication, `queued` means scanner-discovered repair was admitted to the replication queue, `missed` means the queue or worker path could not @@ -197,6 +199,20 @@ The site replication kinds keep passive scanner discovery separate from active resync. Scanner status may report site replication boundary counters, but the scanner should not be treated as the active site replication resync controller. +Use this boundary when interpreting replication pressure: + +| Scenario | Scanner source | Repair kind | Scanner role | Execution owner | Operational meaning | +|---|---|---|---|---|---| +| Bucket object, delete-marker, version-purge, or existing-object repair found during a scan | `bucket_replication` | `object`, `delete_marker`, `version_purge`, `existing_object` | `repair_admission` | `bucket_replication_queue` | Scanner found bucket replication repair work and attempted to admit it to the replication queue. | +| Peer-originated or passive site replication work is observed while scanning | `site_replication` | `passive_requeue` | `boundary_signal` | `site_replication_runtime` | Scanner is reporting a passive site-replication boundary signal; it is not taking ownership of active site resync. | +| Admin-triggered or runtime-owned site resync activity is visible in scanner metrics | `site_replication` | `active_resync` | `boundary_signal` | `site_replication_runtime` | Treat this as a boundary/status signal owned by the site replication runtime, not as scanner-controlled repair execution. | + +If `site_replication` counters grow while bucket replication counters stay +flat, investigate site replication status and resync state before tuning +scanner pacing. If `bucket_replication` `missed` grows, investigate the bucket +replication worker queue or target health before changing scanner cycle +settings. + ## Reading Maintenance Control `metrics.maintenance_control` derives a source-level control snapshot from