gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): define replication boundary contract (#3630)

Browse Source 
* feat(scanner): define replication boundary contract

* fix(scanner): propagate replication boundary metrics

* fix(scanner): preserve repair metadata on merge

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-06-23 21:35:36 +08:00
committed by GitHub
parent eff656e086
commit 7bdb25ae9d
6 changed files with 282 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/metrics.rs
+130
View File
@@ -319,6 +319,36 @@ pub enum ScannerReplicationRepairKind {
SiteActiveResync,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScannerReplicationRepairRole {
RepairAdmission,
BoundarySignal,
}
impl ScannerReplicationRepairRole {
pub fn as_str(self) -> &'static str {
match self {
Self::RepairAdmission => "repair_admission",
Self::BoundarySignal => "boundary_signal",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScannerReplicationExecutionOwner {
BucketReplicationQueue,
SiteReplicationRuntime,
}
impl ScannerReplicationExecutionOwner {
pub fn as_str(self) -> &'static str {
match self {
Self::BucketReplicationQueue => "bucket_replication_queue",
Self::SiteReplicationRuntime => "site_replication_runtime",
}
}
}
impl ScannerReplicationRepairKind {
const ALL: [Self; 6] = [
Self::BucketObject,
@@ -338,6 +368,24 @@ impl ScannerReplicationRepairKind {
}
}
pub fn scanner_role(self) -> ScannerReplicationRepairRole {
match self {
Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
ScannerReplicationRepairRole::RepairAdmission
}
Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationRepairRole::BoundarySignal,
}
}
pub fn execution_owner(self) -> ScannerReplicationExecutionOwner {
match self {
Self::BucketObject | Self::BucketDeleteMarker | Self::BucketVersionPurge | Self::BucketExistingObject => {
ScannerReplicationExecutionOwner::BucketReplicationQueue
}
Self::SitePassiveRequeue | Self::SiteActiveResync => ScannerReplicationExecutionOwner::SiteReplicationRuntime,
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::BucketObject => "object",
@@ -515,6 +563,8 @@ impl ScannerSourceWorkValues {
ScannerReplicationRepairSnapshot {
source: kind.source().as_str().to_string(),
kind: kind.as_str().to_string(),
scanner_role: kind.scanner_role().as_str().to_string(),
execution_owner: kind.execution_owner().as_str().to_string(),
checked: self.checked,
queued: self.queued,
executed: self.executed,
@@ -896,6 +946,10 @@ pub struct ScannerSourceWorkSnapshot {
pub struct ScannerReplicationRepairSnapshot {
pub source: String,
pub kind: String,
#[serde(default)]
pub scanner_role: String,
#[serde(default)]
pub execution_owner: String,
pub checked: u64,
pub queued: u64,
pub executed: u64,
@@ -3407,6 +3461,14 @@ mod tests {
ScannerReplicationRepairKind::BucketExistingObject,
ScannerSourceWorkUpdate::queued(4),
);
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::SitePassiveRequeue,
ScannerSourceWorkUpdate::missed(5),
);
metrics.record_scanner_replication_repair_work(
ScannerReplicationRepairKind::SiteActiveResync,
ScannerSourceWorkUpdate::queued(6),
);
let report = metrics.report().await;
let object = report
@@ -3414,6 +3476,8 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
.expect("bucket object repair work should be visible");
assert_eq!(object.scanner_role, "repair_admission");
assert_eq!(object.execution_owner, "bucket_replication_queue");
assert_eq!(object.queued, 2);
let delete_marker = report
@@ -3421,6 +3485,8 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
.expect("bucket delete-marker repair work should be visible");
assert_eq!(delete_marker.scanner_role, "repair_admission");
assert_eq!(delete_marker.execution_owner, "bucket_replication_queue");
assert_eq!(delete_marker.missed, 1);
let version_purge = report
@@ -3428,6 +3494,8 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "version_purge")
.expect("bucket version purge repair work should be visible");
assert_eq!(version_purge.scanner_role, "repair_admission");
assert_eq!(version_purge.execution_owner, "bucket_replication_queue");
assert_eq!(version_purge.skipped, 3);
let existing_object = report
@@ -3435,7 +3503,69 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "existing_object")
.expect("bucket existing object repair work should be visible");
assert_eq!(existing_object.scanner_role, "repair_admission");
assert_eq!(existing_object.execution_owner, "bucket_replication_queue");
assert_eq!(existing_object.queued, 4);
let passive_requeue = report
.replication_repair
.iter()
.find(|repair| repair.source == "site_replication" && repair.kind == "passive_requeue")
.expect("site passive requeue boundary work should be visible");
assert_eq!(passive_requeue.scanner_role, "boundary_signal");
assert_eq!(passive_requeue.execution_owner, "site_replication_runtime");
assert_eq!(passive_requeue.missed, 5);
let active_resync = report
.replication_repair
.iter()
.find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
.expect("site active resync boundary work should be visible");
assert_eq!(active_resync.scanner_role, "boundary_signal");
assert_eq!(active_resync.execution_owner, "site_replication_runtime");
assert_eq!(active_resync.queued, 6);
}
#[test]
fn scanner_replication_repair_kind_sources_are_stable() {
let bucket_kinds = [
ScannerReplicationRepairKind::BucketObject,
ScannerReplicationRepairKind::BucketDeleteMarker,
ScannerReplicationRepairKind::BucketVersionPurge,
ScannerReplicationRepairKind::BucketExistingObject,
];
for kind in bucket_kinds {
assert_eq!(kind.source(), ScannerWorkSource::BucketReplication);
assert_eq!(kind.scanner_role(), ScannerReplicationRepairRole::RepairAdmission);
assert_eq!(kind.execution_owner(), ScannerReplicationExecutionOwner::BucketReplicationQueue);
}
assert_eq!(
ScannerReplicationRepairKind::SitePassiveRequeue.source(),
ScannerWorkSource::SiteReplication
);
assert_eq!(
ScannerReplicationRepairKind::SiteActiveResync.source(),
ScannerWorkSource::SiteReplication
);
assert_eq!(
ScannerReplicationRepairKind::SitePassiveRequeue.scanner_role(),
ScannerReplicationRepairRole::BoundarySignal
);
assert_eq!(
ScannerReplicationRepairKind::SiteActiveResync.scanner_role(),
ScannerReplicationRepairRole::BoundarySignal
);
assert_eq!(
ScannerReplicationRepairKind::SitePassiveRequeue.execution_owner(),
ScannerReplicationExecutionOwner::SiteReplicationRuntime
);
assert_eq!(
ScannerReplicationRepairKind::SiteActiveResync.execution_owner(),
ScannerReplicationExecutionOwner::SiteReplicationRuntime
);
assert_eq!(ScannerReplicationRepairKind::SitePassiveRequeue.as_str(), "passive_requeue");
assert_eq!(ScannerReplicationRepairKind::SiteActiveResync.as_str(), "active_resync");
}
#[tokio::test]
crates/ecstore/src/metrics_realtime.rs
+18
View File
@@ -294,6 +294,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
scanner_role: repair.scanner_role,
execution_owner: repair.execution_owner,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
@@ -308,6 +310,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
scanner_role: repair.scanner_role,
execution_owner: repair.execution_owner,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
@@ -322,6 +326,8 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
.map(|repair| MadminScannerReplicationRepairSnapshot {
source: repair.source,
kind: repair.kind,
scanner_role: repair.scanner_role,
execution_owner: repair.execution_owner,
checked: repair.checked,
queued: repair.queued,
executed: repair.executed,
@@ -844,6 +850,8 @@ mod test {
replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
checked: 62,
queued: 63,
executed: 64,
@@ -854,6 +862,8 @@ mod test {
current_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
checked: 68,
queued: 69,
executed: 70,
@@ -864,6 +874,8 @@ mod test {
last_cycle_replication_repair: vec![rustfs_common::metrics::ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
scanner_role: "boundary_signal".to_string(),
execution_owner: "site_replication_runtime".to_string(),
checked: 74,
queued: 75,
executed: 76,
@@ -943,11 +955,17 @@ mod test {
assert_eq!(scanner.last_cycle_source_work[0].skipped, 60);
assert_eq!(scanner.replication_repair[0].source, "bucket_replication");
assert_eq!(scanner.replication_repair[0].kind, "object");
assert_eq!(scanner.replication_repair[0].scanner_role, "repair_admission");
assert_eq!(scanner.replication_repair[0].execution_owner, "bucket_replication_queue");
assert_eq!(scanner.replication_repair[0].missed, 67);
assert_eq!(scanner.current_cycle_replication_repair[0].kind, "delete_marker");
assert_eq!(scanner.current_cycle_replication_repair[0].scanner_role, "repair_admission");
assert_eq!(scanner.current_cycle_replication_repair[0].execution_owner, "bucket_replication_queue");
assert_eq!(scanner.current_cycle_replication_repair[0].queued, 69);
assert_eq!(scanner.last_cycle_replication_repair[0].source, "site_replication");
assert_eq!(scanner.last_cycle_replication_repair[0].kind, "active_resync");
assert_eq!(scanner.last_cycle_replication_repair[0].scanner_role, "boundary_signal");
assert_eq!(scanner.last_cycle_replication_repair[0].execution_owner, "site_replication_runtime");
assert_eq!(scanner.last_cycle_replication_repair[0].skipped, 78);
assert_eq!(scanner.partial_cycles, 80);
}
crates/madmin/src/metrics.rs
+107
View File
@@ -164,6 +164,10 @@ pub struct ScannerReplicationRepairSnapshot {
pub source: String,
#[serde(rename = "kind", default)]
pub kind: String,
#[serde(rename = "scanner_role", default)]
pub scanner_role: String,
#[serde(rename = "execution_owner", default)]
pub execution_owner: String,
#[serde(rename = "checked", default)]
pub checked: u64,
#[serde(rename = "queued", default)]
@@ -330,6 +334,12 @@ impl ScannerSourceWorkSnapshot {
impl ScannerReplicationRepairSnapshot {
fn merge(&mut self, other: &Self) {
if self.scanner_role.is_empty() && !other.scanner_role.is_empty() {
self.scanner_role = other.scanner_role.clone();
}
if self.execution_owner.is_empty() && !other.execution_owner.is_empty() {
self.execution_owner = other.execution_owner.clone();
}
self.checked = self.checked.saturating_add(other.checked);
self.queued = self.queued.saturating_add(other.queued);
self.executed = self.executed.saturating_add(other.executed);
@@ -1662,6 +1672,8 @@ mod tests {
replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
queued: 2,
..Default::default()
}],
@@ -1674,18 +1686,24 @@ mod tests {
ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
missed: 3,
..Default::default()
},
ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
skipped: 4,
..Default::default()
},
ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
scanner_role: "boundary_signal".to_string(),
execution_owner: "site_replication_runtime".to_string(),
queued: 5,
..Default::default()
},
@@ -1698,6 +1716,8 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
.expect("bucket object repair snapshot should be merged");
assert_eq!(object.scanner_role, "repair_admission");
assert_eq!(object.execution_owner, "bucket_replication_queue");
assert_eq!(object.queued, 2);
assert_eq!(object.missed, 3);
@@ -1706,6 +1726,8 @@ mod tests {
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
.expect("bucket delete-marker repair snapshot should be merged");
assert_eq!(delete_marker.scanner_role, "repair_admission");
assert_eq!(delete_marker.execution_owner, "bucket_replication_queue");
assert_eq!(delete_marker.skipped, 4);
let site_resync = scanner
@@ -1713,9 +1735,94 @@ mod tests {
.iter()
.find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
.expect("site active resync snapshot should remain separate");
assert_eq!(site_resync.scanner_role, "boundary_signal");
assert_eq!(site_resync.execution_owner, "site_replication_runtime");
assert_eq!(site_resync.queued, 5);
}
#[test]
fn scanner_metrics_merge_preserves_replication_repair_metadata_from_newer_nodes() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
checked: 2,
..Default::default()
}],
current_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
queued: 3,
..Default::default()
}],
last_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
missed: 4,
..Default::default()
}],
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
checked: 5,
..Default::default()
}],
current_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
scanner_role: "repair_admission".to_string(),
execution_owner: "bucket_replication_queue".to_string(),
queued: 7,
..Default::default()
}],
last_cycle_replication_repair: vec![ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
scanner_role: "boundary_signal".to_string(),
execution_owner: "site_replication_runtime".to_string(),
missed: 11,
..Default::default()
}],
..Default::default()
});
let object = scanner
.replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "object")
.expect("mixed-version object repair snapshot should be merged");
assert_eq!(object.scanner_role, "repair_admission");
assert_eq!(object.execution_owner, "bucket_replication_queue");
assert_eq!(object.checked, 7);
let delete_marker = scanner
.current_cycle_replication_repair
.iter()
.find(|repair| repair.source == "bucket_replication" && repair.kind == "delete_marker")
.expect("mixed-version current-cycle repair snapshot should be merged");
assert_eq!(delete_marker.scanner_role, "repair_admission");
assert_eq!(delete_marker.execution_owner, "bucket_replication_queue");
assert_eq!(delete_marker.queued, 10);
let active_resync = scanner
.last_cycle_replication_repair
.iter()
.find(|repair| repair.source == "site_replication" && repair.kind == "active_resync")
.expect("mixed-version last-cycle repair snapshot should be merged");
assert_eq!(active_resync.scanner_role, "boundary_signal");
assert_eq!(active_resync.execution_owner, "site_replication_runtime");
assert_eq!(active_resync.missed, 15);
}
#[test]
fn scanner_metrics_merge_preserves_distributed_status_fields() {
let collected_at = Utc::now();
crates/scanner/src/scanner_folder.rs
+2
View File
@@ -2643,6 +2643,8 @@ mod tests {
.iter()
.find(|repair| repair.source == ScannerWorkSource::BucketReplication.as_str() && repair.kind == "object")
.expect("bucket object repair work should be visible");
assert_eq!(object_repair.scanner_role, "repair_admission");
assert_eq!(object_repair.execution_owner, "bucket_replication_queue");
assert_eq!(object_repair.skipped, 1);
assert_eq!(object_repair.queued, 1);
assert_eq!(object_repair.missed, 1);
docs/operations/scanner-benchmark-runbook.md
+9 -2
View File
@@ -335,7 +335,7 @@ Compare these fields between baseline and tuned runs:
| `metrics.source_work` | Shows cumulative work found, queued, skipped, missed, executed, and failed by source. |
| `metrics.current_cycle_source_work` | Shows which source is consuming the current scan cycle. |
| `metrics.last_cycle_source_work` | Shows which source consumed the previous scan cycle. |
| `metrics.replication_repair` | Splits scanner-discovered replication repair by source and kind, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. |
| `metrics.replication_repair` | Splits scanner-discovered replication repair by source, kind, scanner role, and execution owner, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. |
| `metrics.current_cycle_replication_repair` | Shows which replication repair kind is being discovered or admitted in the current cycle. |
| `metrics.last_cycle_replication_repair` | Shows which replication repair kind consumed the previous cycle. |
| `metrics.lifecycle_expiry.current_queued` | Shows scanner-driven expiry/delete work waiting in the expiry worker queue. |
@@ -425,6 +425,13 @@ Treat these as failure signals:
reduce backlog;
- bucket metrics show zero usage after post-start uploads while dirty usage
remains pending and `life_time_save_usage` does not advance;
- `bucket_replication` missed work with `scanner_role=repair_admission` grows
while replication worker queues or target failures are also growing; treat
this as downstream replication pressure, not only scanner pacing pressure;
- `site_replication` `active_resync` grows and is interpreted as scanner-owned
repair execution; `scanner_role=boundary_signal` and
`execution_owner=site_replication_runtime` mean active site resync remains
owned by the site replication runtime and admin resync path;
- heal or bitrot work moves from `queued` to `missed` after a scanner pacing
change.
@@ -458,7 +465,7 @@ at least these runs:
| Single-node post-start bucket metrics freshness | Empty data path startup, post-start bucket creation/upload, bucket metrics snapshots, `scanner-summary.csv` usage freshness columns, and evidence that `DataUsageInfo` save work occurred before accepting bucket usage metrics. |
| Single-node erasure or multi-disk | Checkpoint movement, active path age, set/disk scan pressure, data usage freshness, and before/after scanner config. |
| Distributed lifecycle backlog | `maintenance_control`, lifecycle expiry/transition queue fields, source work missed/failed counts, and by-host admin metrics. |
| Distributed replication backlog | Bucket replication repair kind counters, site replication passive/active boundary counters, source work queued/skipped/missed counts, and by-host admin metrics. |
| Distributed replication backlog | Bucket replication repair kind counters, `scanner_role`, `execution_owner`, site replication passive/active boundary counters, source work queued/skipped/missed counts, and by-host admin metrics. |
| Heal or bitrot pressure | Background heal `healOperations` queued/active source and priority counts, scanner source work for heal/bitrot, and by-host admin metrics. |
The expected conclusion is MinIO-style scanner behavior at the operational
docs/operations/scanner-runtime-controls.md
+16
View File
@@ -188,6 +188,8 @@ and `missed` counters used by `source_work`, plus:
|---|---|
| `source` | `bucket_replication` for bucket replication repair, or `site_replication` for site replication boundary signals. |
| `kind` | Bucket repair kinds are `object`, `delete_marker`, `version_purge`, and `existing_object`. Site replication boundary kinds are `passive_requeue` and `active_resync`. |
| `scanner_role` | `repair_admission` means scanner found work and attempted to admit it to a worker queue. `boundary_signal` means scanner is reporting state owned by another runtime. |
| `execution_owner` | `bucket_replication_queue` for bucket replication repair execution, or `site_replication_runtime` for site replication resync execution. |
For bucket replication, `queued` means scanner-discovered repair was admitted
to the replication queue, `missed` means the queue or worker path could not
@@ -197,6 +199,20 @@ The site replication kinds keep passive scanner discovery separate from active
resync. Scanner status may report site replication boundary counters, but the
scanner should not be treated as the active site replication resync controller.
Use this boundary when interpreting replication pressure:
| Scenario | Scanner source | Repair kind | Scanner role | Execution owner | Operational meaning |
|---|---|---|---|---|---|
| Bucket object, delete-marker, version-purge, or existing-object repair found during a scan | `bucket_replication` | `object`, `delete_marker`, `version_purge`, `existing_object` | `repair_admission` | `bucket_replication_queue` | Scanner found bucket replication repair work and attempted to admit it to the replication queue. |
| Peer-originated or passive site replication work is observed while scanning | `site_replication` | `passive_requeue` | `boundary_signal` | `site_replication_runtime` | Scanner is reporting a passive site-replication boundary signal; it is not taking ownership of active site resync. |
| Admin-triggered or runtime-owned site resync activity is visible in scanner metrics | `site_replication` | `active_resync` | `boundary_signal` | `site_replication_runtime` | Treat this as a boundary/status signal owned by the site replication runtime, not as scanner-controlled repair execution. |
If `site_replication` counters grow while bucket replication counters stay
flat, investigate site replication status and resync state before tuning
scanner pacing. If `bucket_replication` `missed` grows, investigate the bucket
replication worker queue or target health before changing scanner cycle
settings.
## Reading Maintenance Control
`metrics.maintenance_control` derives a source-level control snapshot from