gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 21:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose target-scoped durable MRF backlog metrics (#5584)

Browse Source 
* feat(replication): expose target durable mrf backlog

Add target ARN attribution to durable MRF entries and surface target-scoped durable backlog metrics without changing existing bucket-only metric labels.

Keep legacy MRF files bucket-only by defaulting missing targetARNs to an empty list, and expose target snapshots through an additive API so existing DurableMrfBacklogSummary callers remain source-compatible.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(replication): expose runtime target backlog (#5586)

Track runtime replication backlog by target ARN for regular, large, delete, and MRF admission paths while preserving the existing bucket-level backlog semantics.

Add target-scoped current backlog metrics and merge them with durable target backlog snapshots for observability.

Co-authored-by: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-02 01:31:50 +08:00
committed by GitHub
parent 4c83bff81c
commit fbec33bd29
14 changed files with 1241 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/replication.rs
+107 -14
View File
@@ -835,6 +835,10 @@ struct MrfTargetBacklog {
failed_count: i64,
#[serde(rename = "FailedSize")]
failed_size: i64,
#[serde(rename = "DurableCount")]
durable_count: i64,
#[serde(rename = "DurableSize")]
durable_size: i64,
#[serde(rename = "ObservationScope")]
observation_scope: &'static str,
}
@@ -842,7 +846,7 @@ struct MrfTargetBacklog {
/// Response body for `GET /v3/replication/mrf`.
///
/// Runtime failed/queued totals and the durable MRF recovery backlog are kept
/// separate because persisted MRF entries do not contain a target ARN.
/// separate because older persisted MRF entries do not contain a target ARN.
#[derive(Debug, Serialize)]
struct MrfResponse {
#[serde(rename = "Bucket")]
@@ -888,32 +892,60 @@ fn build_mrf_response(
"partial_cluster"
};
let mut targets: Vec<MrfTargetBacklog> = Vec::with_capacity(bucket_stats.replication_stats.stats.len());
let mut targets_by_arn: HashMap<String, usize> = HashMap::with_capacity(bucket_stats.replication_stats.stats.len());
let mut total_failed_count: i64 = 0;
let mut total_failed_size: i64 = 0;
for (arn, stat) in &bucket_stats.replication_stats.stats {
total_failed_count = total_failed_count.saturating_add(stat.failed.count);
total_failed_size = total_failed_size.saturating_add(stat.failed.size);
targets_by_arn.insert(arn.clone(), targets.len());
targets.push(MrfTargetBacklog {
arn: arn.clone(),
failed_count: stat.failed.count,
failed_size: stat.failed.size,
durable_count: 0,
durable_size: 0,
observation_scope,
});
}
targets.sort_by(|a, b| a.arn.cmp(&b.arn));
let queued = &bucket_stats.replication_stats.q_stat.curr;
let (durable_count, durable_size) = if durable.available {
durable
.entries
.iter()
.filter(|entry| entry.bucket == bucket)
.fold((0i64, 0i64), |(count, size), entry| {
(count.saturating_add(1), size.saturating_add(entry.size))
})
let (durable_count, durable_size, missing_target_arns) = if durable.available {
durable.entries.iter().filter(|entry| entry.bucket == bucket).fold(
(0i64, 0i64, false),
|(count, size, missing_target_arns), entry| {
let mut missing_target_arns = missing_target_arns;
for target_arn in &entry.target_arns {
if target_arn.is_empty() {
continue;
}
let index = if let Some(index) = targets_by_arn.get(target_arn).copied() {
index
} else {
targets_by_arn.insert(target_arn.clone(), targets.len());
targets.push(MrfTargetBacklog {
arn: target_arn.clone(),
failed_count: 0,
failed_size: 0,
durable_count: 0,
durable_size: 0,
observation_scope,
});
targets.len() - 1
};
targets[index].durable_count = targets[index].durable_count.saturating_add(1);
targets[index].durable_size = targets[index].durable_size.saturating_add(entry.size);
}
if entry.target_arns.is_empty() {
missing_target_arns = true;
}
(count.saturating_add(1), size.saturating_add(entry.size), missing_target_arns)
},
)
} else {
(0, 0)
(0, 0, false)
};
targets.sort_by(|a, b| a.arn.cmp(&b.arn));
MrfResponse {
bucket,
@@ -930,7 +962,7 @@ fn build_mrf_response(
durable_backlog_available: durable.available,
durable_count,
durable_size,
per_target_durable_entries_available: false,
per_target_durable_entries_available: durable.available && !missing_target_arns,
}
}
@@ -940,8 +972,9 @@ fn build_mrf_response(
///
/// Compatibility note: MinIO returns a stream of individual MRF entries. RustFS
/// deliberately returns aggregate runtime and durable counters instead.
/// `PerObjectEntriesAvailable` and `PerTargetDurableEntriesAvailable` remain
/// false until an enumerable API and target-bearing durable format exist.
/// `PerObjectEntriesAvailable` remains false until an enumerable API exists.
/// `PerTargetDurableEntriesAvailable` is false when the durable backlog includes
/// older entries that cannot be attributed to a target.
pub struct ReplicationMrfHandler {}
#[async_trait::async_trait]
@@ -1059,6 +1092,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
},
MrfReplicateEntry {
bucket: "other-bucket".to_string(),
@@ -1070,6 +1104,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
},
],
};
@@ -1087,7 +1122,64 @@ mod tests {
assert_eq!(json["ClusterComplete"], false);
assert_eq!(json["Targets"][0]["ObservationScope"], "partial_cluster");
assert_eq!(json["PerObjectEntriesAvailable"], false);
assert_eq!(json["PerTargetDurableEntriesAvailable"], true);
let targets = json["Targets"].as_array().expect("targets should serialize as an array");
let target_a = targets
.iter()
.find(|target| target["ARN"] == "arn-a")
.expect("runtime target should remain present");
assert_eq!(target_a["FailedCount"], 3);
assert_eq!(target_a["FailedSize"], 900);
assert_eq!(target_a["DurableCount"], 1);
assert_eq!(target_a["DurableSize"], 250);
let target_durable_only = targets
.iter()
.find(|target| target["ARN"] == "arn-durable-only")
.expect("durable-only target should be surfaced");
assert_eq!(target_durable_only["FailedCount"], 0);
assert_eq!(target_durable_only["FailedSize"], 0);
assert_eq!(target_durable_only["DurableCount"], 1);
assert_eq!(target_durable_only["DurableSize"], 250);
}
#[test]
fn mrf_response_keeps_legacy_durable_entries_bucket_only() {
let mut stats = BucketStats::default();
stats.replication_stats.provider_available = true;
stats.replication_stats.cluster_complete = true;
stats.replication_stats.observed_node_count = 1;
stats.replication_stats.expected_node_count = 1;
let durable = DurableMrfBacklog {
available: true,
entries: vec![MrfReplicateEntry {
bucket: "bucket-a".to_string(),
object: "legacy-object".to_string(),
version_id: None,
retry_count: 0,
size: 250,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
}],
};
let response = build_mrf_response("bucket-a".to_string(), &stats, durable);
let json = serde_json::to_value(response).expect("MRF response should serialize");
assert_eq!(json["DurableBacklogAvailable"], true);
assert_eq!(json["DurableCount"], 1);
assert_eq!(json["DurableSize"], 250);
assert_eq!(json["PerTargetDurableEntriesAvailable"], false);
assert_eq!(
json["Targets"]
.as_array()
.expect("targets should serialize as an array")
.len(),
0
);
}
#[test]
@@ -1115,6 +1207,7 @@ mod tests {
assert_eq!(valid_empty_json["DurableBacklogAvailable"], true);
assert_eq!(valid_empty_json["TotalFailedCount"], 0);
assert_eq!(valid_empty_json["DurableCount"], 0);
assert_eq!(valid_empty_json["PerTargetDurableEntriesAvailable"], true);
}
#[test]