fix(heal): surface first failed object in status (#7599)

Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-10 00:49:16 +08:00
committed by GitHub
parent 45806bf295
commit 9e7c5dc932
3 changed files with 81 additions and 9 deletions
+73 -6
View File
@@ -209,7 +209,7 @@ pub fn legacy_wire_status<'a>(
{
return Err(HealOutcomeWireError::ContradictoryCompletion);
}
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters, first_failed_wire_object(wire));
Ok((
adapted,
if adapted != summary {
@@ -233,15 +233,16 @@ fn legacy_execution_status<'a>(
detail: Option<String>,
execution: HealExecutionOutcome,
counters: &HealOutcomeCounters,
first_failure: Option<String>,
) -> (&'a str, Option<String>) {
if summary != "finished" {
return (summary, detail);
}
match execution {
HealExecutionOutcome::CompletedWithErrors => (
"stopped",
Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
),
HealExecutionOutcome::CompletedWithErrors => {
let detail = format!("heal traversal completed with errors: {} failed objects", counters.failed);
("stopped", Some(append_first_failure(detail, first_failure)))
}
HealExecutionOutcome::Aborted(reason) => {
let reason = match reason {
HealAbortReason::Cancelled => "cancelled",
@@ -266,7 +267,7 @@ fn legacy_execution_status<'a>(
impl HealTaskOutcome {
pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
legacy_execution_status(summary, detail, self.execution, &self.counters)
legacy_execution_status(summary, detail, self.execution, &self.counters, self.first_failed_object())
}
pub(crate) fn start(&mut self) {
@@ -345,6 +346,72 @@ impl HealTaskOutcome {
.saturating_add(self.retained_object_bytes)
.saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
}
fn first_failed_object(&self) -> Option<String> {
self.objects.iter().find_map(first_failed_outcome_object)
}
}
fn append_first_failure(mut detail: String, first_failure: Option<String>) -> String {
if let Some(first_failure) = first_failure {
detail.push_str("; ");
detail.push_str(&first_failure);
}
detail
}
fn first_failed_outcome_object(item: &HealObjectOutcome) -> Option<String> {
let HealObjectDisposition::Failed(class) = item.disposition else {
return None;
};
Some(format_first_failed_object(
&item.identity.bucket,
&item.identity.object,
item.identity.version_id.as_deref(),
failure_class_label(class),
item.detail.as_deref(),
))
}
fn first_failed_wire_object(wire: &serde_json::Value) -> Option<String> {
let objects = wire.get("objects")?.as_array()?;
objects.iter().find_map(|item| {
let disposition = item.get("disposition")?;
if disposition.get("state")?.as_str()? != "failed" {
return None;
}
let identity = item.get("identity")?;
let bucket = identity.get("bucket")?.as_str()?;
let object = identity.get("object")?.as_str()?;
let version_id = identity.get("versionId").and_then(serde_json::Value::as_str);
let class = disposition
.get("details")
.and_then(serde_json::Value::as_str)
.unwrap_or("unknown");
let detail = item.get("detail").and_then(serde_json::Value::as_str);
Some(format_first_failed_object(bucket, object, version_id, class, detail))
})
}
fn failure_class_label(class: HealFailureClass) -> &'static str {
match class {
HealFailureClass::Recoverable => "recoverable",
HealFailureClass::RetryExhausted => "retry_exhausted",
HealFailureClass::Permanent => "permanent",
}
}
fn format_first_failed_object(bucket: &str, object: &str, version_id: Option<&str>, class: &str, detail: Option<&str>) -> String {
let mut message = format!("first failed object {bucket}/{object} ({class})");
if let Some(version_id) = version_id.filter(|version_id| !version_id.is_empty()) {
message.push_str(", version ");
message.push_str(version_id);
}
if let Some(detail) = detail.filter(|detail| !detail.is_empty()) {
message.push_str(": ");
message.push_str(detail);
}
message
}
#[cfg(test)]
+6 -1
View File
@@ -362,7 +362,12 @@ mod canonical_outcome {
assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None);
assert_eq!(legacy_summary, "stopped");
assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects"));
assert_eq!(
legacy_detail.as_deref(),
Some(
"heal traversal completed with errors: 1 failed objects; first failed object bucket-a/object-a (retry_exhausted): Storage error: Lock error: Lock acquisition timeout for resource 'object-a' after 5s"
)
);
assert_eq!(
storage.heal_object_calls.lock().expect("object calls").as_slice(),
["object-a", "object-b", "object-a", "object-a", "object-a"]
+2 -2
View File
@@ -98,7 +98,7 @@
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
"detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
@@ -274,7 +274,7 @@
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
"detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,