From 9e7c5dc93228ce0924c543a0c8a1e9efe63046e7 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 10 Sep 2026 00:49:16 +0800 Subject: [PATCH] fix(heal): surface first failed object in status (#7599) Co-authored-by: zhi22915 --- crates/heal/src/heal/outcome.rs | 79 +++++++++++++++++-- crates/heal/src/heal/task/tests.rs | 7 +- .../tests/fixtures/heal-outcome-v3.json | 4 +- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/crates/heal/src/heal/outcome.rs b/crates/heal/src/heal/outcome.rs index 933c53d09..1bcec5c9b 100644 --- a/crates/heal/src/heal/outcome.rs +++ b/crates/heal/src/heal/outcome.rs @@ -209,7 +209,7 @@ pub fn legacy_wire_status<'a>( { return Err(HealOutcomeWireError::ContradictoryCompletion); } - let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters); + let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters, first_failed_wire_object(wire)); Ok(( adapted, if adapted != summary { @@ -233,15 +233,16 @@ fn legacy_execution_status<'a>( detail: Option, execution: HealExecutionOutcome, counters: &HealOutcomeCounters, + first_failure: Option, ) -> (&'a str, Option) { if summary != "finished" { return (summary, detail); } match execution { - HealExecutionOutcome::CompletedWithErrors => ( - "stopped", - Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)), - ), + HealExecutionOutcome::CompletedWithErrors => { + let detail = format!("heal traversal completed with errors: {} failed objects", counters.failed); + ("stopped", Some(append_first_failure(detail, first_failure))) + } HealExecutionOutcome::Aborted(reason) => { let reason = match reason { HealAbortReason::Cancelled => "cancelled", @@ -266,7 +267,7 @@ fn legacy_execution_status<'a>( impl HealTaskOutcome { pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option) -> (&'a str, Option) { - legacy_execution_status(summary, detail, self.execution, &self.counters) + legacy_execution_status(summary, detail, self.execution, &self.counters, self.first_failed_object()) } pub(crate) fn start(&mut self) { @@ -345,6 +346,72 @@ impl HealTaskOutcome { .saturating_add(self.retained_object_bytes) .saturating_add(self.objects.capacity().saturating_mul(size_of::())) } + + fn first_failed_object(&self) -> Option { + self.objects.iter().find_map(first_failed_outcome_object) + } +} + +fn append_first_failure(mut detail: String, first_failure: Option) -> String { + if let Some(first_failure) = first_failure { + detail.push_str("; "); + detail.push_str(&first_failure); + } + detail +} + +fn first_failed_outcome_object(item: &HealObjectOutcome) -> Option { + let HealObjectDisposition::Failed(class) = item.disposition else { + return None; + }; + Some(format_first_failed_object( + &item.identity.bucket, + &item.identity.object, + item.identity.version_id.as_deref(), + failure_class_label(class), + item.detail.as_deref(), + )) +} + +fn first_failed_wire_object(wire: &serde_json::Value) -> Option { + let objects = wire.get("objects")?.as_array()?; + objects.iter().find_map(|item| { + let disposition = item.get("disposition")?; + if disposition.get("state")?.as_str()? != "failed" { + return None; + } + let identity = item.get("identity")?; + let bucket = identity.get("bucket")?.as_str()?; + let object = identity.get("object")?.as_str()?; + let version_id = identity.get("versionId").and_then(serde_json::Value::as_str); + let class = disposition + .get("details") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + let detail = item.get("detail").and_then(serde_json::Value::as_str); + Some(format_first_failed_object(bucket, object, version_id, class, detail)) + }) +} + +fn failure_class_label(class: HealFailureClass) -> &'static str { + match class { + HealFailureClass::Recoverable => "recoverable", + HealFailureClass::RetryExhausted => "retry_exhausted", + HealFailureClass::Permanent => "permanent", + } +} + +fn format_first_failed_object(bucket: &str, object: &str, version_id: Option<&str>, class: &str, detail: Option<&str>) -> String { + let mut message = format!("first failed object {bucket}/{object} ({class})"); + if let Some(version_id) = version_id.filter(|version_id| !version_id.is_empty()) { + message.push_str(", version "); + message.push_str(version_id); + } + if let Some(detail) = detail.filter(|detail| !detail.is_empty()) { + message.push_str(": "); + message.push_str(detail); + } + message } #[cfg(test)] diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 63376d08e..1e93246c8 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -362,7 +362,12 @@ mod canonical_outcome { assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None); assert_eq!(legacy_summary, "stopped"); - assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects")); + assert_eq!( + legacy_detail.as_deref(), + Some( + "heal traversal completed with errors: 1 failed objects; first failed object bucket-a/object-a (retry_exhausted): Storage error: Lock error: Lock acquisition timeout for resource 'object-a' after 5s" + ) + ); assert_eq!( storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a", "object-b", "object-a", "object-a", "object-a"] diff --git a/crates/madmin/tests/fixtures/heal-outcome-v3.json b/crates/madmin/tests/fixtures/heal-outcome-v3.json index 9f634af80..21989d323 100644 --- a/crates/madmin/tests/fixtures/heal-outcome-v3.json +++ b/crates/madmin/tests/fixtures/heal-outcome-v3.json @@ -98,7 +98,7 @@ "cliExit": 1, "response": { "summary": "stopped", - "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated", + "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated", "startTime": "2026-01-01T00:00:00Z", "settings": { "recursive": true, @@ -274,7 +274,7 @@ "cliExit": 1, "response": { "summary": "stopped", - "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated", + "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated", "startTime": "2026-01-01T00:00:00Z", "settings": { "recursive": true,