From 8550a8f9c3635b45c244fed120cd131d26a3124c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 5 Aug 2026 11:54:27 +0800 Subject: [PATCH] refactor(ecstore): unify remaining heal logs to structured event style (#5720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #5719 fixed the issue #5716 per-object heal log amplification (per-object statements demoted, heal spans forced to TRACE, raw metadata dumps banned by guardrail) and superseded the demotion originally proposed here. This PR now carries only the residual cleanup on top of it: - Convert the remaining bare-field and format-arg heal logs in crates/ecstore/src/set_disk/ops/heal.rs to the file's structured convention (event/component/subsystem + context fields): missing-object skip, disk-marked-for-healing, cannot-reconstruct errors, dangling-cleanup error, missing data_dir error, xl.meta regeneration warn, and orphan-reclaim failure warn. - Demote the last remaining info! in the file — the per-set heal_format "set disk formats success, NoHealRequired" no-op message — to a structured debug! (error_count instead of a raw errs dump), and drop its whitelist exclusion in scripts/check_logging_guardrails.sh so the no-INFO check for set-disk heal files is strict. No control flow or behavior changes. --- crates/ecstore/src/set_disk/ops/heal.rs | 59 ++++++++++++++++++++++--- scripts/check_logging_guardrails.sh | 3 +- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 2e164a6f4..1735bd595 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -404,7 +404,16 @@ impl SetDisks { "Set disk object metadata read" ); if DiskError::is_all_not_found(&errs) { - debug!(bucket, object, version_id, "heal_object skipped missing object"); + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + version_id, + state = "missing_object_skipped", + "Set disk heal skipped missing object" + ); let err = if !version_id.is_empty() { DiskError::FileVersionNotFound } else { @@ -525,7 +534,18 @@ impl SetDisks { if is_meta { meta_to_heal_count += 1; } - debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]); + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + version_id, + disk_index = index, + endpoint = %self.set_endpoints[index], + state = "disk_marked_for_healing", + "Set disk marked for healing" + ); } let drive_state = match reason { @@ -609,6 +629,8 @@ impl SetDisks { latest_meta.erasure.data_blocks ); error!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, bucket, object, version_id, @@ -627,6 +649,8 @@ impl SetDisks { latest_meta.erasure.parity_blocks ); error!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, bucket, object, version_id, @@ -682,6 +706,8 @@ impl SetDisks { } Err(err) => { error!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, bucket, object, version_id, @@ -808,8 +834,12 @@ impl SetDisks { None => { if !latest_meta.deleted && !latest_meta.is_remote() { error!( - "heal: latest metadata for {}/{} has no data_dir, cannot heal object data", - bucket, object + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + bucket, + object, + version_id, + "Heal object latest metadata has no data_dir, cannot heal object data" ); return Err(DiskError::FileCorrupt); } @@ -1293,6 +1323,8 @@ impl SetDisks { Ok(()) => wrote += 1, Err(error) => { warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, bucket, object, disk_index = index, @@ -1326,7 +1358,16 @@ impl SetDisks { } Ok(_) => {} Err(e) => { - warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed"); + warn!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + error = %e, + state = "orphan_data_reclaim_failed", + "Set disk orphan data-dir reclaim failed" + ); } } } @@ -1730,7 +1771,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { }; if count_errs(&errs, &DiskError::UnformattedDisk) == 0 { - info!("set disk formats success, NoHealRequired, errs: {:?}", errs); + debug!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + error_count = errs.iter().flatten().count(), + result = "no_heal_required", + "set disk formats success" + ); return Ok((result, Some(StorageError::NoHealRequired))); } diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index 8deeb9fe1..fac8bf0bd 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -711,8 +711,7 @@ if [[ "$heal_function_count" != "$trace_heal_instrumentation_count" ]]; then fi unexpected_heal_info="$( - rg -n '\binfo!' crates/ecstore/src/set_disk/ops/heal.rs crates/ecstore/src/erasure/coding/heal.rs | - rg -v 'set disk formats success, NoHealRequired' || true + rg -n '\binfo!' crates/ecstore/src/set_disk/ops/heal.rs crates/ecstore/src/erasure/coding/heal.rs || true )" if [[ -n "$unexpected_heal_info" ]]; then echo "❌ logging guardrail violation: per-object set-disk heal events must not be emitted at INFO" >&2