refactor(logging): standardize heal and scanner events (#3414)

* refactor(logging): standardize heal and scanner events

* chore(git): untrack local logging governance note

* chore(git): ignore local logging governance note
This commit is contained in:
houseme
2026-06-14 01:47:39 +08:00
committed by GitHub
parent 9059a9c68d
commit 7da10db852
14 changed files with 3425 additions and 363 deletions
+290 -33
View File
@@ -54,6 +54,14 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
const SINGLE_DISK_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
const EVENT_SCANNER_CYCLE_STATE: &str = "scanner_cycle_state";
const EVENT_SCANNER_LOCK_STATE: &str = "scanner_lock_state";
const EVENT_SCANNER_PERSIST_STATE: &str = "scanner_persist_state";
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state";
#[cfg(test)]
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
@@ -82,7 +90,15 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
match lookup_scanner_runtime_config(config.as_ref()) {
Ok(config) => config,
Err(err) => {
warn!(error = %err, "Failed to resolve scanner runtime config, using last applied config");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "resolve_failed",
error = %err,
"Scanner runtime config resolution failed; using last applied config"
);
current_scanner_runtime_config()
}
}
@@ -156,8 +172,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
Ok(data) => data,
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_inspect_failed",
error = %err,
"Failed to inspect persisted data usage cache; keeping configured scanner startup delay"
"Scanner startup cache inspection failed; keeping configured startup delay"
);
return false;
}
@@ -169,8 +191,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
Ok(info) => data_usage_info_is_cold(&info),
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_decode_failed",
error = %err,
"Failed to decode persisted data usage cache; running initial scanner cycle without startup delay"
"Scanner startup cache decode failed; skipping startup delay"
);
true
}
@@ -188,8 +216,13 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
Ok(buckets) => !buckets.is_empty(),
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_bucket_inspect_failed",
error = %err,
"Failed to inspect buckets for scanner startup delay; keeping configured scanner startup delay"
"Scanner startup bucket inspection failed; keeping configured startup delay"
);
false
}
@@ -207,7 +240,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
// Force init global sleeper so config is read once at startup.
let _ = &*SCANNER_SLEEPER;
if let Err(err) = refresh_scanner_runtime_config_from_global() {
warn!(error = %err, "Failed to apply scanner runtime config at startup");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_apply_failed",
error = %err,
"Scanner runtime config apply failed at startup"
);
}
let ctx_clone = ctx;
@@ -220,7 +261,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
has_buckets,
);
if sleep_time.is_zero() {
info!("Skipping initial data scanner delay because persisted data usage cache is cold");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_delay_skipped",
reason = "usage_cache_cold",
"Scanner startup delay skipped"
);
} else {
tokio::time::sleep(sleep_time).await;
}
@@ -231,7 +280,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
}
if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await {
error!("Failed to run data scanner: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "run_failed",
error = %e,
"Scanner runtime iteration failed"
);
}
// Backoff before retrying after lock contention or scanner-level failures.
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
@@ -276,8 +333,13 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Ok(buckets) => buckets,
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "maintenance_feature_inspect_failed",
error = %err,
"Failed to inspect scanner maintenance features; preserving speed-based scanner cycle"
"Scanner maintenance feature inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
return features;
@@ -293,9 +355,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Err(EcstoreError::ConfigNotFound) => {}
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
bucket = %bucket.name,
state = "lifecycle_inspect_failed",
error = %err,
"Failed to inspect bucket lifecycle config; preserving speed-based scanner cycle"
"Scanner lifecycle inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
}
@@ -310,9 +377,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Err(EcstoreError::ConfigNotFound) => {}
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
bucket = %bucket.name,
state = "replication_inspect_failed",
error = %err,
"Failed to inspect bucket replication config; preserving speed-based scanner cycle"
"Scanner replication inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
}
@@ -334,6 +406,10 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
set_scanner_default_speed(ScannerSpeed::Slowest);
set_scanner_default_cycle_secs(default_cycle_secs);
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
env_speed = ENV_SCANNER_SPEED,
env_cycle = ENV_SCANNER_CYCLE,
env_start_delay = ENV_SCANNER_START_DELAY_SECS,
@@ -341,7 +417,8 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
lifecycle_active = features.lifecycle,
replication_active = features.replication,
feature_inspection_failed = features.inspection_failed,
"Using slower scanner defaults for single-disk deployments; regular scanner cycles are preserved when maintenance features are active"
state = "single_disk_defaults_applied",
"Scanner defaults updated for single-disk deployment"
);
} else {
set_scanner_default_speed(ScannerSpeed::Default);
@@ -477,13 +554,31 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
// Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
error!("Failed to unmarshal background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal state decode failed"
);
BackgroundHealInfo::default()
}),
Err(e) => {
// Only log if it's not a ConfigNotFound error
if e != EcstoreError::ConfigNotFound {
warn!("Failed to read background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "read_failed",
error = %e,
"Scanner background heal state read failed"
);
}
BackgroundHealInfo::default()
}
@@ -502,14 +597,32 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
let data = match serde_json::to_vec(&info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal background heal info: {}", e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "encode_failed",
error = %e,
"Scanner background heal state encode failed"
);
return;
}
};
// Save configuration
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!("Failed to save background heal info to {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "save_failed",
error = %e,
"Scanner background heal state save failed"
);
}
}
@@ -530,7 +643,15 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
let _activity_guard = ScannerActivityGuard::new();
if let Err(err) = refresh_scanner_runtime_config_from_global() {
warn!(error = %err, "Failed to refresh scanner runtime config, using last applied config");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "refresh_failed",
error = %err,
"Scanner runtime config refresh failed; using last applied config"
);
}
let configured_cycle_interval = scanner_cycle_interval();
let configured_bitrot_cycle = scanner_bitrot_cycle();
@@ -542,7 +663,6 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_budget_config.max_objects,
cycle_budget_config.max_directories,
);
info!("Start run data scanner cycle");
cycle_info.current = cycle_info.next;
let now = Instant::now();
cycle_info.started = Utc::now();
@@ -557,6 +677,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
background_heal_info.bitrot_start_time,
configured_bitrot_cycle,
);
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "started",
"Scanner cycle state updated"
);
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = background_heal_info_for_scan_start(
background_heal_info.clone(),
@@ -589,12 +719,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
global_metrics().finish_scan_cycle_work(cycle_work_start);
if budget_elapsed {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
);
let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source(
@@ -605,7 +741,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
mark_scan_cycle_idle(cycle_info).await;
return;
}
error!(duration = ?now.elapsed(), "Fail run data scanner cycle: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "failed",
duration = ?now.elapsed(),
error = %e,
"Scanner cycle state updated"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
save_background_heal_info(storeapi.clone(), new_heal_info).await;
@@ -614,12 +761,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
}
if cycle_budget.budget_elapsed() && !ctx.is_cancelled() {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
);
global_metrics().finish_scan_cycle_work(cycle_work_start);
let budget_reason = cycle_budget.reason();
@@ -643,7 +796,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_info.cycle_completed.push(Utc::now());
global_metrics().clear_current_scan_mode();
info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "completed",
duration = ?now.elapsed(),
cycles_total = cycle_info.cycle_completed.len(),
"Scanner cycle state updated"
);
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
global_metrics().set_cycle(Some(cycle_info.clone())).await;
@@ -655,9 +819,26 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
buf.extend_from_slice(&cycle_info_buf);
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf).await {
error!("Failed to save data usage bloom name to {}: {}", &*DATA_USAGE_BLOOM_NAME_PATH, e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "failed",
error = %e,
"Scanner state persistence failed"
);
} else {
info!("Data usage bloom name saved successfully");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "saved",
"Scanner state persisted"
);
}
}
@@ -666,16 +847,42 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
let _guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
Ok(guard) => {
debug!("run_data_scanner: acquired leader write lock");
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "acquired",
"Scanner leader lock state updated"
);
guard
}
Err(e) => {
debug!("run_data_scanner: other node is running, failed to acquire leader write lock: {:?}", e);
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "contended",
error = ?e,
"Scanner leader lock state updated"
);
return Ok(());
}
},
Err(e) => {
error!("run_data_scanner: failed to create namespace lock: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "create_failed",
error = %e,
"Scanner leader lock state updated"
);
return Ok(());
}
};
@@ -689,7 +896,16 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
} else if buf.len() > 8 {
cycle_info.next = u64::from_le_bytes(buf[0..8].try_into().unwrap_or_default());
if let Err(e) = cycle_info.unmarshal(&buf[8..]) {
warn!("Failed to unmarshal cycle info: {e}");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "cycle_decode_failed",
error = %e,
"Scanner cycle state decode failed"
);
}
}
@@ -714,7 +930,14 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
global_metrics().set_cycle(None).await;
debug!("Data scanner done");
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped",
"Scanner runtime stopped"
);
Ok(())
}
@@ -755,8 +978,15 @@ pub async fn store_data_usage_in_backend(
&& new_ts <= existing_ts
{
info!(
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
new_ts, existing_ts
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
incoming_last_update = ?new_ts,
existing_last_update = ?existing_ts,
state = "skip_stale_update",
"Scanner data usage persistence skipped stale update"
);
continue;
}
@@ -765,7 +995,16 @@ pub async fn store_data_usage_in_backend(
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal data usage info: {}", e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "encode_failed",
error = %e,
"Scanner data usage persistence encode failed"
);
continue;
}
};
@@ -775,7 +1014,16 @@ pub async fn store_data_usage_in_backend(
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %backup_path,
state = "backup_save_failed",
error = %e,
"Scanner data usage backup save failed"
);
}
done_save();
attempts = 1;
@@ -784,7 +1032,16 @@ pub async fn store_data_usage_in_backend(
// Save main configuration
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "save_failed",
error = %e,
"Scanner data usage persistence failed"
);
} else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
}