mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 05:26:50 +00:00
refactor(logging): standardize heal and scanner events (#3414)
* refactor(logging): standardize heal and scanner events * chore(git): untrack local logging governance note * chore(git): ignore local logging governance note
This commit is contained in:
+290
-33
@@ -54,6 +54,14 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
const SINGLE_DISK_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
||||
const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
|
||||
const EVENT_SCANNER_CYCLE_STATE: &str = "scanner_cycle_state";
|
||||
const EVENT_SCANNER_LOCK_STATE: &str = "scanner_lock_state";
|
||||
const EVENT_SCANNER_PERSIST_STATE: &str = "scanner_persist_state";
|
||||
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
|
||||
const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state";
|
||||
#[cfg(test)]
|
||||
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||
|
||||
@@ -82,7 +90,15 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
|
||||
match lookup_scanner_runtime_config(config.as_ref()) {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
warn!(error = %err, "Failed to resolve scanner runtime config, using last applied config");
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "resolve_failed",
|
||||
error = %err,
|
||||
"Scanner runtime config resolution failed; using last applied config"
|
||||
);
|
||||
current_scanner_runtime_config()
|
||||
}
|
||||
}
|
||||
@@ -156,8 +172,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
|
||||
Ok(data) => data,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "startup_inspect_failed",
|
||||
error = %err,
|
||||
"Failed to inspect persisted data usage cache; keeping configured scanner startup delay"
|
||||
"Scanner startup cache inspection failed; keeping configured startup delay"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
@@ -169,8 +191,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
|
||||
Ok(info) => data_usage_info_is_cold(&info),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "startup_decode_failed",
|
||||
error = %err,
|
||||
"Failed to decode persisted data usage cache; running initial scanner cycle without startup delay"
|
||||
"Scanner startup cache decode failed; skipping startup delay"
|
||||
);
|
||||
true
|
||||
}
|
||||
@@ -188,8 +216,13 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
|
||||
Ok(buckets) => !buckets.is_empty(),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "startup_bucket_inspect_failed",
|
||||
error = %err,
|
||||
"Failed to inspect buckets for scanner startup delay; keeping configured scanner startup delay"
|
||||
"Scanner startup bucket inspection failed; keeping configured startup delay"
|
||||
);
|
||||
false
|
||||
}
|
||||
@@ -207,7 +240,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
// Force init global sleeper so config is read once at startup.
|
||||
let _ = &*SCANNER_SLEEPER;
|
||||
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
||||
warn!(error = %err, "Failed to apply scanner runtime config at startup");
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "startup_apply_failed",
|
||||
error = %err,
|
||||
"Scanner runtime config apply failed at startup"
|
||||
);
|
||||
}
|
||||
|
||||
let ctx_clone = ctx;
|
||||
@@ -220,7 +261,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
has_buckets,
|
||||
);
|
||||
if sleep_time.is_zero() {
|
||||
info!("Skipping initial data scanner delay because persisted data usage cache is cold");
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "startup_delay_skipped",
|
||||
reason = "usage_cache_cold",
|
||||
"Scanner startup delay skipped"
|
||||
);
|
||||
} else {
|
||||
tokio::time::sleep(sleep_time).await;
|
||||
}
|
||||
@@ -231,7 +280,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
}
|
||||
|
||||
if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await {
|
||||
error!("Failed to run data scanner: {e}");
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "run_failed",
|
||||
error = %e,
|
||||
"Scanner runtime iteration failed"
|
||||
);
|
||||
}
|
||||
// Backoff before retrying after lock contention or scanner-level failures.
|
||||
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
|
||||
@@ -276,8 +333,13 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
|
||||
Ok(buckets) => buckets,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "maintenance_feature_inspect_failed",
|
||||
error = %err,
|
||||
"Failed to inspect scanner maintenance features; preserving speed-based scanner cycle"
|
||||
"Scanner maintenance feature inspection failed; preserving speed-based cycle"
|
||||
);
|
||||
features.inspection_failed = true;
|
||||
return features;
|
||||
@@ -293,9 +355,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
|
||||
Err(EcstoreError::ConfigNotFound) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
bucket = %bucket.name,
|
||||
state = "lifecycle_inspect_failed",
|
||||
error = %err,
|
||||
"Failed to inspect bucket lifecycle config; preserving speed-based scanner cycle"
|
||||
"Scanner lifecycle inspection failed; preserving speed-based cycle"
|
||||
);
|
||||
features.inspection_failed = true;
|
||||
}
|
||||
@@ -310,9 +377,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
|
||||
Err(EcstoreError::ConfigNotFound) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
bucket = %bucket.name,
|
||||
state = "replication_inspect_failed",
|
||||
error = %err,
|
||||
"Failed to inspect bucket replication config; preserving speed-based scanner cycle"
|
||||
"Scanner replication inspection failed; preserving speed-based cycle"
|
||||
);
|
||||
features.inspection_failed = true;
|
||||
}
|
||||
@@ -334,6 +406,10 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
|
||||
set_scanner_default_speed(ScannerSpeed::Slowest);
|
||||
set_scanner_default_cycle_secs(default_cycle_secs);
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
env_speed = ENV_SCANNER_SPEED,
|
||||
env_cycle = ENV_SCANNER_CYCLE,
|
||||
env_start_delay = ENV_SCANNER_START_DELAY_SECS,
|
||||
@@ -341,7 +417,8 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
|
||||
lifecycle_active = features.lifecycle,
|
||||
replication_active = features.replication,
|
||||
feature_inspection_failed = features.inspection_failed,
|
||||
"Using slower scanner defaults for single-disk deployments; regular scanner cycles are preserved when maintenance features are active"
|
||||
state = "single_disk_defaults_applied",
|
||||
"Scanner defaults updated for single-disk deployment"
|
||||
);
|
||||
} else {
|
||||
set_scanner_default_speed(ScannerSpeed::Default);
|
||||
@@ -477,13 +554,31 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
|
||||
// Get last healing information
|
||||
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
|
||||
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
|
||||
error!("Failed to unmarshal background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "decode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state decode failed"
|
||||
);
|
||||
BackgroundHealInfo::default()
|
||||
}),
|
||||
Err(e) => {
|
||||
// Only log if it's not a ConfigNotFound error
|
||||
if e != EcstoreError::ConfigNotFound {
|
||||
warn!("Failed to read background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "read_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state read failed"
|
||||
);
|
||||
}
|
||||
BackgroundHealInfo::default()
|
||||
}
|
||||
@@ -502,14 +597,32 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
|
||||
let data = match serde_json::to_vec(&info) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
error!("Failed to marshal background heal info: {}", e);
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "encode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state encode failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Save configuration
|
||||
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
|
||||
warn!("Failed to save background heal info to {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state save failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -530,7 +643,15 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
|
||||
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
||||
warn!(error = %err, "Failed to refresh scanner runtime config, using last applied config");
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "refresh_failed",
|
||||
error = %err,
|
||||
"Scanner runtime config refresh failed; using last applied config"
|
||||
);
|
||||
}
|
||||
let configured_cycle_interval = scanner_cycle_interval();
|
||||
let configured_bitrot_cycle = scanner_bitrot_cycle();
|
||||
@@ -542,7 +663,6 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
cycle_budget_config.max_objects,
|
||||
cycle_budget_config.max_directories,
|
||||
);
|
||||
info!("Start run data scanner cycle");
|
||||
cycle_info.current = cycle_info.next;
|
||||
let now = Instant::now();
|
||||
cycle_info.started = Utc::now();
|
||||
@@ -557,6 +677,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
background_heal_info.bitrot_start_time,
|
||||
configured_bitrot_cycle,
|
||||
);
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
scan_mode = ?scan_mode,
|
||||
state = "started",
|
||||
"Scanner cycle state updated"
|
||||
);
|
||||
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_start(
|
||||
background_heal_info.clone(),
|
||||
@@ -589,12 +719,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
global_metrics().finish_scan_cycle_work(cycle_work_start);
|
||||
if budget_elapsed {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
duration = ?now.elapsed(),
|
||||
reason = ?cycle_budget.reason(),
|
||||
max_duration = ?cycle_budget.max_duration(),
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
"Data scanner cycle stopped after reaching its cycle budget"
|
||||
state = "budget_reached",
|
||||
"Scanner cycle stopped after reaching budget"
|
||||
);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
emit_scan_cycle_partial_with_source(
|
||||
@@ -605,7 +741,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return;
|
||||
}
|
||||
error!(duration = ?now.elapsed(), "Fail run data scanner cycle: {e}");
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
scan_mode = ?scan_mode,
|
||||
state = "failed",
|
||||
duration = ?now.elapsed(),
|
||||
error = %e,
|
||||
"Scanner cycle state updated"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
@@ -614,12 +761,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
}
|
||||
if cycle_budget.budget_elapsed() && !ctx.is_cancelled() {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
duration = ?now.elapsed(),
|
||||
reason = ?cycle_budget.reason(),
|
||||
max_duration = ?cycle_budget.max_duration(),
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
"Data scanner cycle stopped after reaching its cycle budget"
|
||||
state = "budget_reached",
|
||||
"Scanner cycle stopped after reaching budget"
|
||||
);
|
||||
global_metrics().finish_scan_cycle_work(cycle_work_start);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
@@ -643,7 +796,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
cycle_info.cycle_completed.push(Utc::now());
|
||||
global_metrics().clear_current_scan_mode();
|
||||
|
||||
info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
scan_mode = ?scan_mode,
|
||||
state = "completed",
|
||||
duration = ?now.elapsed(),
|
||||
cycles_total = cycle_info.cycle_completed.len(),
|
||||
"Scanner cycle state updated"
|
||||
);
|
||||
|
||||
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
@@ -655,9 +819,26 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
buf.extend_from_slice(&cycle_info_buf);
|
||||
|
||||
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf).await {
|
||||
error!("Failed to save data usage bloom name to {}: {}", &*DATA_USAGE_BLOOM_NAME_PATH, e);
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "failed",
|
||||
error = %e,
|
||||
"Scanner state persistence failed"
|
||||
);
|
||||
} else {
|
||||
info!("Data usage bloom name saved successfully");
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "saved",
|
||||
"Scanner state persisted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -666,16 +847,42 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
let _guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
||||
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => {
|
||||
debug!("run_data_scanner: acquired leader write lock");
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_LOCK_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
lock_name = "leader.lock",
|
||||
state = "acquired",
|
||||
"Scanner leader lock state updated"
|
||||
);
|
||||
guard
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("run_data_scanner: other node is running, failed to acquire leader write lock: {:?}", e);
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_LOCK_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
lock_name = "leader.lock",
|
||||
state = "contended",
|
||||
error = ?e,
|
||||
"Scanner leader lock state updated"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("run_data_scanner: failed to create namespace lock: {e}");
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_LOCK_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
lock_name = "leader.lock",
|
||||
state = "create_failed",
|
||||
error = %e,
|
||||
"Scanner leader lock state updated"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
@@ -689,7 +896,16 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
} else if buf.len() > 8 {
|
||||
cycle_info.next = u64::from_le_bytes(buf[0..8].try_into().unwrap_or_default());
|
||||
if let Err(e) = cycle_info.unmarshal(&buf[8..]) {
|
||||
warn!("Failed to unmarshal cycle info: {e}");
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "cycle_decode_failed",
|
||||
error = %e,
|
||||
"Scanner cycle state decode failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -714,7 +930,14 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
|
||||
global_metrics().set_cycle(None).await;
|
||||
|
||||
debug!("Data scanner done");
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "stopped",
|
||||
"Scanner runtime stopped"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -755,8 +978,15 @@ pub async fn store_data_usage_in_backend(
|
||||
&& new_ts <= existing_ts
|
||||
{
|
||||
info!(
|
||||
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
|
||||
new_ts, existing_ts
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
incoming_last_update = ?new_ts,
|
||||
existing_last_update = ?existing_ts,
|
||||
state = "skip_stale_update",
|
||||
"Scanner data usage persistence skipped stale update"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -765,7 +995,16 @@ pub async fn store_data_usage_in_backend(
|
||||
let data = match serde_json::to_vec(&data_usage_info) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
error!("Failed to marshal data usage info: {}", e);
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "encode_failed",
|
||||
error = %e,
|
||||
"Scanner data usage persistence encode failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -775,7 +1014,16 @@ pub async fn store_data_usage_in_backend(
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
|
||||
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %backup_path,
|
||||
state = "backup_save_failed",
|
||||
error = %e,
|
||||
"Scanner data usage backup save failed"
|
||||
);
|
||||
}
|
||||
done_save();
|
||||
attempts = 1;
|
||||
@@ -784,7 +1032,16 @@ pub async fn store_data_usage_in_backend(
|
||||
// Save main configuration
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
|
||||
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner data usage persistence failed"
|
||||
);
|
||||
} else {
|
||||
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user