gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 02:33:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): standardize heal and scanner events (#3414)

Browse Source 
* refactor(logging): standardize heal and scanner events

* chore(git): untrack local logging governance note

* chore(git): ignore local logging governance note
This commit is contained in:
houseme
2026-06-14 01:47:39 +08:00
committed by GitHub
parent 9059a9c68d
commit 7da10db852
14 changed files with 3425 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/data_usage_define.rs
+25 -2
View File
@@ -54,6 +54,10 @@ const METRIC_CACHE_SAVE_ATTEMPT_TOTAL: &str = "rustfs_scanner_cache_save_attempt
const METRIC_CACHE_SAVE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cache_save_timeout_total";
const METRIC_CACHE_SAVE_RETRY_TOTAL: &str = "rustfs_scanner_cache_save_retry_total";
const METRIC_CACHE_SAVE_DURATION_SECONDS: &str = "rustfs_scanner_cache_save_duration_seconds";
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_CACHE: &str = "cache";
const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state";
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
@@ -651,7 +655,16 @@ impl DataUsageCache {
}
if retries == 5 {
warn!("maximum retry reached to load the data usage cache `{}`", name);
warn!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_LOAD_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_CACHE,
cache_name = %name,
retries,
state = "max_retries_reached",
"Scanner cache load reached retry limit"
);
}
Ok(())
@@ -891,7 +904,17 @@ impl DataUsageCache {
Self::save_path_with_retry(store, &backup_path, &buf, backup_timeout_duration, DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES)
.await
{
warn!("Failed to save data usage cache backup: {e}");
warn!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_SAVE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_CACHE,
cache_name = %name,
backup_path = %backup_path,
state = "backup_save_failed",
error = %e,
"Scanner cache backup save failed"
);
}
Ok(())
}
crates/scanner/src/runtime_config.rs
+21 -2
View File
@@ -39,6 +39,10 @@ use std::sync::{LazyLock, RwLock};
use std::time::Duration;
use tracing::warn;
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_RUNTIME_CONFIG: &str = "runtime_config";
const EVENT_SCANNER_RUNTIME_CONFIG_PARSE: &str = "scanner_runtime_config_parse";
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
const NO_DEFAULT_CYCLE_OVERRIDE: u64 = 0;
const MAX_SCANNER_DELAY_FACTOR: f64 = 10_000.0;
@@ -274,7 +278,17 @@ fn parse_env_delay(key: &'static str, value: String, fallback: f64) -> f64 {
match parse_config_delay(key, value.clone()) {
Ok(parsed) => parsed,
Err(_) => {
warn!(env = key, value, fallback, "Invalid scanner delay config, using derived value");
warn!(
target: "rustfs::scanner::runtime_config",
event = EVENT_SCANNER_RUNTIME_CONFIG_PARSE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME_CONFIG,
env = key,
value,
fallback,
state = "invalid_delay",
"Scanner runtime config used derived delay fallback"
);
fallback
}
}
@@ -294,10 +308,15 @@ fn parse_env_bitrot_cycle(value: String) -> Option<Duration> {
"false" | "off" | "no" | "disabled" => None,
value => value.parse::<u64>().ok().map(Duration::from_secs).or_else(|| {
warn!(
target: "rustfs::scanner::runtime_config",
event = EVENT_SCANNER_RUNTIME_CONFIG_PARSE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME_CONFIG,
env = ENV_SCANNER_BITROT_CYCLE_SECS,
value,
default_secs = DEFAULT_HEAL_BITROT_CYCLE_SECS,
"Invalid scanner bitrot cycle, using default"
state = "invalid_bitrot_cycle",
"Scanner runtime config used default bitrot cycle"
);
Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS))
}),
crates/scanner/src/scanner.rs
+290 -33
View File
@@ -54,6 +54,14 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
const SINGLE_DISK_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
const EVENT_SCANNER_CYCLE_STATE: &str = "scanner_cycle_state";
const EVENT_SCANNER_LOCK_STATE: &str = "scanner_lock_state";
const EVENT_SCANNER_PERSIST_STATE: &str = "scanner_persist_state";
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state";
#[cfg(test)]
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
@@ -82,7 +90,15 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
match lookup_scanner_runtime_config(config.as_ref()) {
Ok(config) => config,
Err(err) => {
warn!(error = %err, "Failed to resolve scanner runtime config, using last applied config");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "resolve_failed",
error = %err,
"Scanner runtime config resolution failed; using last applied config"
);
current_scanner_runtime_config()
}
}
@@ -156,8 +172,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
Ok(data) => data,
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_inspect_failed",
error = %err,
"Failed to inspect persisted data usage cache; keeping configured scanner startup delay"
"Scanner startup cache inspection failed; keeping configured startup delay"
);
return false;
}
@@ -169,8 +191,14 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
Ok(info) => data_usage_info_is_cold(&info),
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_decode_failed",
error = %err,
"Failed to decode persisted data usage cache; running initial scanner cycle without startup delay"
"Scanner startup cache decode failed; skipping startup delay"
);
true
}
@@ -188,8 +216,13 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
Ok(buckets) => !buckets.is_empty(),
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_bucket_inspect_failed",
error = %err,
"Failed to inspect buckets for scanner startup delay; keeping configured scanner startup delay"
"Scanner startup bucket inspection failed; keeping configured startup delay"
);
false
}
@@ -207,7 +240,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
// Force init global sleeper so config is read once at startup.
let _ = &*SCANNER_SLEEPER;
if let Err(err) = refresh_scanner_runtime_config_from_global() {
warn!(error = %err, "Failed to apply scanner runtime config at startup");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_apply_failed",
error = %err,
"Scanner runtime config apply failed at startup"
);
}
let ctx_clone = ctx;
@@ -220,7 +261,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
has_buckets,
);
if sleep_time.is_zero() {
info!("Skipping initial data scanner delay because persisted data usage cache is cold");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_delay_skipped",
reason = "usage_cache_cold",
"Scanner startup delay skipped"
);
} else {
tokio::time::sleep(sleep_time).await;
}
@@ -231,7 +280,15 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
}
if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await {
error!("Failed to run data scanner: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "run_failed",
error = %e,
"Scanner runtime iteration failed"
);
}
// Backoff before retrying after lock contention or scanner-level failures.
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
@@ -276,8 +333,13 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Ok(buckets) => buckets,
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "maintenance_feature_inspect_failed",
error = %err,
"Failed to inspect scanner maintenance features; preserving speed-based scanner cycle"
"Scanner maintenance feature inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
return features;
@@ -293,9 +355,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Err(EcstoreError::ConfigNotFound) => {}
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
bucket = %bucket.name,
state = "lifecycle_inspect_failed",
error = %err,
"Failed to inspect bucket lifecycle config; preserving speed-based scanner cycle"
"Scanner lifecycle inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
}
@@ -310,9 +377,14 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> Scanner
Err(EcstoreError::ConfigNotFound) => {}
Err(err) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
bucket = %bucket.name,
state = "replication_inspect_failed",
error = %err,
"Failed to inspect bucket replication config; preserving speed-based scanner cycle"
"Scanner replication inspection failed; preserving speed-based cycle"
);
features.inspection_failed = true;
}
@@ -334,6 +406,10 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
set_scanner_default_speed(ScannerSpeed::Slowest);
set_scanner_default_cycle_secs(default_cycle_secs);
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
env_speed = ENV_SCANNER_SPEED,
env_cycle = ENV_SCANNER_CYCLE,
env_start_delay = ENV_SCANNER_START_DELAY_SECS,
@@ -341,7 +417,8 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
lifecycle_active = features.lifecycle,
replication_active = features.replication,
feature_inspection_failed = features.inspection_failed,
"Using slower scanner defaults for single-disk deployments; regular scanner cycles are preserved when maintenance features are active"
state = "single_disk_defaults_applied",
"Scanner defaults updated for single-disk deployment"
);
} else {
set_scanner_default_speed(ScannerSpeed::Default);
@@ -477,13 +554,31 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
// Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
error!("Failed to unmarshal background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal state decode failed"
);
BackgroundHealInfo::default()
}),
Err(e) => {
// Only log if it's not a ConfigNotFound error
if e != EcstoreError::ConfigNotFound {
warn!("Failed to read background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "read_failed",
error = %e,
"Scanner background heal state read failed"
);
}
BackgroundHealInfo::default()
}
@@ -502,14 +597,32 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
let data = match serde_json::to_vec(&info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal background heal info: {}", e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "encode_failed",
error = %e,
"Scanner background heal state encode failed"
);
return;
}
};
// Save configuration
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!("Failed to save background heal info to {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "save_failed",
error = %e,
"Scanner background heal state save failed"
);
}
}
@@ -530,7 +643,15 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
let _activity_guard = ScannerActivityGuard::new();
if let Err(err) = refresh_scanner_runtime_config_from_global() {
warn!(error = %err, "Failed to refresh scanner runtime config, using last applied config");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_RUNTIME_CONFIG,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "refresh_failed",
error = %err,
"Scanner runtime config refresh failed; using last applied config"
);
}
let configured_cycle_interval = scanner_cycle_interval();
let configured_bitrot_cycle = scanner_bitrot_cycle();
@@ -542,7 +663,6 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_budget_config.max_objects,
cycle_budget_config.max_directories,
);
info!("Start run data scanner cycle");
cycle_info.current = cycle_info.next;
let now = Instant::now();
cycle_info.started = Utc::now();
@@ -557,6 +677,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
background_heal_info.bitrot_start_time,
configured_bitrot_cycle,
);
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "started",
"Scanner cycle state updated"
);
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = background_heal_info_for_scan_start(
background_heal_info.clone(),
@@ -589,12 +719,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
global_metrics().finish_scan_cycle_work(cycle_work_start);
if budget_elapsed {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
);
let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source(
@@ -605,7 +741,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
mark_scan_cycle_idle(cycle_info).await;
return;
}
error!(duration = ?now.elapsed(), "Fail run data scanner cycle: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "failed",
duration = ?now.elapsed(),
error = %e,
"Scanner cycle state updated"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
save_background_heal_info(storeapi.clone(), new_heal_info).await;
@@ -614,12 +761,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
}
if cycle_budget.budget_elapsed() && !ctx.is_cancelled() {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
);
global_metrics().finish_scan_cycle_work(cycle_work_start);
let budget_reason = cycle_budget.reason();
@@ -643,7 +796,18 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_info.cycle_completed.push(Utc::now());
global_metrics().clear_current_scan_mode();
info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "completed",
duration = ?now.elapsed(),
cycles_total = cycle_info.cycle_completed.len(),
"Scanner cycle state updated"
);
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
global_metrics().set_cycle(Some(cycle_info.clone())).await;
@@ -655,9 +819,26 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
buf.extend_from_slice(&cycle_info_buf);
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf).await {
error!("Failed to save data usage bloom name to {}: {}", &*DATA_USAGE_BLOOM_NAME_PATH, e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "failed",
error = %e,
"Scanner state persistence failed"
);
} else {
info!("Data usage bloom name saved successfully");
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "saved",
"Scanner state persisted"
);
}
}
@@ -666,16 +847,42 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
let _guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
Ok(guard) => {
debug!("run_data_scanner: acquired leader write lock");
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "acquired",
"Scanner leader lock state updated"
);
guard
}
Err(e) => {
debug!("run_data_scanner: other node is running, failed to acquire leader write lock: {:?}", e);
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "contended",
error = ?e,
"Scanner leader lock state updated"
);
return Ok(());
}
},
Err(e) => {
error!("run_data_scanner: failed to create namespace lock: {e}");
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_LOCK_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "create_failed",
error = %e,
"Scanner leader lock state updated"
);
return Ok(());
}
};
@@ -689,7 +896,16 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
} else if buf.len() > 8 {
cycle_info.next = u64::from_le_bytes(buf[0..8].try_into().unwrap_or_default());
if let Err(e) = cycle_info.unmarshal(&buf[8..]) {
warn!("Failed to unmarshal cycle info: {e}");
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "cycle_decode_failed",
error = %e,
"Scanner cycle state decode failed"
);
}
}
@@ -714,7 +930,14 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
global_metrics().set_cycle(None).await;
debug!("Data scanner done");
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped",
"Scanner runtime stopped"
);
Ok(())
}
@@ -755,8 +978,15 @@ pub async fn store_data_usage_in_backend(
&& new_ts <= existing_ts
{
info!(
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
new_ts, existing_ts
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
incoming_last_update = ?new_ts,
existing_last_update = ?existing_ts,
state = "skip_stale_update",
"Scanner data usage persistence skipped stale update"
);
continue;
}
@@ -765,7 +995,16 @@ pub async fn store_data_usage_in_backend(
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal data usage info: {}", e);
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "encode_failed",
error = %e,
"Scanner data usage persistence encode failed"
);
continue;
}
};
@@ -775,7 +1014,16 @@ pub async fn store_data_usage_in_backend(
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %backup_path,
state = "backup_save_failed",
error = %e,
"Scanner data usage backup save failed"
);
}
done_save();
attempts = 1;
@@ -784,7 +1032,16 @@ pub async fn store_data_usage_in_backend(
// Save main configuration
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "save_failed",
error = %e,
"Scanner data usage persistence failed"
);
} else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
crates/scanner/src/scanner_folder.rs
+417 -47
View File
@@ -68,6 +68,16 @@ use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_FOLDER: &str = "folder";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const LOG_SUBSYSTEM_HEAL: &str = "heal";
const EVENT_SCANNER_FOLDER_STATE: &str = "scanner_folder_state";
const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action";
const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission";
const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state";
const EVENT_SCANNER_COMPAT_STATE: &str = "scanner_compat_state";
const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16;
const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
@@ -361,8 +371,13 @@ fn warn_inline_heal_compat_requested() {
SCANNER_INLINE_HEAL_WARN_ONCE.call_once(|| {
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_COMPAT_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
env = rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE,
"Inline scanner heal rollback is no longer supported; continuing to enqueue heal candidates asynchronously"
state = "inline_heal_rollback_unsupported",
"Scanner inline-heal rollback is unsupported; using async heal admission"
);
});
}
@@ -541,13 +556,18 @@ async fn send_required_scanner_heal_request(
record_high_priority_heal_escalation(candidate_type, priority, result);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
candidate_type,
bucket,
object = object.unwrap_or(""),
priority = heal_priority_label(priority),
admission = result.result_label(),
reason = result.reason_label(),
"High-priority heal request was not admitted; escalating to scanner failure"
state = "high_priority_not_admitted",
"Scanner high-priority heal admission failed"
);
Err(build_high_priority_heal_admission_error(candidate_type, bucket, object, priority, result))
}
@@ -601,15 +621,39 @@ impl ScannerItem {
size_summary: &mut SizeSummary,
) {
if object_infos.is_empty() {
debug!("apply_actions: no object infos for object: {}", self.object_path());
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "no_object_versions",
"Scanner lifecycle action skipped"
);
return;
}
debug!("apply_actions: applying actions for object: {}", self.object_path());
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "started",
"Scanner lifecycle action evaluation started"
);
let versioning_config = match BucketVersioningSys::get(&self.bucket).await {
Ok(versioning_config) => versioning_config,
Err(_) => {
warn!("apply_actions: Failed to get versioning configuration for bucket {}", self.bucket);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
state = "versioning_lookup_failed",
"Scanner lifecycle action skipped"
);
return;
}
};
@@ -620,7 +664,16 @@ impl ScannerItem {
let actual_size = match oi.get_actual_size() {
Ok(size) => size,
Err(_) => {
warn!("apply_actions: Failed to get actual size for object {}", oi.name);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %oi.name,
state = "size_lookup_failed",
"Scanner lifecycle action used fallback size"
);
continue;
}
};
@@ -634,7 +687,15 @@ impl ScannerItem {
self.alert_excessive_versions(object_infos.len(), cumulative_size);
debug!("apply_actions: done for now no lifecycle config");
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "no_lifecycle_config",
"Scanner lifecycle action finished without lifecycle rules"
);
return;
};
@@ -651,7 +712,16 @@ impl ScannerItem {
{
Ok(events) => events,
Err(e) => {
warn!("apply_actions: Failed to evaluate lifecycle for object: {}", e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "evaluate_failed",
error = %e,
"Scanner lifecycle action evaluation failed"
);
return;
}
};
@@ -666,7 +736,16 @@ impl ScannerItem {
let actual_size = match oi.get_actual_size() {
Ok(size) => size,
Err(_) => {
warn!("apply_actions: Failed to get actual size for object {}", oi.name);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %oi.name,
state = "size_lookup_failed",
"Scanner lifecycle action used fallback size"
);
0
}
};
@@ -676,7 +755,17 @@ impl ScannerItem {
match event.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %oi.name,
action = %event.action,
state = "apply_expiry_rule",
"Scanner lifecycle action dispatched"
);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
@@ -693,7 +782,16 @@ impl ScannerItem {
let retained_size = match retained.get_actual_size() {
Ok(size) => size,
Err(_) => {
warn!("apply_actions: Failed to get actual size for object {}", retained.name);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %retained.name,
state = "size_lookup_failed",
"Scanner lifecycle action used fallback size"
);
0
}
};
@@ -709,7 +807,17 @@ impl ScannerItem {
}
IlmAction::DeleteAction | IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %oi.name,
action = %event.action,
state = "apply_expiry_rule",
"Scanner lifecycle action dispatched"
);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
@@ -737,7 +845,17 @@ impl ScannerItem {
noncurrent_events.push(event.clone());
}
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
object = %oi.name,
action = %event.action,
state = "apply_transition_rule",
"Scanner lifecycle action dispatched"
);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
@@ -853,10 +971,15 @@ impl ScannerItem {
async fn enqueue_heal(&mut self, oi: &ObjectInfo) {
let done_heal = Metrics::time(Metric::HealAbandonedObject);
debug!(
"enqueue_heal: bucket: {}, object_path: {}, version_id: {}",
self.bucket,
self.object_path(),
oi.version_id.unwrap_or_default()
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
bucket = %self.bucket,
object = %self.object_path(),
version_id = %oi.version_id.unwrap_or_default(),
state = "request_started",
"Scanner heal admission started"
);
let now = OffsetDateTime::now_utc();
@@ -868,6 +991,10 @@ impl ScannerItem {
age.whole_seconds().max(0)
});
info!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
bucket = %self.bucket,
object = %self.object_path(),
version_id = %oi.version_id.unwrap_or_default(),
@@ -875,7 +1002,8 @@ impl ScannerItem {
cooldown_secs = cooldown.as_secs(),
original_scan_mode = %HealScanMode::Deep.as_str(),
effective_scan_mode = %scan_mode.as_str(),
"scanner deep heal downgraded to normal during new-object cooldown"
state = "downgraded_to_normal",
"Scanner heal admission downgraded deep scan during cooldown"
);
}
@@ -898,13 +1026,28 @@ impl ScannerItem {
Ok(HealAdmissionResult::Accepted | HealAdmissionResult::Merged) => {}
Ok(result @ (HealAdmissionResult::Full | HealAdmissionResult::Dropped(_))) => {
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
bucket = %self.bucket,
object = %self.object_path(),
admission = %describe_heal_admission(result),
"enqueue_heal: low-priority heal request was not admitted"
state = "not_admitted",
"Scanner heal admission rejected low-priority request"
);
}
Err(e) => warn!("enqueue_heal: failed to submit heal request: {}", e),
Err(e) => warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_HEAL,
bucket = %self.bucket,
object = %self.object_path(),
state = "submit_failed",
error = %e,
"Scanner heal admission submission failed"
),
}
if admitted {
done_heal();
@@ -922,11 +1065,16 @@ impl ScannerItem {
)
.increment(1);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_ALERT_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
bucket = %self.bucket,
object = %self.object_path(),
versions = remaining_versions,
threshold = scanner_excess_versions_threshold(),
"scanner detected object with excessive retained versions"
state = "excess_versions",
"Scanner alert recorded excessive retained versions"
);
}
if too_large_versions {
@@ -937,12 +1085,17 @@ impl ScannerItem {
)
.increment(1);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_ALERT_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
bucket = %self.bucket,
object = %self.object_path(),
versions = remaining_versions,
cumulative_size,
threshold = scanner_excess_version_size_threshold(),
"scanner detected object with excessive retained version size"
state = "excess_version_size",
"Scanner alert recorded excessive retained version size"
);
}
}
@@ -1077,11 +1230,16 @@ impl FolderScanner {
)
.increment(1);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_ALERT_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
root = %self.root,
folder,
folders = total_folders,
threshold,
"scanner detected folder with excessive direct subfolders"
state = "excess_folders",
"Scanner alert recorded excessive direct subfolders"
);
}
@@ -1140,7 +1298,16 @@ impl FolderScanner {
{
// Try to send without blocking
if let Err(e) = updates.send(flat.clone()).await {
error!("send_update: failed to send update: {}", e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
root = %self.new_cache.info.name,
state = "update_send_failed",
error = %e,
"Scanner folder update send failed"
);
}
self.last_update = SystemTime::now();
}
@@ -1190,7 +1357,16 @@ impl FolderScanner {
abandoned_children = self.old_cache.find_children_copy(this_hash.clone());
}
debug!("scan_folder : {}/{}", &self.root, &folder.name);
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
root = %self.root,
folder = %folder.name,
state = "scan_started",
"Scanner folder state updated"
);
let (_, prefix) = path2_bucket_object_with_base_path(&self.root, &folder.name);
let active_life_cycle = if self
@@ -1224,12 +1400,29 @@ impl FolderScanner {
let dir_path = path_join_buf(&[&self.root, &folder.name]);
debug!("scan_folder: dir_path: {:?}", dir_path);
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = ?dir_path,
state = "dir_open",
"Scanner folder state updated"
);
let mut dir_reader = match tokio::fs::read_dir(&dir_path).await {
Ok(dir_reader) => dir_reader,
Err(e) if e.kind() == ErrorKind::NotFound => {
warn!("scan_folder: directory disappeared before read {}: {}", dir_path, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = %dir_path,
state = "dir_missing_before_read",
error = %e,
"Scanner folder state updated"
);
return Ok(());
}
Err(e) => return Err(ScannerError::Io(e)),
@@ -1240,11 +1433,29 @@ impl FolderScanner {
Ok(Some(entry)) => entry,
Ok(None) => break,
Err(e) if e.kind() == ErrorKind::NotFound => {
warn!("scan_folder: directory disappeared during iteration {}: {}", dir_path, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = %dir_path,
state = "dir_missing_during_iteration",
error = %e,
"Scanner folder state updated"
);
break;
}
Err(e) if e.kind() == ErrorKind::NotADirectory => {
warn!("scan_folder: path became non-directory during iteration {}: {}", dir_path, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = %dir_path,
state = "dir_became_non_directory",
error = %e,
"Scanner folder state updated"
);
break;
}
Err(e) => return Err(ScannerError::Io(e)),
@@ -1269,11 +1480,29 @@ impl FolderScanner {
let mut entry_type = match entry.file_type().await {
Ok(entry_type) => entry_type,
Err(e) if e.kind() == ErrorKind::NotFound => {
warn!("scan_folder: entry disappeared before type lookup {}: {}", entry_name, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
entry = %entry_name,
state = "entry_missing_before_type_lookup",
error = %e,
"Scanner folder state updated"
);
continue;
}
Err(e) if e.kind() == ErrorKind::TooManyLinks => {
warn!("scan_folder: entry hit symlink loop before type lookup {}: {}", entry_name, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
entry = %entry_name,
state = "entry_symlink_loop_before_type_lookup",
error = %e,
"Scanner folder state updated"
);
continue;
}
Err(e) => return Err(ScannerError::Io(e)),
@@ -1283,18 +1512,44 @@ impl FolderScanner {
let metadata = match tokio::fs::metadata(&file_path).await {
Ok(metadata) => metadata,
Err(e) if e.kind() == ErrorKind::NotFound => {
warn!("scan_folder: symlink target disappeared before metadata lookup {}: {}", file_path, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
file_path = %file_path,
state = "symlink_target_missing_before_metadata",
error = %e,
"Scanner folder state updated"
);
continue;
}
Err(e) if e.kind() == ErrorKind::TooManyLinks => {
warn!("scan_folder: symlink target hit loop before metadata lookup {}: {}", file_path, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
file_path = %file_path,
state = "symlink_target_loop_before_metadata",
error = %e,
"Scanner folder state updated"
);
continue;
}
Err(e) => return Err(ScannerError::Io(e)),
};
if metadata.is_dir() {
warn!("scan_folder: ignoring symlinked directory {}", file_path);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
file_path = %file_path,
state = "symlink_directory_ignored",
"Scanner folder state updated"
);
continue;
}
@@ -1384,8 +1639,15 @@ impl FolderScanner {
if should_log_failed_object(into.failed_objects) {
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
path = %item.path,
failed_objects = into.failed_objects,
"scan_folder: failed to get size for item {}: {}", item.path, e
state = "get_size_failed",
error = %e,
"Scanner folder failed to get object size"
);
}
}
@@ -1426,7 +1688,15 @@ impl FolderScanner {
if found_objects && is_erasure().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
info!("scan_folder: done for now found an object in erasure mode");
info!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
state = "erasure_object_found",
"Scanner folder stopped descending after finding erasure object"
);
break;
}
@@ -1445,7 +1715,16 @@ impl FolderScanner {
existing_folders.clear();
if self.data_usage_scanner_debug {
debug!("scan_folder: Preemptively compacting: {}, entries: {}", folder.name, new_folders.len());
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
entry_count = new_folders.len(),
state = "preemptive_compaction",
"Scanner folder switched to compacted mode"
);
}
}
@@ -1570,7 +1849,17 @@ impl FolderScanner {
if ctx.is_cancelled() {
return Err(e);
}
warn!("scan_folder: failed to scan child folder {}: {}", folder_item.name, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
child_folder = %folder_item.name,
state = "child_scan_failed",
error = %e,
"Scanner child folder scan failed"
);
continue;
}
tokio::task::yield_now().await;
@@ -1596,13 +1885,31 @@ impl FolderScanner {
// Scan for healing
if abandoned_children.is_empty() || !self.should_heal().await {
debug!("scan_folder: done for now abandoned children are empty or we are not healing");
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
state = "heal_skip_no_abandoned_children",
"Scanner folder skipped heal scan for abandoned children"
);
// If we are not heal scanning, return now.
break;
}
if self.disks.is_empty() || self.disks_quorum == 0 {
debug!("scan_folder: done for now disks are empty or quorum is 0");
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
disks = self.disks.len(),
quorum = self.disks_quorum,
state = "heal_skip_no_quorum",
"Scanner folder skipped heal scan because quorum is unavailable"
);
break;
}
@@ -1660,7 +1967,16 @@ impl FolderScanner {
let agreed_tx = agreed_tx.clone();
Box::pin(async move {
if let Err(e) = agreed_tx.send(entry_name).await {
error!("scan_folder: list_path_raw: failed to send entry name: {}: {}", entry.name, e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
entry = %entry.name,
state = "list_path_agreed_send_failed",
error = %e,
"Scanner list_path_raw agreed callback failed"
);
}
})
})),
@@ -1668,7 +1984,15 @@ impl FolderScanner {
let partial_tx = partial_tx.clone();
Box::pin(async move {
if let Err(e) = partial_tx.send(entries).await {
error!("scan_folder: list_path_raw: failed to send partial err: {}", e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_partial_send_failed",
error = %e,
"Scanner list_path_raw partial callback failed"
);
}
})
})),
@@ -1677,7 +2001,15 @@ impl FolderScanner {
let errs_clone = errs.to_vec();
Box::pin(async move {
if let Err(e) = finished_tx.send(errs_clone).await {
error!("scan_folder: list_path_raw: failed to send finished errs: {}", e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_finished_send_failed",
error = %e,
"Scanner list_path_raw finished callback failed"
);
}
})
})),
@@ -1686,7 +2018,17 @@ impl FolderScanner {
)
.await
{
error!("scan_folder: failed to list path: {}/{}: {}", bucket_clone, prefix_clone, e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
bucket = %bucket_clone,
prefix = %prefix_clone,
state = "list_path_failed",
error = %e,
"Scanner list_path_raw failed"
);
}
});
@@ -1745,7 +2087,17 @@ impl FolderScanner {
let fivs = match entry.file_info_versions(&bucket) {
Ok(fivs) => fivs,
Err(e) => {
error!("scan_folder: list_path_raw: failed to get file info versions: {}", e);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
bucket = %bucket,
entry = %entry.name,
state = "file_info_versions_failed",
error = %e,
"Scanner list_path_raw failed to resolve file versions"
);
send_required_scanner_heal_request(
"object",
&bucket,
@@ -1789,7 +2141,15 @@ impl FolderScanner {
finished_closed = true;
continue;
};
error!("scan_folder: list_path_raw: failed to get finished errs: {:?}", errs);
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_finished_with_errors",
errors = ?errs,
"Scanner list_path_raw finished with disk errors"
);
child_ctx.cancel();
}
_ = child_ctx.cancelled() => {
@@ -1820,7 +2180,17 @@ impl FolderScanner {
if ctx.is_cancelled() {
return Err(e);
}
warn!("scan_folder: failed to scan child folder {}: {}", folder_item.name, e);
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
child_folder = %folder_item.name,
state = "heal_child_scan_failed",
error = %e,
"Scanner heal child folder scan failed"
);
continue;
}
tokio::task::yield_now().await;
crates/scanner/src/scanner_io.rs
+328 -34
View File
@@ -59,6 +59,12 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_IO: &str = "io";
const EVENT_SCANNER_DISK_BUCKET_STATE: &str = "scanner_disk_bucket_state";
const EVENT_SCANNER_DATA_USAGE_STREAM: &str = "scanner_data_usage_stream";
const EVENT_SCANNER_CACHE_PERSIST_STATE: &str = "scanner_cache_persist_state";
const EVENT_SCANNER_SET_STATE: &str = "scanner_set_state";
const METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_set_scan_concurrency_limit";
const METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_disk_scan_concurrency_limit";
@@ -305,12 +311,30 @@ async fn persist_and_publish_cache_snapshot<S: ObjectIO>(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
error!("Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "save_failed",
error = %e,
"Scanner cache snapshot persistence failed"
);
}
done_save();
if let Err(e) = updates.send(cache_snapshot).await {
error!("Failed to send data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publish_failed",
error = %e,
"Scanner cache snapshot publish failed"
);
}
last_update
@@ -379,7 +403,15 @@ impl ScannerIO for ECStore {
if all_buckets.is_empty() {
reset_set_scan_gauges();
if let Err(e) = updates.send(DataUsageInfo::default()).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "empty_bucket_publish_failed",
error = %e,
"Scanner set state update failed"
);
}
return Ok(());
}
@@ -389,7 +421,15 @@ impl ScannerIO for ECStore {
total_results += pool.disk_set.len();
}
if total_results == 0 {
warn!("nsscanner: no disk sets available for non-empty bucket list");
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket_count = all_buckets.len(),
state = "no_disk_sets",
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(());
}
@@ -397,9 +437,14 @@ impl ScannerIO for ECStore {
let set_scan_limit = scanner_max_concurrent_set_scans(total_results);
record_set_scan_concurrency_limit(set_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
total_sets = total_results,
concurrency_limit = set_scan_limit,
"nsscanner: scanner set concurrency budget"
state = "concurrency_budget",
"Scanner set concurrency budget resolved"
);
let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit));
let queued_set_scans = Arc::new(AtomicUsize::new(total_results));
@@ -495,10 +540,15 @@ impl ScannerIO for ECStore {
)
.increment(1);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = %pool_label,
set = %set_label,
error = %e,
"Failed to scan set; continuing scanner cycle"
state = "set_scan_failed",
"Scanner set scan failed; continuing cycle"
);
let mut first_err = first_err_mutex_clone.lock().await;
record_set_scan_failure(&mut first_err, e);
@@ -540,7 +590,15 @@ impl ScannerIO for ECStore {
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "send_merged_failed",
error = %e,
"Scanner merged data usage publish failed"
);
}
}
break;
@@ -559,7 +617,15 @@ impl ScannerIO for ECStore {
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "send_merged_failed",
error = %e,
"Scanner merged data usage publish failed"
);
}
has_sent_once = true;
last_update = merged_last_update;
@@ -604,18 +670,32 @@ impl ScannerIOCache for SetDisks {
let (disks, healing) = self.get_online_disks_with_healing(false).await;
if disks.is_empty() {
debug!("nsscanner_cache: no online disks available for set");
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "no_online_disks",
"Scanner set state found no online disks"
);
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return Ok(());
}
let disk_scan_limit = scanner_max_concurrent_disk_scans(disks.len());
record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
online_disks = disks.len(),
concurrency_limit = disk_scan_limit,
"nsscanner_cache: scanner disk concurrency budget"
state = "disk_concurrency_budget",
"Scanner disk concurrency budget resolved"
);
let disk_scan_semaphore = Arc::new(Semaphore::new(disk_scan_limit));
let queued_disk_bucket_scans = Arc::new(AtomicUsize::new(buckets.len()));
@@ -646,7 +726,16 @@ impl ScannerIOCache for SetDisks {
if old_cache.find(&bucket.name).is_none()
&& let Err(e) = bucket_tx.send(bucket.clone()).await
{
error!("Failed to send bucket info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_bucket_failed",
error = %e,
"Scanner bucket dispatch failed"
);
}
}
@@ -658,7 +747,16 @@ impl ScannerIOCache for SetDisks {
}
if let Err(e) = bucket_tx.send(bucket.clone()).await {
error!("Failed to send bucket info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_bucket_failed",
error = %e,
"Scanner bucket dispatch failed"
);
}
}
}
@@ -795,13 +893,31 @@ impl ScannerIOCache for SetDisks {
set_label_clone.clone(),
);
debug!("nsscanner_disk: got bucket: {}", bucket.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_started",
"Scanner disk bucket state updated"
);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let mut cache = DataUsageCache::default();
if let Err(e) = cache.load(store_clone_clone.clone(), &cache_name).await {
error!("Failed to load data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket state updated"
);
}
if cache.info.name.is_empty() {
@@ -818,7 +934,16 @@ impl ScannerIOCache for SetDisks {
};
}
debug!("nsscanner_disk: cache.info.name: {:?}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = ?cache.info.name,
state = "cache_ready",
"Scanner disk bucket state updated"
);
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
@@ -841,7 +966,16 @@ impl ScannerIOCache for SetDisks {
})
.await
{
error!("Failed to send data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket_name_clone,
state = "send_failed",
error = %e,
"Scanner data usage stream failed"
);
}
}
});
@@ -855,9 +989,27 @@ impl ScannerIOCache for SetDisks {
Ok(scan_outcome) => scan_outcome,
Err(e) => {
if ctx_clone.is_cancelled() {
debug!("Scanner disk scan stopped after cancellation: {}", e);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "cancelled",
error = %e,
"Scanner disk bucket state updated"
);
} else {
error!("Failed to scan disk: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_failed",
error = %e,
"Scanner disk bucket state updated"
);
}
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
@@ -865,13 +1017,32 @@ impl ScannerIOCache for SetDisks {
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
}
done_save();
}
if let Err(e) = update_fut.await {
error!("Failed to update data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "update_join_failed",
error = %e,
"Scanner bucket update task failed"
);
}
continue;
}
@@ -882,40 +1053,114 @@ impl ScannerIOCache for SetDisks {
ScannerDiskScanOutcome::Partial(cache) => {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save partial data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "partial_save_failed",
error = %e,
"Scanner partial bucket cache save failed"
);
}
done_save();
if let Err(e) = update_fut.await {
error!("Failed to update partial data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "partial_update_join_failed",
error = %e,
"Scanner partial bucket update task failed"
);
}
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("Failed to send partial data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_partial_root_failed",
error = %e,
"Scanner partial root entry publish failed"
);
}
continue;
}
};
debug!("nsscanner_disk: got cache: {}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "scan_completed",
"Scanner disk bucket state updated"
);
if let Err(e) = update_fut.await {
error!("nsscanner_disk: Failed to update data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "update_join_failed",
error = %e,
"Scanner bucket update task failed"
);
}
if ctx_clone.is_cancelled() {
break;
}
debug!("nsscanner_disk: sending data usage entry info: {}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "send_root_entry",
"Scanner data usage stream progress updated"
);
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_root_failed",
error = %e,
"Scanner root entry publish failed"
);
}
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
}
done_save();
}
@@ -931,7 +1176,14 @@ impl ScannerIOCache for SetDisks {
send_update_fut.await?;
debug!("nsscanner_cache: done");
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed",
"Scanner set-level disk scan completed"
);
Ok(())
}
@@ -966,7 +1218,17 @@ impl ScannerIODisk for Disk {
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions,
Err(e) => {
error!("Failed to get file info versions: {}/{}, err: {e}", item.bucket, item.object_path());
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %item.bucket,
object = %item.object_path(),
state = "file_info_versions_failed",
error = %e,
"Scanner disk bucket failed to resolve file info versions"
);
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
};
@@ -1072,28 +1334,60 @@ impl ScannerIODisk for Disk {
// TODO: object lock
let Some(ecstore) = new_object_layer_fn() else {
error!("ECStore not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "ecstore_unavailable",
"Scanner disk bucket missing object layer"
);
return Err(StorageError::other("ECStore not available".to_string()));
};
let disk_location = self.get_disk_location();
let (Some(pool_idx), Some(set_idx)) = (disk_location.pool_idx, disk_location.set_idx) else {
error!("Disk location not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "disk_location_unavailable",
"Scanner disk bucket missing disk location"
);
return Err(StorageError::other("Disk location not available".to_string()));
};
let disks_result = StorageAdminApi::disk_set_inventory(ecstore.as_ref(), DiskSetSelector::new(pool_idx, set_idx)).await?;
let Some(disk_idx) = disk_location.disk_idx else {
error!("Disk index not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "disk_index_unavailable",
"Scanner disk bucket missing disk index"
);
return Err(StorageError::other("Disk index not available".to_string()));
};
let local_disk = if let Some(Some(local_disk)) = disks_result.get(disk_idx) {
local_disk.clone()
} else {
error!("Local disk not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "local_disk_unavailable",
"Scanner disk bucket missing local disk"
);
return Err(StorageError::other("Local disk not available".to_string()));
};