gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 23:47:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): standardize heal and scanner events (#3414)

Browse Source 
* refactor(logging): standardize heal and scanner events

* chore(git): untrack local logging governance note

* chore(git): ignore local logging governance note
This commit is contained in:
houseme
2026-06-14 01:47:39 +08:00
committed by GitHub
parent 9059a9c68d
commit 7da10db852
14 changed files with 3425 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_io.rs
+328 -34
View File
@@ -59,6 +59,12 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_IO: &str = "io";
const EVENT_SCANNER_DISK_BUCKET_STATE: &str = "scanner_disk_bucket_state";
const EVENT_SCANNER_DATA_USAGE_STREAM: &str = "scanner_data_usage_stream";
const EVENT_SCANNER_CACHE_PERSIST_STATE: &str = "scanner_cache_persist_state";
const EVENT_SCANNER_SET_STATE: &str = "scanner_set_state";
const METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_set_scan_concurrency_limit";
const METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_disk_scan_concurrency_limit";
@@ -305,12 +311,30 @@ async fn persist_and_publish_cache_snapshot<S: ObjectIO>(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
error!("Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "save_failed",
error = %e,
"Scanner cache snapshot persistence failed"
);
}
done_save();
if let Err(e) = updates.send(cache_snapshot).await {
error!("Failed to send data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publish_failed",
error = %e,
"Scanner cache snapshot publish failed"
);
}
last_update
@@ -379,7 +403,15 @@ impl ScannerIO for ECStore {
if all_buckets.is_empty() {
reset_set_scan_gauges();
if let Err(e) = updates.send(DataUsageInfo::default()).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "empty_bucket_publish_failed",
error = %e,
"Scanner set state update failed"
);
}
return Ok(());
}
@@ -389,7 +421,15 @@ impl ScannerIO for ECStore {
total_results += pool.disk_set.len();
}
if total_results == 0 {
warn!("nsscanner: no disk sets available for non-empty bucket list");
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket_count = all_buckets.len(),
state = "no_disk_sets",
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(());
}
@@ -397,9 +437,14 @@ impl ScannerIO for ECStore {
let set_scan_limit = scanner_max_concurrent_set_scans(total_results);
record_set_scan_concurrency_limit(set_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
total_sets = total_results,
concurrency_limit = set_scan_limit,
"nsscanner: scanner set concurrency budget"
state = "concurrency_budget",
"Scanner set concurrency budget resolved"
);
let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit));
let queued_set_scans = Arc::new(AtomicUsize::new(total_results));
@@ -495,10 +540,15 @@ impl ScannerIO for ECStore {
)
.increment(1);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = %pool_label,
set = %set_label,
error = %e,
"Failed to scan set; continuing scanner cycle"
state = "set_scan_failed",
"Scanner set scan failed; continuing cycle"
);
let mut first_err = first_err_mutex_clone.lock().await;
record_set_scan_failure(&mut first_err, e);
@@ -540,7 +590,15 @@ impl ScannerIO for ECStore {
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "send_merged_failed",
error = %e,
"Scanner merged data usage publish failed"
);
}
}
break;
@@ -559,7 +617,15 @@ impl ScannerIO for ECStore {
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "send_merged_failed",
error = %e,
"Scanner merged data usage publish failed"
);
}
has_sent_once = true;
last_update = merged_last_update;
@@ -604,18 +670,32 @@ impl ScannerIOCache for SetDisks {
let (disks, healing) = self.get_online_disks_with_healing(false).await;
if disks.is_empty() {
debug!("nsscanner_cache: no online disks available for set");
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "no_online_disks",
"Scanner set state found no online disks"
);
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return Ok(());
}
let disk_scan_limit = scanner_max_concurrent_disk_scans(disks.len());
record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
online_disks = disks.len(),
concurrency_limit = disk_scan_limit,
"nsscanner_cache: scanner disk concurrency budget"
state = "disk_concurrency_budget",
"Scanner disk concurrency budget resolved"
);
let disk_scan_semaphore = Arc::new(Semaphore::new(disk_scan_limit));
let queued_disk_bucket_scans = Arc::new(AtomicUsize::new(buckets.len()));
@@ -646,7 +726,16 @@ impl ScannerIOCache for SetDisks {
if old_cache.find(&bucket.name).is_none()
&& let Err(e) = bucket_tx.send(bucket.clone()).await
{
error!("Failed to send bucket info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_bucket_failed",
error = %e,
"Scanner bucket dispatch failed"
);
}
}
@@ -658,7 +747,16 @@ impl ScannerIOCache for SetDisks {
}
if let Err(e) = bucket_tx.send(bucket.clone()).await {
error!("Failed to send bucket info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_bucket_failed",
error = %e,
"Scanner bucket dispatch failed"
);
}
}
}
@@ -795,13 +893,31 @@ impl ScannerIOCache for SetDisks {
set_label_clone.clone(),
);
debug!("nsscanner_disk: got bucket: {}", bucket.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_started",
"Scanner disk bucket state updated"
);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let mut cache = DataUsageCache::default();
if let Err(e) = cache.load(store_clone_clone.clone(), &cache_name).await {
error!("Failed to load data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket state updated"
);
}
if cache.info.name.is_empty() {
@@ -818,7 +934,16 @@ impl ScannerIOCache for SetDisks {
};
}
debug!("nsscanner_disk: cache.info.name: {:?}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = ?cache.info.name,
state = "cache_ready",
"Scanner disk bucket state updated"
);
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
@@ -841,7 +966,16 @@ impl ScannerIOCache for SetDisks {
})
.await
{
error!("Failed to send data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket_name_clone,
state = "send_failed",
error = %e,
"Scanner data usage stream failed"
);
}
}
});
@@ -855,9 +989,27 @@ impl ScannerIOCache for SetDisks {
Ok(scan_outcome) => scan_outcome,
Err(e) => {
if ctx_clone.is_cancelled() {
debug!("Scanner disk scan stopped after cancellation: {}", e);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "cancelled",
error = %e,
"Scanner disk bucket state updated"
);
} else {
error!("Failed to scan disk: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_failed",
error = %e,
"Scanner disk bucket state updated"
);
}
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
@@ -865,13 +1017,32 @@ impl ScannerIOCache for SetDisks {
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
}
done_save();
}
if let Err(e) = update_fut.await {
error!("Failed to update data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "update_join_failed",
error = %e,
"Scanner bucket update task failed"
);
}
continue;
}
@@ -882,40 +1053,114 @@ impl ScannerIOCache for SetDisks {
ScannerDiskScanOutcome::Partial(cache) => {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save partial data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "partial_save_failed",
error = %e,
"Scanner partial bucket cache save failed"
);
}
done_save();
if let Err(e) = update_fut.await {
error!("Failed to update partial data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "partial_update_join_failed",
error = %e,
"Scanner partial bucket update task failed"
);
}
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("Failed to send partial data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_partial_root_failed",
error = %e,
"Scanner partial root entry publish failed"
);
}
continue;
}
};
debug!("nsscanner_disk: got cache: {}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "scan_completed",
"Scanner disk bucket state updated"
);
if let Err(e) = update_fut.await {
error!("nsscanner_disk: Failed to update data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "update_join_failed",
error = %e,
"Scanner bucket update task failed"
);
}
if ctx_clone.is_cancelled() {
break;
}
debug!("nsscanner_disk: sending data usage entry info: {}", cache.info.name);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "send_root_entry",
"Scanner data usage stream progress updated"
);
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_root_failed",
error = %e,
"Scanner root entry publish failed"
);
}
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
}
done_save();
}
@@ -931,7 +1176,14 @@ impl ScannerIOCache for SetDisks {
send_update_fut.await?;
debug!("nsscanner_cache: done");
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed",
"Scanner set-level disk scan completed"
);
Ok(())
}
@@ -966,7 +1218,17 @@ impl ScannerIODisk for Disk {
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions,
Err(e) => {
error!("Failed to get file info versions: {}/{}, err: {e}", item.bucket, item.object_path());
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %item.bucket,
object = %item.object_path(),
state = "file_info_versions_failed",
error = %e,
"Scanner disk bucket failed to resolve file info versions"
);
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
};
@@ -1072,28 +1334,60 @@ impl ScannerIODisk for Disk {
// TODO: object lock
let Some(ecstore) = new_object_layer_fn() else {
error!("ECStore not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "ecstore_unavailable",
"Scanner disk bucket missing object layer"
);
return Err(StorageError::other("ECStore not available".to_string()));
};
let disk_location = self.get_disk_location();
let (Some(pool_idx), Some(set_idx)) = (disk_location.pool_idx, disk_location.set_idx) else {
error!("Disk location not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "disk_location_unavailable",
"Scanner disk bucket missing disk location"
);
return Err(StorageError::other("Disk location not available".to_string()));
};
let disks_result = StorageAdminApi::disk_set_inventory(ecstore.as_ref(), DiskSetSelector::new(pool_idx, set_idx)).await?;
let Some(disk_idx) = disk_location.disk_idx else {
error!("Disk index not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "disk_index_unavailable",
"Scanner disk bucket missing disk index"
);
return Err(StorageError::other("Disk index not available".to_string()));
};
let local_disk = if let Some(Some(local_disk)) = disks_result.get(disk_idx) {
local_disk.clone()
} else {
error!("Local disk not available");
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket,
state = "local_disk_unavailable",
"Scanner disk bucket missing local disk"
);
return Err(StorageError::other("Local disk not available".to_string()));
};