feat(obs): complete metric dimension coverage (#6314)

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-20 22:43:40 +08:00
committed by GitHub
parent d65fba9142
commit 830e553a3c
15 changed files with 821 additions and 32 deletions
+36 -2
View File
@@ -147,11 +147,19 @@ impl Drop for DiskBucketScanActiveGuard {
pub(super) struct BucketDriveFailureGuard {
failed: bool,
source: rustfs_common::metrics::ScannerWorkSource,
bucket: String,
drive: String,
}
impl BucketDriveFailureGuard {
pub(super) fn new() -> Self {
Self { failed: true }
pub(super) fn new(source: rustfs_common::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
Self {
failed: true,
source,
bucket: bucket.to_string(),
drive: drive.to_string(),
}
}
pub(super) fn mark_not_failed(&mut self) {
@@ -161,6 +169,7 @@ impl BucketDriveFailureGuard {
impl Drop for BucketDriveFailureGuard {
fn drop(&mut self) {
global_metrics().record_scan_bucket_drive_end(self.source, &self.bucket, &self.drive);
if self.failed {
global_metrics().record_scan_bucket_drive_failure();
}
@@ -272,3 +281,28 @@ pub(super) fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error)
pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
Error::other(format!("{stage} task join failed: {err}"))
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_common::metrics::{ScannerWorkSource, global_metrics};
#[test]
fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
let source = ScannerWorkSource::Usage;
let bucket = "__guard_active_lifecycle_test__";
let drive = "/__guard_active_lifecycle_test__";
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
{
let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
guard.mark_not_failed();
}
assert!(
!global_metrics()
.scanner_runtime_details_report()
.active_bucket_drive_scans
.iter()
.any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
);
}
}
+11 -7
View File
@@ -147,8 +147,12 @@ impl ScannerIODisk for Disk {
let drive_start = std::time::Instant::now();
let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string();
global_metrics().record_scan_bucket_drive_start();
let mut failure_guard = BucketDriveFailureGuard::new();
let source = match scan_mode {
HealScanMode::Deep => rustfs_common::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_common::metrics::ScannerWorkSource::Usage,
};
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
let _guard = self.start_scan();
let mut cache = cache;
@@ -196,32 +200,32 @@ impl ScannerIODisk for Disk {
match result {
Ok(mut data_usage_info) => {
done_drive();
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
data_usage_info.info.last_update = Some(SystemTime::now());
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
}
Err(ScannerError::PartialCache(mut partial_cache)) => {
done_drive();
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
}
Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
done_drive();
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
}
Err(e) => {
if ctx.is_cancelled() {
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
failure_guard.mark_not_failed();
} else {
done_drive();
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
emit_scan_bucket_drive_complete(source, false, &bucket, &disk_path, drive_start.elapsed());
}
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
}