mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
feat(obs): complete metric dimension coverage (#6314)
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -147,11 +147,19 @@ impl Drop for DiskBucketScanActiveGuard {
|
||||
|
||||
pub(super) struct BucketDriveFailureGuard {
|
||||
failed: bool,
|
||||
source: rustfs_common::metrics::ScannerWorkSource,
|
||||
bucket: String,
|
||||
drive: String,
|
||||
}
|
||||
|
||||
impl BucketDriveFailureGuard {
|
||||
pub(super) fn new() -> Self {
|
||||
Self { failed: true }
|
||||
pub(super) fn new(source: rustfs_common::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
|
||||
Self {
|
||||
failed: true,
|
||||
source,
|
||||
bucket: bucket.to_string(),
|
||||
drive: drive.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mark_not_failed(&mut self) {
|
||||
@@ -161,6 +169,7 @@ impl BucketDriveFailureGuard {
|
||||
|
||||
impl Drop for BucketDriveFailureGuard {
|
||||
fn drop(&mut self) {
|
||||
global_metrics().record_scan_bucket_drive_end(self.source, &self.bucket, &self.drive);
|
||||
if self.failed {
|
||||
global_metrics().record_scan_bucket_drive_failure();
|
||||
}
|
||||
@@ -272,3 +281,28 @@ pub(super) fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error)
|
||||
pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
|
||||
Error::other(format!("{stage} task join failed: {err}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::metrics::{ScannerWorkSource, global_metrics};
|
||||
|
||||
#[test]
|
||||
fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
|
||||
let source = ScannerWorkSource::Usage;
|
||||
let bucket = "__guard_active_lifecycle_test__";
|
||||
let drive = "/__guard_active_lifecycle_test__";
|
||||
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
|
||||
{
|
||||
let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
|
||||
guard.mark_not_failed();
|
||||
}
|
||||
assert!(
|
||||
!global_metrics()
|
||||
.scanner_runtime_details_report()
|
||||
.active_bucket_drive_scans
|
||||
.iter()
|
||||
.any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,8 +147,12 @@ impl ScannerIODisk for Disk {
|
||||
let drive_start = std::time::Instant::now();
|
||||
let bucket = cache.info.name.clone();
|
||||
let disk_path = self.path().to_string_lossy().to_string();
|
||||
global_metrics().record_scan_bucket_drive_start();
|
||||
let mut failure_guard = BucketDriveFailureGuard::new();
|
||||
let source = match scan_mode {
|
||||
HealScanMode::Deep => rustfs_common::metrics::ScannerWorkSource::Bitrot,
|
||||
HealScanMode::Normal | HealScanMode::Unknown => rustfs_common::metrics::ScannerWorkSource::Usage,
|
||||
};
|
||||
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
|
||||
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
|
||||
let _guard = self.start_scan();
|
||||
|
||||
let mut cache = cache;
|
||||
@@ -196,32 +200,32 @@ impl ScannerIODisk for Disk {
|
||||
match result {
|
||||
Ok(mut data_usage_info) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
|
||||
emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
|
||||
data_usage_info.info.last_update = Some(SystemTime::now());
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
|
||||
}
|
||||
Err(ScannerError::PartialCache(mut partial_cache)) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
|
||||
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
|
||||
}
|
||||
Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
|
||||
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
failure_guard.mark_not_failed();
|
||||
Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
|
||||
}
|
||||
Err(e) => {
|
||||
if ctx.is_cancelled() {
|
||||
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
|
||||
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
|
||||
failure_guard.mark_not_failed();
|
||||
} else {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
|
||||
emit_scan_bucket_drive_complete(source, false, &bucket, &disk_path, drive_start.elapsed());
|
||||
}
|
||||
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user