gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 03:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(metrics): remove duplicate scanner counter producers (#6245)

Browse Source 
* fix(metrics): remove duplicate scanner counter producers

* chore(deps): centralize metrics test dependencies

* ci: avoid apt mirror for ripgrep setup

* test(protos): track read-version encoder refactor

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-08-19 21:58:22 +08:00
committed by GitHub
parent 9fed675185
commit 24cfce12ed
14 changed files with 71 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/metrics.rs
-20
View File
@@ -1401,25 +1401,11 @@ impl CurrentCycle {
}
/// OTEL metric name constants for scanner metrics
const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
fn emit_otel_counter(metric: usize, count: u64) {
match Metric::from_index(metric) {
Some(Metric::ScanObject) => {
metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
}
Some(Metric::ScanFolder) => {
metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
}
_ => {}
}
}
fn scan_cycle_result_label(result: u8) -> &'static str {
match result {
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
@@ -1960,7 +1946,6 @@ impl Metrics {
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
global_metrics().record_source_work_for_metric(metric, 1);
emit_otel_counter(metric_idx, 1);
if metric_idx < Metric::LastRealtime as usize {
global_metrics().latency[metric_idx].add(duration);
}
@@ -1976,7 +1961,6 @@ impl Metrics {
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
global_metrics().record_source_work_for_metric(metric, 1);
emit_otel_counter(metric_idx, 1);
if metric_idx < Metric::LastRealtime as usize {
global_metrics().latency[metric_idx].add_size(duration, size);
}
@@ -1992,7 +1976,6 @@ impl Metrics {
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
global_metrics().record_source_work_for_metric(metric, 1);
emit_otel_counter(metric_idx, 1);
if metric_idx < Metric::LastRealtime as usize {
global_metrics().latency[metric_idx].add(duration);
}
@@ -2010,7 +1993,6 @@ impl Metrics {
let count = usize_to_u64_saturated(count);
global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
global_metrics().record_source_work_for_metric(metric, count);
emit_otel_counter(metric_idx, count);
if metric_idx < Metric::LastRealtime as usize {
global_metrics().latency[metric_idx].add(duration);
}
@@ -2031,7 +2013,6 @@ impl Metrics {
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
let metric_idx = Metric::Ilm as usize;
global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
emit_otel_counter(metric_idx, versions);
global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
global_metrics().actions_latency[a_idx].add(duration);
})
@@ -2044,7 +2025,6 @@ impl Metrics {
let metric_idx = metric as usize;
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
global_metrics().record_source_work_for_metric(metric, 1);
emit_otel_counter(metric_idx, 1);
if metric_idx < Metric::LastRealtime as usize {
global_metrics().latency[metric_idx].add(duration);
}
crates/io-metrics/Cargo.toml
+1 -1
View File
@@ -58,7 +58,7 @@ sysinfo = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
metrics-util = { version = "0.20", features = ["debugging"] }
metrics-util = { workspace = true, features = ["debugging"] }
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
[lints]
crates/kms/Cargo.toml
+1 -1
View File
@@ -94,7 +94,7 @@ aws-smithy-types = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
# Debugging recorder for asserting emitted metrics in tests.
metrics-util = { version = "0.20", features = ["debugging"] }
metrics-util = { workspace = true, features = ["debugging"] }
insta = { workspace = true, features = ["yaml", "json"] }
tempfile = { workspace = true }
temp-env = { workspace = true }
crates/lifecycle/Cargo.toml
+1 -1
View File
@@ -67,7 +67,7 @@ url.workspace = true
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
[dev-dependencies]
metrics-util = { version = "0.20", features = ["debugging"] }
metrics-util = { workspace = true, features = ["debugging"] }
proptest = "1"
serial_test.workspace = true
temp-env.workspace = true
crates/object-data-cache/Cargo.toml
+1 -1
View File
@@ -58,7 +58,7 @@ tracing = { workspace = true, optional = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
metrics-util = { version = "0.20", features = ["debugging"] }
metrics-util = { workspace = true, features = ["debugging"] }
# `rt-multi-thread` lets the concurrency stress tests run tasks on real worker
# threads, so they exercise true parallelism on the shared singleflight/index
# state rather than only cooperative interleaving.
crates/obs/Cargo.toml
+2 -1
View File
@@ -163,4 +163,5 @@ libc = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
temp-env = { workspace = true }
log = "0.4"
log.workspace = true
metrics-util = { workspace = true, features = ["debugging"] }
crates/obs/src/metrics/collectors/scanner.rs
+51
View File
@@ -465,6 +465,57 @@ fn bool_metric_value(enabled: bool) -> f64 {
mod tests {
use super::*;
use crate::metrics::report::report_metrics;
use metrics_util::debugging::DebuggingRecorder;
use rustfs_common::metrics::{Metric, Metrics};
fn prometheus_counter_name(name: &str) -> String {
if name.ends_with("_total") {
name.to_string()
} else {
format!("{name}_total")
}
}
#[test]
fn scanner_lifetime_counters_have_one_prometheus_producer() {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let scanner_metrics = collect_scanner_metrics(&ScannerStats {
directories_scanned: 3,
objects_scanned: 7,
..Default::default()
});
metrics::with_local_recorder(&recorder, || {
Metrics::time(Metric::ScanObject)();
Metrics::time(Metric::ScanFolder)();
report_metrics(&scanner_metrics);
});
let normalized_counter_names: Vec<_> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter_map(|(composite, _, _, value)| {
matches!(value, metrics_util::debugging::DebugValue::Counter(_))
.then(|| prometheus_counter_name(composite.key().name()))
})
.collect();
for name in [
"rustfs_scanner_objects_scanned_total",
"rustfs_scanner_directories_scanned_total",
] {
assert_eq!(
normalized_counter_names
.iter()
.filter(|candidate| candidate.as_str() == name)
.count(),
1,
"scanner lifetime counter must have exactly one producer after Prometheus name normalization"
);
}
}
#[test]
fn test_collect_scanner_metrics() {
crates/protos/src/lib.rs
+1 -1
View File
@@ -2433,7 +2433,7 @@ mod tests {
json_field: "opts",
bin_field: "opts_bin",
},
json_encoder: "let opts_str = compat_json(opts)?;",
json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));",
policy: RequestJsonPolicy::MsgpackOnlyEligible,
},
RequestCompatSendSite {