fix(metrics): remove duplicate scanner counter producers (#6245)

* fix(metrics): remove duplicate scanner counter producers

* chore(deps): centralize metrics test dependencies

* ci: avoid apt mirror for ripgrep setup

* test(protos): track read-version encoder refactor

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

.github/workflows/architecture-migration-rules.yml

@@ -53,9 +53,9 @@ jobs:
           persist-credentials: false
 
       - name: Install ripgrep
-        run: |
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-          sudo apt-get update
+        with:
-          sudo apt-get install -y ripgrep
+          tool: ripgrep@15.2.0
 
       - name: Check architecture migration rules
         run: ./scripts/check_architecture_migration_rules.sh

.github/workflows/ci-docs-only.yml

@@ -83,7 +83,9 @@ jobs:
           persist-credentials: false
 
       - name: Install ripgrep
-        run: sudo apt-get update && sudo apt-get install -y ripgrep
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: ripgrep@15.2.0
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

.github/workflows/ci.yml

@@ -118,7 +118,9 @@ jobs:
           persist-credentials: false
 
       - name: Install ripgrep
-        run: sudo apt-get update && sudo apt-get install -y ripgrep
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: ripgrep@15.2.0
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

Cargo.lock

@@ -9933,6 +9933,7 @@ dependencies = [
  "libc",
  "log",
  "metrics",
+ "metrics-util",
  "num_cpus",
  "nvml-wrapper",
  "opentelemetry",

Cargo.toml

@@ -264,6 +264,7 @@ lazy_static = "1.5.0"
 libc = "0.2.189"
 libsystemd = "0.7.2"
 local-ip-address = "0.6.13"
+log = "0.4"
 memmap2 = "0.9.11"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -326,6 +327,7 @@ zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
+metrics-util = "0.20"
 dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.32.0" }
 opentelemetry-appender-tracing = { version = "0.32.0" }

crates/common/src/metrics.rs

@@ -1401,25 +1401,11 @@ impl CurrentCycle {
 }
 
 /// OTEL metric name constants for scanner metrics
-const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
-const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
 const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
 
-fn emit_otel_counter(metric: usize, count: u64) {
-    match Metric::from_index(metric) {
-        Some(Metric::ScanObject) => {
-            metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
-        }
-        Some(Metric::ScanFolder) => {
-            metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
-        }
-        _ => {}
-    }
-}
-
 fn scan_cycle_result_label(result: u8) -> &'static str {
     match result {
         SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
@@ -1960,7 +1946,6 @@ impl Metrics {
             let duration = SystemTime::now().duration_since(start).unwrap_or_default();
             global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
             global_metrics().record_source_work_for_metric(metric, 1);
-            emit_otel_counter(metric_idx, 1);
             if metric_idx < Metric::LastRealtime as usize {
                 global_metrics().latency[metric_idx].add(duration);
             }
@@ -1976,7 +1961,6 @@ impl Metrics {
             let duration = SystemTime::now().duration_since(start).unwrap_or_default();
             global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
             global_metrics().record_source_work_for_metric(metric, 1);
-            emit_otel_counter(metric_idx, 1);
             if metric_idx < Metric::LastRealtime as usize {
                 global_metrics().latency[metric_idx].add_size(duration, size);
             }
@@ -1992,7 +1976,6 @@ impl Metrics {
             let duration = SystemTime::now().duration_since(start).unwrap_or_default();
             global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
             global_metrics().record_source_work_for_metric(metric, 1);
-            emit_otel_counter(metric_idx, 1);
             if metric_idx < Metric::LastRealtime as usize {
                 global_metrics().latency[metric_idx].add(duration);
             }
@@ -2010,7 +1993,6 @@ impl Metrics {
                 let count = usize_to_u64_saturated(count);
                 global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
                 global_metrics().record_source_work_for_metric(metric, count);
-                emit_otel_counter(metric_idx, count);
                 if metric_idx < Metric::LastRealtime as usize {
                     global_metrics().latency[metric_idx].add(duration);
                 }
@@ -2031,7 +2013,6 @@ impl Metrics {
                 let duration = SystemTime::now().duration_since(start).unwrap_or_default();
                 let metric_idx = Metric::Ilm as usize;
                 global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
-                emit_otel_counter(metric_idx, versions);
                 global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
                 global_metrics().actions_latency[a_idx].add(duration);
             })
@@ -2044,7 +2025,6 @@ impl Metrics {
         let metric_idx = metric as usize;
         global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
         global_metrics().record_source_work_for_metric(metric, 1);
-        emit_otel_counter(metric_idx, 1);
         if metric_idx < Metric::LastRealtime as usize {
             global_metrics().latency[metric_idx].add(duration);
         }

crates/io-metrics/Cargo.toml

@@ -58,7 +58,7 @@ sysinfo = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["html_reports"] }
-metrics-util = { version = "0.20", features = ["debugging"] }
+metrics-util = { workspace = true, features = ["debugging"] }
 tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
 
 [lints]

crates/kms/Cargo.toml

@@ -94,7 +94,7 @@ aws-smithy-types = { workspace = true }
 [dev-dependencies]
 anyhow = { workspace = true }
 # Debugging recorder for asserting emitted metrics in tests.
-metrics-util = { version = "0.20", features = ["debugging"] }
+metrics-util = { workspace = true, features = ["debugging"] }
 insta = { workspace = true, features = ["yaml", "json"] }
 tempfile = { workspace = true }
 temp-env = { workspace = true }

crates/lifecycle/Cargo.toml

@@ -67,7 +67,7 @@ url.workspace = true
 uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
 
 [dev-dependencies]
-metrics-util = { version = "0.20", features = ["debugging"] }
+metrics-util = { workspace = true, features = ["debugging"] }
 proptest = "1"
 serial_test.workspace = true
 temp-env.workspace = true

crates/object-data-cache/Cargo.toml

@@ -58,7 +58,7 @@ tracing = { workspace = true, optional = true }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["html_reports"] }
-metrics-util = { version = "0.20", features = ["debugging"] }
+metrics-util = { workspace = true, features = ["debugging"] }
 # `rt-multi-thread` lets the concurrency stress tests run tasks on real worker
 # threads, so they exercise true parallelism on the shared singleflight/index
 # state rather than only cooperative interleaving.

crates/obs/Cargo.toml

@@ -163,4 +163,5 @@ libc = { workspace = true }
 [dev-dependencies]
 tempfile = { workspace = true }
 temp-env = { workspace = true }
-log = "0.4"
+log.workspace = true
+metrics-util = { workspace = true, features = ["debugging"] }

crates/obs/src/metrics/collectors/scanner.rs

@@ -465,6 +465,57 @@ fn bool_metric_value(enabled: bool) -> f64 {
 mod tests {
     use super::*;
     use crate::metrics::report::report_metrics;
+    use metrics_util::debugging::DebuggingRecorder;
+    use rustfs_common::metrics::{Metric, Metrics};
+
+    fn prometheus_counter_name(name: &str) -> String {
+        if name.ends_with("_total") {
+            name.to_string()
+        } else {
+            format!("{name}_total")
+        }
+    }
+
+    #[test]
+    fn scanner_lifetime_counters_have_one_prometheus_producer() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let scanner_metrics = collect_scanner_metrics(&ScannerStats {
+            directories_scanned: 3,
+            objects_scanned: 7,
+            ..Default::default()
+        });
+
+        metrics::with_local_recorder(&recorder, || {
+            Metrics::time(Metric::ScanObject)();
+            Metrics::time(Metric::ScanFolder)();
+            report_metrics(&scanner_metrics);
+        });
+
+        let normalized_counter_names: Vec<_> = snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .filter_map(|(composite, _, _, value)| {
+                matches!(value, metrics_util::debugging::DebugValue::Counter(_))
+                    .then(|| prometheus_counter_name(composite.key().name()))
+            })
+            .collect();
+
+        for name in [
+            "rustfs_scanner_objects_scanned_total",
+            "rustfs_scanner_directories_scanned_total",
+        ] {
+            assert_eq!(
+                normalized_counter_names
+                    .iter()
+                    .filter(|candidate| candidate.as_str() == name)
+                    .count(),
+                1,
+                "scanner lifetime counter must have exactly one producer after Prometheus name normalization"
+            );
+        }
+    }
 
     #[test]
     fn test_collect_scanner_metrics() {

crates/protos/src/lib.rs

@@ -2433,7 +2433,7 @@ mod tests {
                 json_field: "opts",
                 bin_field: "opts_bin",
             },
-            json_encoder: "let opts_str = compat_json(opts)?;",
+            json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));",
             policy: RequestJsonPolicy::MsgpackOnlyEligible,
         },
         RequestCompatSendSite {

rustfs/Cargo.toml

@@ -350,7 +350,7 @@ proptest = "1"
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
 temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
-metrics-util = { version = "0.20", features = ["debugging"] }
+metrics-util = { workspace = true, features = ["debugging"] }
 opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 rsa = { workspace = true }
 rcgen = { workspace = true }