fix(io-metrics): drop client-controlled bucket label from s3 ops counter (backlog#806) (#4580)

fix(io-metrics): drop client-controlled bucket label from rustfs_s3_operations_total (backlog#806-22)

The bucket dimension on the rustfs_s3_operations_total counter is
client-controlled and unbounded, letting any client explode metric
cardinality (a Prometheus/OTEL cardinality DoS that grows the in-process
OTEL aggregation store even without a scrape). Drop the bucket label so the
series is keyed by the bounded op dimension only (<=122 variants), matching
MinIO which never labels its default operation counters with bucket.

BREAKING: rustfs_s3_operations_total no longer carries a "bucket" label.

record_s3_op(op, bucket) -> record_s3_op(op); all call sites in
rustfs/src/storage/{ecfs.rs,helper.rs} updated (bucket bindings retained
where still used by audit/notify paths). Add metrics-util debugging recorder
dev-dep and tests asserting the series carries only the op label and that
series count equals distinct-op count.

crates/io-metrics/Cargo.toml

@@ -39,6 +39,7 @@ sysinfo = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }
+metrics-util = { version = "0.20", features = ["debugging"] }
 tokio = { workspace = true, features = ["test-util","rt","macros"] }
 
 [lints]

crates/io-metrics/src/s3_api_metrics.rs

@@ -17,8 +17,16 @@ use std::sync::OnceLock;
 
 const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
 
-pub fn record_s3_op(op: S3Operation, bucket: &str) {
-    counter!(S3_OPS_METRIC, "op" => op.as_str(), "bucket" => bucket.to_owned()).increment(1);
+/// Record a handled S3 API operation.
+///
+/// The series is labeled by `op` only. The bucket name is deliberately NOT a
+/// label: it is client-controlled and unbounded, so labeling by bucket would
+/// let any client explode metric cardinality (a Prometheus/OTEL cardinality
+/// DoS that grows the in-process OTEL aggregation store even without a scrape).
+/// This mirrors MinIO, which never labels its default operation counters with
+/// bucket. The `op` dimension is bounded (<= 122 variants).
+pub fn record_s3_op(op: S3Operation) {
+    counter!(S3_OPS_METRIC, "op" => op.as_str()).increment(1);
 }
 
 pub fn init_s3_metrics() {
@@ -27,3 +35,82 @@ pub fn init_s3_metrics() {
         describe_counter!(S3_OPS_METRIC, "Total number of S3 API operations handled");
     });
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use metrics::with_local_recorder;
+    use metrics_util::debugging::DebuggingRecorder;
+    use std::collections::HashSet;
+
+    /// Collect the label-key sets recorded against `rustfs_s3_operations_total`.
+    fn ops_metric_label_key_sets(recorder: &DebuggingRecorder) -> Vec<HashSet<String>> {
+        let snapshotter = recorder.snapshotter();
+        with_local_recorder(recorder, || {
+            record_s3_op(S3Operation::GetObject);
+        });
+        snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .filter(|(composite, _, _, _)| composite.key().name() == S3_OPS_METRIC)
+            .map(|(composite, _, _, _)| composite.key().labels().map(|label| label.key().to_string()).collect())
+            .collect()
+    }
+
+    #[test]
+    fn record_s3_op_labels_by_op_only_no_bucket() {
+        let recorder = DebuggingRecorder::new();
+        let label_key_sets = ops_metric_label_key_sets(&recorder);
+
+        assert_eq!(label_key_sets.len(), 1, "exactly one series expected for a single op");
+        let keys = &label_key_sets[0];
+        assert_eq!(
+            keys,
+            &HashSet::from(["op".to_string()]),
+            "series must carry the op label only; the client-controlled bucket label must be absent"
+        );
+        assert!(!keys.contains("bucket"), "bucket label must never be emitted (cardinality DoS)");
+    }
+
+    #[test]
+    fn record_s3_op_cardinality_bounded_by_distinct_ops() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+
+        // Same op recorded twice + two other ops => distinct series == distinct ops.
+        let ops = [
+            S3Operation::GetObject,
+            S3Operation::GetObject,
+            S3Operation::PutObject,
+            S3Operation::ListObjectsV2,
+        ];
+        with_local_recorder(&recorder, || {
+            for op in ops {
+                record_s3_op(op);
+            }
+        });
+
+        let series: HashSet<String> = snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .filter(|(composite, _, _, _)| composite.key().name() == S3_OPS_METRIC)
+            .map(|(composite, _, _, _)| {
+                composite
+                    .key()
+                    .labels()
+                    .map(|label| format!("{}={}", label.key(), label.value()))
+                    .collect::<Vec<_>>()
+                    .join(",")
+            })
+            .collect();
+
+        let distinct_ops: HashSet<&str> = ops.iter().map(|op| op.as_str()).collect();
+        assert_eq!(
+            series.len(),
+            distinct_ops.len(),
+            "series count must equal the number of distinct ops, never the bucket count"
+        );
+    }
+}