feat(obs): add bounded metrics dimensions (#5645)

* feat(obs): add drive topology detail metrics Expose additive drive info, topology, state, and per-drive API metrics while preserving the existing drive metric label sets. Backlog: rustfs/backlog#1655 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): preserve suspect drive runtime state Keep suspect as a bounded drive runtime state and avoid all-zero runtime_state samples for that storage health state. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): skip unknown drive inode samples Avoid exporting zero inode gauges for missing or stale drive snapshots and ignore zero-count API latency buckets. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add scanner source work detail metrics Expose additive scanner source and cycle work metrics with bounded server/source/state labels while leaving the existing aggregate scanner metrics unchanged. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add ilm action detail metrics Expose additive ILM action/state task metrics with a server label while preserving the existing aggregate ILM series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add delivery target server metrics Expose additive audit and notification delivery target metrics with server labels and extend removed-target tombstones for the server-aware series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add replication target flow metrics Expose additive bucket replication target sent and failed-flow metrics while preserving existing bucket aggregates and target backlog series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add request server metrics Expose additive API request metrics with server labels while preserving the existing request and traffic metric label sets. Co-Authored-By: heihutu <heihutu@gmail.com> * style(obs): apply rustfmt to metrics changes Apply rustfmt output to the metrics dimension changes without altering behavior. Co-Authored-By: heihutu <heihutu@gmail.com> * style(obs): reuse audit target label constant Use the exported audit target_id label constant for legacy audit target metrics. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): populate drive disk metrics Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add scanner bucket drive result metrics Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add replication proxy server metrics Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metric liveness review Use checked division for drive API latency aggregation and keep recovered drive, scanner current-cycle, replication flow, audit target, and notification target series from retaining stale values. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metric dimension review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address additional metric review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): count drive calls at start Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metrics dimension review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address dimension review gaps Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address scanner review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address runtime review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): reduce disk metric contention Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address runtime review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): retire stale dimension series Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-09-01 09:48:20 +00:00 · 2026-08-03 09:03:34 +08:00
parent 988cd8adbb
commit 035ce5d784
36 changed files with 6282 additions and 666 deletions
@@ -246,7 +246,7 @@ pub use process_lock_metrics::{
    record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts, snapshot_process_lock_events,
    snapshot_process_platform_stats,
 };
-pub use s3_api_metrics::{init_s3_metrics, record_s3_op};
+pub use s3_api_metrics::{S3OperationMetricSnapshot, init_s3_metrics, record_s3_op, s3_op_metrics_snapshot};
 pub use sampler::{
    ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform,
    snapshot_process_resource, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
@@ -14,8 +14,25 @@

 use rustfs_s3_ops::S3Operation;
 use std::sync::OnceLock;
+use std::sync::atomic::{AtomicU64, Ordering};

 const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
+static S3_OP_COUNTERS: OnceLock<Box<[AtomicU64]>> = OnceLock::new();
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct S3OperationMetricSnapshot {
+    pub op: &'static str,
+    pub total: u64,
+}
+
+fn s3_op_counters() -> &'static [AtomicU64] {
+    S3_OP_COUNTERS.get_or_init(|| {
+        std::iter::repeat_with(|| AtomicU64::new(0))
+            .take(S3Operation::ALL.len())
+            .collect::<Vec<_>>()
+            .into_boxed_slice()
+    })
+}

 /// Record a handled S3 API operation.
 ///
@@ -26,9 +43,22 @@ const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
 /// This mirrors MinIO, which never labels its default operation counters with
 /// bucket. The `op` dimension is bounded (<= 122 variants).
 pub fn record_s3_op(op: S3Operation) {
+    if let Some(counter) = s3_op_counters().get(op.metric_index()) {
+        counter.fetch_add(1, Ordering::Relaxed);
+    }
    counter!(S3_OPS_METRIC, "op" => op.as_str()).increment(1);
 }

+pub fn s3_op_metrics_snapshot() -> Vec<S3OperationMetricSnapshot> {
+    S3Operation::ALL
+        .iter()
+        .filter_map(|op| {
+            let total = s3_op_counters().get(op.metric_index())?.load(Ordering::Relaxed);
+            (total > 0).then_some(S3OperationMetricSnapshot { op: op.as_str(), total })
+        })
+        .collect()
+}
+
 pub fn init_s3_metrics() {
    static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
    METRICS_DESC_INIT.get_or_init(|| {
@@ -42,6 +72,9 @@ mod tests {
    use metrics::with_local_recorder;
    use metrics_util::debugging::DebuggingRecorder;
    use std::collections::HashSet;
+    use std::sync::Mutex;
+
+    static S3_OP_TEST_LOCK: Mutex<()> = Mutex::new(());

    /// Collect the label-key sets recorded against `rustfs_s3_operations_total`.
    fn ops_metric_label_key_sets(recorder: &DebuggingRecorder) -> Vec<HashSet<String>> {
@@ -60,6 +93,7 @@ mod tests {

    #[test]
    fn record_s3_op_labels_by_op_only_no_bucket() {
+        let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
        let recorder = DebuggingRecorder::new();
        let label_key_sets = ops_metric_label_key_sets(&recorder);

@@ -75,6 +109,7 @@ mod tests {

    #[test]
    fn record_s3_op_cardinality_bounded_by_distinct_ops() {
+        let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();

@@ -113,4 +148,24 @@ mod tests {
            "series count must equal the number of distinct ops, never the bucket count"
        );
    }
+
+    #[test]
+    fn s3_op_metrics_snapshot_reports_recorded_totals() {
+        let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        let before = s3_op_metrics_snapshot()
+            .into_iter()
+            .find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
+            .map(|snapshot| snapshot.total)
+            .unwrap_or_default();
+
+        record_s3_op(S3Operation::GetObject);
+        record_s3_op(S3Operation::GetObject);
+
+        let after = s3_op_metrics_snapshot()
+            .into_iter()
+            .find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
+            .map(|snapshot| snapshot.total)
+            .expect("GetObject snapshot should be present after recording");
+        assert_eq!(after, before + 2);
+    }
 }