feat(obs): add bounded metrics dimensions (#5645)

* feat(obs): add drive topology detail metrics Expose additive drive info, topology, state, and per-drive API metrics while preserving the existing drive metric label sets. Backlog: rustfs/backlog#1655 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): preserve suspect drive runtime state Keep suspect as a bounded drive runtime state and avoid all-zero runtime_state samples for that storage health state. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): skip unknown drive inode samples Avoid exporting zero inode gauges for missing or stale drive snapshots and ignore zero-count API latency buckets. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add scanner source work detail metrics Expose additive scanner source and cycle work metrics with bounded server/source/state labels while leaving the existing aggregate scanner metrics unchanged. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add ilm action detail metrics Expose additive ILM action/state task metrics with a server label while preserving the existing aggregate ILM series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add delivery target server metrics Expose additive audit and notification delivery target metrics with server labels and extend removed-target tombstones for the server-aware series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add replication target flow metrics Expose additive bucket replication target sent and failed-flow metrics while preserving existing bucket aggregates and target backlog series. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add request server metrics Expose additive API request metrics with server labels while preserving the existing request and traffic metric label sets. Co-Authored-By: heihutu <heihutu@gmail.com> * style(obs): apply rustfmt to metrics changes Apply rustfmt output to the metrics dimension changes without altering behavior. Co-Authored-By: heihutu <heihutu@gmail.com> * style(obs): reuse audit target label constant Use the exported audit target_id label constant for legacy audit target metrics. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): populate drive disk metrics Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add scanner bucket drive result metrics Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): add replication proxy server metrics Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metric liveness review Use checked division for drive API latency aggregation and keep recovered drive, scanner current-cycle, replication flow, audit target, and notification target series from retaining stale values. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metric dimension review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address additional metric review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): count drive calls at start Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): address metrics dimension review Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address dimension review gaps Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address scanner review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address runtime review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): reduce disk metric contention Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): address runtime review follow-ups Co-Authored-By: heihutu <heihutu@gmail.com> * fix(metrics): retire stale dimension series Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-09-04 11:15:39 +00:00 · 2026-08-03 09:03:34 +08:00
parent 988cd8adbb
commit 035ce5d784
36 changed files with 6282 additions and 666 deletions
@@ -21,6 +21,8 @@ use std::sync::LazyLock;
 pub const BUCKET_L: &str = "bucket";
 /// Replication operation
 pub const OPERATION_L: &str = "operation";
+/// Replication proxy result
+pub const RESULT_L: &str = "result";
 /// Replication target ARN
 pub const TARGET_ARN_L: &str = "target_arn";
 /// Replication range
@@ -48,6 +50,16 @@ const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
 const MRF_MISSED_COUNT: &str = "mrf_missed_count";
 const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures";
 const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis";
+const TARGET_SENT_BYTES: &str = "target_sent_bytes";
+const TARGET_SENT_COUNT: &str = "target_sent_count";
+const TARGET_TOTAL_FAILED_BYTES: &str = "target_total_failed_bytes";
+const TARGET_TOTAL_FAILED_COUNT: &str = "target_total_failed_count";
+const TARGET_LAST_MIN_FAILED_BYTES: &str = "target_last_min_failed_bytes";
+const TARGET_LAST_MIN_FAILED_COUNT: &str = "target_last_min_failed_count";
+const TARGET_LAST_HOUR_FAILED_BYTES: &str = "target_last_hour_failed_bytes";
+const TARGET_LAST_HOUR_FAILED_COUNT: &str = "target_last_hour_failed_count";
+
+const BUCKET_TARGET_LABELS: [&str; 2] = [BUCKET_L, TARGET_ARN_L];

 pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
@@ -58,6 +70,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
    )
 });

+pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(TARGET_LAST_HOUR_FAILED_BYTES),
+        "Total number of bytes failed at least once to replicate in the last hour on a bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::LastHourFailedCount,
@@ -67,6 +88,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
    )
 });

+pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(TARGET_LAST_HOUR_FAILED_COUNT),
+        "Total number of objects which failed replication in the last hour on a bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::LastMinFailedBytes,
@@ -76,6 +106,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = La
    )
 });

+pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(TARGET_LAST_MIN_FAILED_BYTES),
+        "Total number of bytes failed at least once to replicate in the last full minute on a bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::LastMinFailedCount,
@@ -85,6 +124,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = La
    )
 });

+pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(TARGET_LAST_MIN_FAILED_COUNT),
+        "Total number of objects which failed replication in the last full minute on a bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::LatencyMilliSec,
@@ -337,6 +385,15 @@ pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
    )
 });

+pub static BUCKET_REPL_TARGET_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(TARGET_SENT_BYTES),
+        "Total number of bytes replicated to a bucket replication target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_counter_md(
        MetricName::SentCount,
@@ -346,6 +403,15 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new
    )
 });

+pub static BUCKET_REPL_TARGET_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(TARGET_SENT_COUNT),
+        "Total number of objects replicated to a bucket replication target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_counter_md(
        MetricName::from(RESYNC_STARTED_TOTAL),
@@ -400,6 +466,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyL
    )
 });

+pub static BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(TARGET_TOTAL_FAILED_BYTES),
+        "Total number of bytes failed at least once to replicate since server start by bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_counter_md(
        MetricName::TotalFailedCount,
@@ -409,6 +484,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyL
    )
 });

+pub static BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(TARGET_TOTAL_FAILED_COUNT),
+        "Total number of objects which failed replication since server start by bucket and target ARN",
+        &BUCKET_TARGET_LABELS,
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::BandwidthLimitBytesPerSecond,
@@ -435,3 +519,12 @@ pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<Met
        subsystems::BUCKET_REPLICATION,
    )
 });
+
+pub static BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("proxy_requests_total".to_string()),
+        "Total number of bucket replication proxy requests by operation and result",
+        &[BUCKET_L, OPERATION_L, RESULT_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});