feat(metrics): async collection with configurable intervals & graceful shutdown (#1768)

crates/metrics/src/collectors/bucket.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket metrics collector.
+//!
+//! Collects usage metrics for each bucket in the cluster, including
+//! size, object counts, and quota information.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+use std::borrow::Cow;
+
+/// Usage statistics for a single bucket.
+#[derive(Debug, Clone, Default)]
+pub struct BucketStats {
+    /// Bucket name
+    pub name: String,
+    /// Total bytes used by the bucket
+    pub size_bytes: u64,
+    /// Total number of objects in the bucket
+    pub objects_count: u64,
+    /// Quota limit in bytes (0 means no quota)
+    pub quota_bytes: u64,
+}
+
+// Static metric definitions
+const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes";
+const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total";
+const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes";
+
+const HELP_SIZE: &str = "Total bytes used by the bucket";
+const HELP_OBJECTS: &str = "Total number of objects in the bucket";
+const HELP_QUOTA: &str = "Quota limit in bytes for the bucket";
+
+/// Collects per-bucket usage metrics from the provided bucket statistics.
+///
+/// # Metrics Produced
+///
+/// For each bucket, the following metrics are produced with a `bucket` label:
+///
+/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket
+/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket
+/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured)
+///
+/// # Arguments
+///
+/// * `buckets` - Slice of bucket statistics
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats};
+///
+/// let buckets = vec![
+///     BucketStats {
+///         name: "my-bucket".to_string(),
+///         size_bytes: 1_000_000,
+///         objects_count: 100,
+///         quota_bytes: 10_000_000,
+///     },
+/// ];
+/// let metrics = collect_bucket_metrics(&buckets);
+/// assert_eq!(metrics.len(), 3); // size, objects, quota
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
+    if buckets.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(buckets.len() * 3);
+
+    for bucket in buckets {
+        let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
+
+        // Bucket size in bytes
+        metrics.push(
+            PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        // Object count
+        metrics.push(
+            PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        // Quota (always emit, 0 when no quota configured for consistent PromQL queries)
+        metrics.push(
+            PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64)
+                .with_label("bucket", bucket_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_bucket_metrics() {
+        let buckets = vec![
+            BucketStats {
+                name: "test-bucket".to_string(),
+                size_bytes: 1000,
+                objects_count: 50,
+                quota_bytes: 0,
+            },
+            BucketStats {
+                name: "other-bucket".to_string(),
+                size_bytes: 2000,
+                objects_count: 100,
+                quota_bytes: 0,
+            },
+        ];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here.
+
+        // 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
+        assert_eq!(metrics.len(), 6);
+
+        // Verify test-bucket metrics
+        let test_bucket_size = metrics
+            .iter()
+            .find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
+        assert!(test_bucket_size.is_some());
+        assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_with_quotas() {
+        let buckets = vec![BucketStats {
+            name: "quota-bucket".to_string(),
+            size_bytes: 500,
+            objects_count: 10,
+            quota_bytes: 10000,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // 1 bucket * 3 metrics (size, objects, quota) = 3 metrics
+        assert_eq!(metrics.len(), 3);
+
+        // Verify quota metric exists
+        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        assert!(quota_metric.is_some());
+        assert_eq!(quota_metric.map(|m| m.value), Some(10000.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_empty() {
+        let buckets: Vec<BucketStats> = vec![];
+        let metrics = collect_bucket_metrics(&buckets);
+        assert!(metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_zero_quota_always_reported() {
+        let buckets = vec![BucketStats {
+            name: "no-quota-bucket".to_string(),
+            size_bytes: 100,
+            objects_count: 5,
+            quota_bytes: 0,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
+        assert_eq!(metrics.len(), 3);
+        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        assert!(quota_metric.is_some());
+        assert_eq!(quota_metric.map(|m| m.value), Some(0.0));
+    }
+
+    #[test]
+    fn test_bucket_stats_default() {
+        let stats = BucketStats::default();
+        assert!(stats.name.is_empty());
+        assert_eq!(stats.size_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.quota_bytes, 0);
+    }
+}

crates/metrics/src/collectors/cluster.rs

@@ -0,0 +1,196 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Cluster-wide metrics collector.
+//!
+//! Collects aggregate metrics across the entire RustFS cluster including
+//! total capacity, usage, and object counts.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+
+/// Cluster capacity and usage statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting cluster metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterStats {
+    /// Total raw storage capacity across all disks in bytes
+    pub raw_capacity_bytes: u64,
+    /// Usable capacity after erasure coding overhead in bytes
+    pub usable_capacity_bytes: u64,
+    /// Currently used storage in bytes
+    pub used_bytes: u64,
+    /// Available free storage in bytes
+    pub free_bytes: u64,
+    /// Total number of objects in the cluster
+    pub objects_count: u64,
+    /// Total number of buckets in the cluster
+    pub buckets_count: u64,
+}
+
+// Static metric definitions to avoid allocations
+const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes";
+const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes";
+const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes";
+const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes";
+const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total";
+const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total";
+
+const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks";
+const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)";
+const HELP_USED: &str = "Total used storage capacity in bytes";
+const HELP_FREE: &str = "Total free storage capacity in bytes";
+const HELP_OBJECTS: &str = "Total number of objects in the cluster";
+const HELP_BUCKETS: &str = "Total number of buckets in the cluster";
+
+/// Number of metrics produced by this collector.
+const METRIC_COUNT: usize = 6;
+
+/// Collects cluster-wide metrics from the provided statistics.
+///
+/// # Metrics Produced
+///
+/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks
+/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead
+/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity
+/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity
+/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster
+/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster
+///
+/// # Arguments
+///
+/// * `stats` - Cluster statistics containing capacity and usage data
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats};
+///
+/// let stats = ClusterStats {
+///     raw_capacity_bytes: 10_000_000_000,
+///     usable_capacity_bytes: 8_000_000_000,
+///     used_bytes: 2_000_000_000,
+///     free_bytes: 6_000_000_000,
+///     objects_count: 1000,
+///     buckets_count: 10,
+/// };
+/// let metrics = collect_cluster_metrics(&stats);
+/// assert_eq!(metrics.len(), 6);
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(METRIC_COUNT);
+
+    metrics.push(PrometheusMetric::new(
+        METRIC_RAW_CAPACITY,
+        MetricType::Gauge,
+        HELP_RAW_CAPACITY,
+        stats.raw_capacity_bytes as f64,
+    ));
+    metrics.push(PrometheusMetric::new(
+        METRIC_USABLE_CAPACITY,
+        MetricType::Gauge,
+        HELP_USABLE_CAPACITY,
+        stats.usable_capacity_bytes as f64,
+    ));
+    metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64));
+    metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64));
+    metrics.push(PrometheusMetric::new(
+        METRIC_OBJECTS,
+        MetricType::Gauge,
+        HELP_OBJECTS,
+        stats.objects_count as f64,
+    ));
+    metrics.push(PrometheusMetric::new(
+        METRIC_BUCKETS,
+        MetricType::Gauge,
+        HELP_BUCKETS,
+        stats.buckets_count as f64,
+    ));
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_cluster_metrics() {
+        let stats = ClusterStats {
+            raw_capacity_bytes: 3000,
+            usable_capacity_bytes: 2500,
+            used_bytes: 1200,
+            free_bytes: 1300,
+            objects_count: 100,
+            buckets_count: 5,
+        };
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // Verify raw capacity
+        let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY);
+        assert!(raw_capacity.is_some());
+        assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0));
+
+        // Verify used capacity
+        let used = metrics.iter().find(|m| m.name == METRIC_USED);
+        assert!(used.is_some());
+        assert_eq!(used.map(|m| m.value), Some(1200.0));
+
+        // Verify object count
+        let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS);
+        assert!(objects.is_some());
+        assert_eq!(objects.map(|m| m.value), Some(100.0));
+
+        // Verify bucket count
+        let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS);
+        assert!(buckets.is_some());
+        assert_eq!(buckets.map(|m| m.value), Some(5.0));
+    }
+
+    #[test]
+    fn test_collect_cluster_metrics_empty() {
+        let stats = ClusterStats::default();
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // All values should be zero
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_cluster_stats_default() {
+        let stats = ClusterStats::default();
+        assert_eq!(stats.raw_capacity_bytes, 0);
+        assert_eq!(stats.usable_capacity_bytes, 0);
+        assert_eq!(stats.used_bytes, 0);
+        assert_eq!(stats.free_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.buckets_count, 0);
+    }
+}

crates/metrics/src/collectors/global.rs

@@ -0,0 +1,316 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::collectors::{
+    BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics, collect_cluster_metrics, collect_node_metrics,
+    collect_resource_metrics,
+};
+use crate::constants::{
+    DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL,
+    DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
+    ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
+};
+use crate::format::report_metrics;
+use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
+use rustfs_ecstore::data_usage::load_data_usage_from_backend;
+use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
+use rustfs_ecstore::store_api::BucketOptions;
+use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use rustfs_utils::get_env_opt_u64;
+use std::sync::OnceLock;
+use std::time::{Duration, Instant};
+use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+
+/// Process start time for calculating uptime.
+static PROCESS_START: OnceLock<Instant> = OnceLock::new();
+
+/// Get the process start time, initializing it on first call.
+#[inline]
+fn get_process_start() -> &'static Instant {
+    PROCESS_START.get_or_init(Instant::now)
+}
+
+/// Collect cluster statistics from the storage layer.
+async fn collect_cluster_stats() -> ClusterStats {
+    let Some(store) = new_object_layer_fn() else {
+        return ClusterStats::default();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
+    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
+    let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
+    let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
+
+    // Get bucket and object counts from data usage info
+    let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
+        Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
+        Err(e) => {
+            warn!("Failed to load data usage from backend: {}", e);
+            // Fall back to bucket list for buckets_count, objects_count stays 0
+            let buckets = store
+                .list_bucket(&BucketOptions {
+                    cached: true,
+                    ..Default::default()
+                })
+                .await
+                .unwrap_or_else(|e| {
+                    warn!("Failed to list buckets for cluster metrics: {}", e);
+                    Vec::new()
+                });
+            (buckets.len() as u64, 0)
+        }
+    };
+
+    ClusterStats {
+        raw_capacity_bytes: raw_capacity,
+        usable_capacity_bytes: usable_capacity,
+        used_bytes: used,
+        free_bytes: free,
+        objects_count,
+        buckets_count,
+    }
+}
+
+/// Collect bucket statistics from the storage layer.
+async fn collect_bucket_stats() -> Vec<BucketStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    // Load data usage info from backend to get bucket sizes and object counts
+    let data_usage = match load_data_usage_from_backend(store.clone()).await {
+        Ok(info) => Some(info),
+        Err(e) => {
+            warn!("Failed to load data usage from backend for bucket metrics: {}", e);
+            None
+        }
+    };
+
+    let buckets = match store
+        .list_bucket(&BucketOptions {
+            cached: true,
+            ..Default::default()
+        })
+        .await
+    {
+        Ok(b) => b,
+        Err(e) => {
+            warn!("Failed to list buckets for metrics: {}", e);
+            return Vec::new();
+        }
+    };
+
+    // Build bucket stats with real data from DataUsageInfo
+    let mut stats = Vec::with_capacity(buckets.len());
+    for bucket in buckets {
+        if bucket.name.starts_with('.') {
+            continue;
+        }
+
+        // Get size and objects_count from data usage info
+        let (size_bytes, objects_count) = data_usage
+            .as_ref()
+            .and_then(|du| du.buckets_usage.get(&bucket.name))
+            .map(|bui| (bui.size, bui.objects_count))
+            .unwrap_or((0, 0));
+
+        // Get quota from bucket metadata
+        let quota_bytes = match get_quota_config(&bucket.name).await {
+            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
+            Err(_) => 0, // No quota configured or error
+        };
+
+        stats.push(BucketStats {
+            name: bucket.name,
+            size_bytes,
+            objects_count,
+            quota_bytes,
+        });
+    }
+
+    stats
+}
+
+/// Collect disk statistics from the storage layer.
+async fn collect_disk_stats() -> Vec<DiskStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    storage_info
+        .disks
+        .iter()
+        .map(|disk| DiskStats {
+            server: disk.endpoint.clone(),
+            drive: disk.drive_path.clone(),
+            total_bytes: disk.total_space,
+            used_bytes: disk.used_space,
+            free_bytes: disk.available_space,
+        })
+        .collect()
+}
+
+/// Collect resource statistics for the current process.
+///
+/// Collects:
+/// - Uptime: Calculated from process start time
+/// - Memory: Process resident set size from sysinfo
+/// - CPU: Process CPU usage percentage from sysinfo
+#[inline]
+fn collect_process_stats() -> ResourceStats {
+    let uptime_seconds = get_process_start().elapsed().as_secs();
+
+    // Use sysinfo for process metrics
+    let mut sys = System::new();
+    let pid = Pid::from_u32(std::process::id());
+    sys.refresh_processes_specifics(
+        ProcessesToUpdate::Some(&[pid]),
+        true,
+        ProcessRefreshKind::nothing().with_cpu().with_memory(),
+    );
+
+    if let Some(process) = sys.process(pid) {
+        ResourceStats {
+            cpu_percent: process.cpu_usage() as f64,
+            memory_bytes: process.memory(),
+            uptime_seconds,
+        }
+    } else {
+        // Fallback if process not found
+        ResourceStats {
+            cpu_percent: 0.0,
+            memory_bytes: 0,
+            uptime_seconds,
+        }
+    }
+}
+
+/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics.
+///
+/// This function spawns background tasks that periodically collect metrics
+/// and report them using the `metrics` crate.
+///
+/// # Arguments
+///
+/// * `token` - A cancellation token to gracefully stop the metrics collection tasks.
+pub fn init_metrics_collectors(token: CancellationToken) {
+    // Initialize process start time
+    get_process_start();
+
+    // Helper closure to determine interval for a specific metric type
+    let get_interval = |env_key: &str, type_default: Duration| -> Duration {
+        // 1. Try specific env var
+        // 2. Fallback to global default env var (if set differently from hardcoded default)
+        // 3. Fallback to type specific default
+
+        // Helper to check if value is valid (non-zero)
+        let is_valid = |v: u64| v > 0;
+
+        if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) {
+            Duration::from_secs(val)
+        } else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) {
+            Duration::from_secs(val)
+        } else {
+            type_default
+        }
+    };
+
+    let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
+    let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
+    let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
+    let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
+
+    // Spawn task for cluster metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(cluster_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_cluster_stats().await;
+                    let metrics = collect_cluster_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for cluster stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for bucket metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(bucket_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_bucket_stats().await;
+                    let metrics = collect_bucket_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for bucket stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for node/disk metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(node_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_disk_stats().await;
+                    let metrics = collect_node_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for node/disk stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for resource metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(resource_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    // Resource stats collection is synchronous but fast
+                    let stats = collect_process_stats();
+                    let metrics = collect_resource_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for resource stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+}

crates/metrics/src/collectors/mod.rs

@@ -0,0 +1,73 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Prometheus metric collectors for RustFS.
+//!
+//! This module provides collectors that convert RustFS data into Prometheus
+//! metrics format. Each collector is responsible for a specific domain:
+//!
+//! - [`cluster`]: Cluster-wide capacity and object statistics
+//! - [`bucket`]: Per-bucket usage and quota metrics
+//! - [`node`]: Per-node disk capacity and health metrics
+//! - [`resource`]: System resource metrics (CPU, memory, uptime)
+//!
+//! # Design Philosophy
+//!
+//! Collectors accept simple data structs rather than internal RustFS types.
+//! This design allows HTTP handlers to populate the structs from their
+//! available data sources without creating circular dependencies.
+//!
+//! # Example
+//!
+//! ```
+//! use rustfs_metrics::collectors::{
+//!     collect_cluster_metrics, ClusterStats,
+//!     collect_bucket_metrics, BucketStats,
+//!     collect_node_metrics, DiskStats,
+//!     collect_resource_metrics, ResourceStats,
+//! };
+//! use rustfs_metrics::report_metrics;
+//!
+//! // Collect cluster metrics
+//! let cluster_stats = ClusterStats {
+//!     raw_capacity_bytes: 1_000_000_000,
+//!     used_bytes: 500_000_000,
+//!     ..Default::default()
+//! };
+//! let mut metrics = collect_cluster_metrics(&cluster_stats);
+//!
+//! // Add bucket metrics
+//! let bucket_stats = vec![BucketStats {
+//!     name: "my-bucket".to_string(),
+//!     size_bytes: 100_000,
+//!     objects_count: 50,
+//!     ..Default::default()
+//! }];
+//! metrics.extend(collect_bucket_metrics(&bucket_stats));
+//!
+//! // Report to metrics system
+//! report_metrics(&metrics);
+//! ```
+
+mod bucket;
+mod cluster;
+pub(crate) mod global;
+mod node;
+mod resource;
+
+pub use bucket::{BucketStats, collect_bucket_metrics};
+pub use cluster::{ClusterStats, collect_cluster_metrics};
+pub use global::init_metrics_collectors;
+pub use node::{DiskStats, collect_node_metrics};
+pub use resource::{ResourceStats, collect_resource_metrics};

crates/metrics/src/collectors/node.rs

@@ -0,0 +1,196 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-node and per-disk metrics collector.
+//!
+//! Collects storage metrics for each disk/drive in the cluster,
+//! including capacity, usage, and health status.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+use std::borrow::Cow;
+
+/// Statistics for a single disk/drive.
+#[derive(Debug, Clone, Default)]
+pub struct DiskStats {
+    /// Server endpoint (e.g., "node1:9000")
+    pub server: String,
+    /// Drive path (e.g., "/data/disk1")
+    pub drive: String,
+    /// Total capacity in bytes
+    pub total_bytes: u64,
+    /// Used space in bytes
+    pub used_bytes: u64,
+    /// Free space in bytes
+    pub free_bytes: u64,
+}
+
+// Static metric definitions
+const METRIC_TOTAL: &str = "rustfs_node_disk_total_bytes";
+const METRIC_USED: &str = "rustfs_node_disk_used_bytes";
+const METRIC_FREE: &str = "rustfs_node_disk_free_bytes";
+
+const HELP_TOTAL: &str = "Total disk capacity in bytes";
+const HELP_USED: &str = "Used disk space in bytes";
+const HELP_FREE: &str = "Free disk space in bytes";
+
+/// Collects per-node disk metrics from the provided disk statistics.
+///
+/// # Metrics Produced
+///
+/// For each disk, the following metrics are produced with `server` and `drive` labels:
+///
+/// - `rustfs_node_disk_total_bytes`: Total capacity of the disk
+/// - `rustfs_node_disk_used_bytes`: Used space on the disk
+/// - `rustfs_node_disk_free_bytes`: Free space on the disk
+///
+/// # Arguments
+///
+/// * `disks` - Slice of disk statistics
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_node_metrics, DiskStats};
+///
+/// let disks = vec![
+///     DiskStats {
+///         server: "node1:9000".to_string(),
+///         drive: "/data/disk1".to_string(),
+///         total_bytes: 1_000_000_000,
+///         used_bytes: 400_000_000,
+///         free_bytes: 600_000_000,
+///     },
+/// ];
+/// let metrics = collect_node_metrics(&disks);
+/// assert_eq!(metrics.len(), 3);
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec<PrometheusMetric> {
+    if disks.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(disks.len() * 3);
+
+    for disk in disks {
+        let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone());
+        let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone());
+
+        metrics.push(
+            PrometheusMetric::new(METRIC_TOTAL, MetricType::Gauge, HELP_TOTAL, disk.total_bytes as f64)
+                .with_label("server", server_label.clone())
+                .with_label("drive", drive_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, disk.used_bytes as f64)
+                .with_label("server", server_label.clone())
+                .with_label("drive", drive_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, disk.free_bytes as f64)
+                .with_label("server", server_label)
+                .with_label("drive", drive_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_node_metrics() {
+        let disks = vec![
+            DiskStats {
+                server: "node1:9000".to_string(),
+                drive: "/data/disk1".to_string(),
+                total_bytes: 1000000,
+                used_bytes: 400000,
+                free_bytes: 600000,
+            },
+            DiskStats {
+                server: "node2:9000".to_string(),
+                drive: "/data/disk2".to_string(),
+                total_bytes: 2000000,
+                used_bytes: 800000,
+                free_bytes: 1200000,
+            },
+        ];
+
+        let metrics = collect_node_metrics(&disks);
+
+        // 2 disks * 3 metrics each = 6 metrics
+        assert_eq!(metrics.len(), 6);
+
+        // Verify node1 disk1 total bytes
+        let node1_total = metrics.iter().find(|m| {
+            m.name == METRIC_TOTAL
+                && m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000")
+                && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1")
+        });
+        assert!(node1_total.is_some());
+        assert_eq!(node1_total.map(|m| m.value), Some(1000000.0));
+
+        // Verify node2 disk2 used bytes
+        let node2_used = metrics.iter().find(|m| {
+            m.name == METRIC_USED
+                && m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000")
+                && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2")
+        });
+        assert!(node2_used.is_some());
+        assert_eq!(node2_used.map(|m| m.value), Some(800000.0));
+    }
+
+    #[test]
+    fn test_collect_node_metrics_empty() {
+        let disks: Vec<DiskStats> = vec![];
+        let metrics = collect_node_metrics(&disks);
+        assert!(metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_node_metrics_labels() {
+        let disks = vec![DiskStats {
+            server: "localhost:9000".to_string(),
+            drive: "/mnt/data".to_string(),
+            total_bytes: 500,
+            used_bytes: 200,
+            free_bytes: 300,
+        }];
+
+        let metrics = collect_node_metrics(&disks);
+
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 2);
+            assert!(metric.labels.iter().any(|(k, _)| *k == "server"));
+            assert!(metric.labels.iter().any(|(k, _)| *k == "drive"));
+        }
+    }
+
+    #[test]
+    fn test_disk_stats_default() {
+        let stats = DiskStats::default();
+        assert!(stats.server.is_empty());
+        assert!(stats.drive.is_empty());
+        assert_eq!(stats.total_bytes, 0);
+        assert_eq!(stats.used_bytes, 0);
+        assert_eq!(stats.free_bytes, 0);
+    }
+}

crates/metrics/src/collectors/resource.rs

@@ -0,0 +1,170 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! System resource metrics collector.
+//!
+//! Collects system-level metrics for the RustFS process including
+//! CPU usage, memory consumption, and process uptime.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+
+/// Resource statistics for a RustFS process.
+///
+/// This struct encapsulates the resource usage data that can be
+/// collected from the operating system for the current process.
+#[derive(Debug, Clone, Default)]
+pub struct ResourceStats {
+    /// CPU usage as a percentage (0.0 to 100.0+)
+    pub cpu_percent: f64,
+    /// Resident memory usage in bytes
+    pub memory_bytes: u64,
+    /// Process uptime in seconds
+    pub uptime_seconds: u64,
+}
+
+// Static metric definitions
+const METRIC_CPU: &str = "rustfs_process_cpu_percent";
+const METRIC_MEMORY: &str = "rustfs_process_memory_bytes";
+const METRIC_UPTIME: &str = "rustfs_process_uptime_seconds";
+
+const HELP_CPU: &str = "CPU usage of the RustFS process as a percentage";
+const HELP_MEMORY: &str = "Resident memory usage of the RustFS process in bytes";
+const HELP_UPTIME: &str = "Uptime of the RustFS process in seconds";
+
+/// Number of metrics produced by this collector.
+const METRIC_COUNT: usize = 3;
+
+/// Collects system resource metrics from the provided statistics.
+///
+/// # Metrics Produced
+///
+/// - `rustfs_process_cpu_percent`: CPU usage as a percentage
+/// - `rustfs_process_memory_bytes`: Resident memory usage in bytes
+/// - `rustfs_process_uptime_seconds`: Process uptime in seconds
+///
+/// # Arguments
+///
+/// * `stats` - Resource statistics for the current process
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_resource_metrics, ResourceStats};
+///
+/// let stats = ResourceStats {
+///     cpu_percent: 25.5,
+///     memory_bytes: 1024 * 1024 * 512, // 512 MB
+///     uptime_seconds: 3600,
+/// };
+/// let metrics = collect_resource_metrics(&stats);
+/// assert_eq!(metrics.len(), 3);
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(METRIC_COUNT);
+
+    metrics.push(PrometheusMetric::new(METRIC_CPU, MetricType::Gauge, HELP_CPU, stats.cpu_percent));
+
+    metrics.push(PrometheusMetric::new(
+        METRIC_MEMORY,
+        MetricType::Gauge,
+        HELP_MEMORY,
+        stats.memory_bytes as f64,
+    ));
+
+    metrics.push(PrometheusMetric::new(
+        METRIC_UPTIME,
+        MetricType::Gauge,
+        HELP_UPTIME,
+        stats.uptime_seconds as f64,
+    ));
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_resource_metrics() {
+        let stats = ResourceStats {
+            cpu_percent: 45.5,
+            memory_bytes: 1024 * 1024 * 256,
+            uptime_seconds: 7200,
+        };
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        // Verify CPU metric
+        let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
+        assert!(cpu.is_some());
+        assert_eq!(cpu.map(|m| m.value), Some(45.5));
+
+        // Verify memory metric
+        let memory = metrics.iter().find(|m| m.name == METRIC_MEMORY);
+        assert!(memory.is_some());
+        assert_eq!(memory.map(|m| m.value), Some((1024 * 1024 * 256) as f64));
+
+        // Verify uptime metric
+        let uptime = metrics.iter().find(|m| m.name == METRIC_UPTIME);
+        assert!(uptime.is_some());
+        assert_eq!(uptime.map(|m| m.value), Some(7200.0));
+    }
+
+    #[test]
+    fn test_collect_resource_metrics_zero_values() {
+        let stats = ResourceStats::default();
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_collect_resource_metrics_high_cpu() {
+        let stats = ResourceStats {
+            cpu_percent: 150.0, // Can exceed 100% on multi-core systems
+            memory_bytes: 0,
+            uptime_seconds: 0,
+        };
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
+        assert!(cpu.is_some());
+        assert_eq!(cpu.map(|m| m.value), Some(150.0));
+    }
+
+    #[test]
+    fn test_resource_stats_default() {
+        let stats = ResourceStats::default();
+        assert_eq!(stats.cpu_percent, 0.0);
+        assert_eq!(stats.memory_bytes, 0);
+        assert_eq!(stats.uptime_seconds, 0);
+    }
+}

crates/metrics/src/constants/mod.rs

@@ -0,0 +1,42 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+/// Environment variable key for the global default metrics interval (seconds).
+pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
+/// Default interval for metrics collection if not specified otherwise.
+#[allow(dead_code)]
+pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for cluster metrics interval (seconds).
+pub const ENV_CLUSTER_METRICS_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL_SEC";
+/// Default interval for collecting cluster-wide metrics (capacity, object counts).
+pub const DEFAULT_CLUSTER_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for bucket metrics interval (seconds).
+pub const ENV_BUCKET_METRICS_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL_SEC";
+/// Default interval for collecting per-bucket metrics (usage, quotas).
+/// This can be expensive if there are many buckets, so a longer interval is recommended.
+pub const DEFAULT_BUCKET_METRICS_INTERVAL: Duration = Duration::from_secs(300);
+
+/// Environment variable key for node metrics interval (seconds).
+pub const ENV_NODE_METRICS_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL_SEC";
+/// Default interval for collecting node/disk metrics.
+pub const DEFAULT_NODE_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for resource metrics interval (seconds).
+pub const ENV_RESOURCE_METRICS_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL_SEC";
+/// Default interval for collecting system resource metrics (CPU, memory).
+pub const DEFAULT_RESOURCE_METRICS_INTERVAL: Duration = Duration::from_secs(15);

crates/metrics/src/format.rs

@@ -0,0 +1,128 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Prometheus text exposition format renderer.
+//!
+//! This module renders metrics in the standard Prometheus text format.
+//! Optimized for minimal allocations and fast rendering.
+
+use crate::MetricType;
+use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge};
+use std::borrow::Cow;
+
+/// Report metrics using the `metrics` crate.
+///
+/// This function iterates over the provided metrics and reports them using
+/// the `metrics` crate's API. This allows integration with various metrics
+/// exporters (e.g., Prometheus) that are configured globally.
+pub fn report_metrics(metrics: &[PrometheusMetric]) {
+    for metric in metrics {
+        // Register metric description (help text)
+        // Note: In a real-world scenario, descriptions should ideally be registered once at startup.
+        // However, the `metrics` crate handles duplicate registrations gracefully.
+        match metric.metric_type {
+            MetricType::Counter => describe_counter!(metric.name, metric.help),
+            MetricType::Gauge => describe_gauge!(metric.name, metric.help),
+            MetricType::Histogram => describe_histogram!(metric.name, metric.help),
+        }
+
+        // Convert labels to the format expected by `metrics` crate
+        let labels: Vec<(String, String)> = metric.labels.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
+
+        // Report the metric value
+        match metric.metric_type {
+            MetricType::Counter => {
+                // Use counter! macro to get a handle, then set absolute value.
+                // Note: `metrics` crate counters are typically monotonic and support `increment`.
+                // Setting an absolute value directly requires `absolute` method if supported by the backend/handle,
+                // or we assume the value provided is the absolute count we want to report.
+                //
+                // Since `metrics` 0.21+, `Counter` has an `absolute` method which sets the counter to a specific value.
+                // This is useful for mirroring an external counter.
+                let counter = counter!(metric.name, &labels);
+                counter.absolute(metric.value as u64);
+            }
+            MetricType::Gauge => {
+                let gauge = gauge!(metric.name, &labels);
+                gauge.set(metric.value);
+            }
+            MetricType::Histogram => {
+                let histogram = metrics::histogram!(metric.name, &labels);
+                histogram.record(metric.value);
+            }
+        }
+    }
+}
+
+/// A single Prometheus metric with labels and value.
+///
+/// This struct is optimized for performance by using `&'static str` for
+/// the name and help text, which are typically compile-time constants.
+/// Labels use `Cow<'static, str>` to avoid allocations when possible.
+#[derive(Debug, Clone)]
+pub struct PrometheusMetric {
+    /// The metric name (e.g., "http_requests_total").
+    pub name: &'static str,
+    /// The type of this metric (counter, gauge, or histogram).
+    pub metric_type: MetricType,
+    /// Human-readable description shown in Prometheus UI.
+    pub help: &'static str,
+    /// Key-value label pairs for this metric instance.
+    /// Uses Cow to avoid allocations for static label keys.
+    pub labels: Vec<(&'static str, Cow<'static, str>)>,
+    /// The numeric value of this metric.
+    pub value: f64,
+}
+
+impl PrometheusMetric {
+    /// Creates a new metric with the given name, type, help text, and value.
+    ///
+    /// Uses static strings to avoid heap allocations for metric metadata.
+    #[inline]
+    pub const fn new(name: &'static str, metric_type: MetricType, help: &'static str, value: f64) -> Self {
+        Self {
+            name,
+            metric_type,
+            help,
+            labels: Vec::new(),
+            value,
+        }
+    }
+
+    /// Adds a single label with a static value to this metric.
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
+        self.labels.push((key, value.into()));
+        self
+    }
+
+    /// Adds a label with an owned string value.
+    ///
+    /// Use this when the label value is dynamically generated.
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
+        self.labels.push((key, Cow::Owned(value)));
+        self
+    }
+
+    /// Sets all labels for this metric, replacing any existing labels.
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
+        self.labels = labels;
+        self
+    }
+}

crates/metrics/src/global.rs

@@ -0,0 +1,39 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use tokio_util::sync::CancellationToken;
+
+/// Initializes the global metrics system. This should be called once at the start of the application.
+/// The provided `CancellationToken` will be used to gracefully shut down the metrics system when needed.
+///
+/// # Arguments
+/// * `token` - A `CancellationToken` that can be used to signal the metrics system to shut down gracefully.
+///
+/// # Example
+/// ```ignore
+/// use tokio_util::sync::CancellationToken;
+/// use rustfs_metrics::init_metrics_system;
+///
+/// let token = CancellationToken::new();
+/// init_metrics_system(token.clone());
+///
+/// // Later, when you want to shut down the metrics system:
+/// token.cancel();
+/// ```
+/// Note: This function should only be called once during the application's lifecycle. Calling it multiple times may lead to unexpected behavior.
+pub fn init_metrics_system(token: CancellationToken) {
+    tracing::info!("init metrics system start");
+    crate::collectors::init_metrics_collectors(token);
+    tracing::info!("init metrics system done");
+}

crates/metrics/src/lib.rs

@@ -0,0 +1,23 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod collectors;
+pub mod constants;
+pub mod format;
+mod global;
+mod metrics_type;
+
+pub use format::report_metrics;
+pub use global::init_metrics_system;
+pub use metrics_type::*;

crates/metrics/src/metrics_type/audit.rs

@@ -0,0 +1,52 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+const TARGET_ID: &str = "target_id";
+pub const RESULT: &str = "result"; // success / failure
+pub const STATUS: &str = "status"; // success / failure
+
+pub const SUCCESS: &str = "success";
+pub const FAILURE: &str = "failure";
+
+pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::AuditFailedMessages,
+        "Total number of messages that failed to send since start",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});
+
+pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::AuditTargetQueueLength,
+        "Number of unsent messages in queue for target",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});
+
+pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::AuditTotalMessages,
+        "Total number of messages sent since start",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});

crates/metrics/src/metrics_type/bucket.rs

@@ -0,0 +1,90 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, new_histogram_md, subsystems};
+use std::sync::LazyLock;
+
+pub static BUCKET_API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficSentBytes,
+        "Total number of bytes received for a bucket",
+        &["bucket", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficRecvBytes,
+        "Total number of bytes sent for a bucket",
+        &["bucket", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_IN_FLIGHT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsInFlightTotal,
+        "Total number of requests currently in flight for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTotal,
+        "Total number of requests for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_CANCELED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsCanceledTotal,
+        "Total number of requests canceled by the client for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_4XX_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests4xxErrorsTotal,
+        "Total number of requests with 4xx errors for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_5XX_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests5xxErrorsTotal,
+        "Total number of requests with 5xx errors for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_histogram_md(
+        MetricName::ApiRequestsTTFBSecondsDistribution,
+        "Distribution of time to first byte across API calls for a bucket",
+        &["bucket", "name", "le", "type"],
+        subsystems::BUCKET_API,
+    )
+});

crates/metrics/src/metrics_type/bucket_replication.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Bucket level replication metric descriptor
+pub const BUCKET_L: &str = "bucket";
+/// Replication operation
+pub const OPERATION_L: &str = "operation";
+/// Replication target ARN
+pub const TARGET_ARN_L: &str = "targetArn";
+/// Replication range
+pub const RANGE_L: &str = "range";
+
+pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastHourFailedBytes,
+        "Total number of bytes failed at least once to replicate in the last hour on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastHourFailedCount,
+        "Total number of objects which failed replication in the last hour on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastMinFailedBytes,
+        "Total number of bytes failed at least once to replicate in the last full minute on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastMinFailedCount,
+        "Total number of objects which failed replication in the last full minute on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LatencyMilliSec,
+        "Replication latency on a bucket in milliseconds",
+        &[BUCKET_L, OPERATION_L, RANGE_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedDeleteTaggingRequestsTotal,
+        "Number of DELETE tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetRequestsFailures,
+        "Number of failures in GET requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetRequestsTotal,
+        "Number of GET requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of PUT requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetTaggingRequestFailures,
+        "Number of failures in GET tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetTaggingRequestsTotal,
+        "Number of GET tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedHeadRequestsFailures,
+        "Number of failures in HEAD requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedHeadRequestsTotal,
+        "Number of HEAD requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of PUT requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedPutTaggingRequestFailures,
+        "Number of failures in PUT tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedPutTaggingRequestsTotal,
+        "Number of PUT tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SentBytes,
+        "Total number of bytes replicated to the target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SentCount,
+        "Total number of objects replicated to the target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TotalFailedBytes,
+        "Total number of bytes failed at least once to replicate since server start",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TotalFailedCount,
+        "Total number of objects which failed replication since server start",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of DELETE requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedDeleteTaggingRequestFailures,
+        "Number of failures in DELETE tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});

crates/metrics/src/metrics_type/cluster_config.rs

@@ -0,0 +1,36 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static CONFIG_RRS_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ConfigRRSParity,
+        "Reduced redundancy storage class parity",
+        &[],
+        subsystems::CLUSTER_CONFIG,
+    )
+});
+
+pub static CONFIG_STANDARD_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ConfigStandardParity,
+        "Standard storage class parity",
+        &[],
+        subsystems::CLUSTER_CONFIG,
+    )
+});

crates/metrics/src/metrics_type/cluster_erasure_set.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// The label for the pool ID
+pub const POOL_ID_L: &str = "pool_id";
+/// The label for the pool ID
+pub const SET_ID_L: &str = "set_id";
+
+pub static ERASURE_SET_OVERALL_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOverallWriteQuorum,
+        "Overall write quorum across pools and sets",
+        &[],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_OVERALL_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOverallHealth,
+        "Overall health across pools and sets (1=healthy, 0=unhealthy)",
+        &[],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadQuorum,
+        "Read quorum for the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteQuorum,
+        "Write quorum for the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_ONLINE_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOnlineDrivesCount,
+        "Count of online drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_HEALING_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetHealingDrivesCount,
+        "Count of healing drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetHealth,
+        "Health of the erasure set in a pool (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_TOLERANCE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadTolerance,
+        "No of drive failures that can be tolerated without disrupting read operations",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_TOLERANCE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteTolerance,
+        "No of drive failures that can be tolerated without disrupting write operations",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadHealth,
+        "Health of the erasure set in a pool for read operations (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteHealth,
+        "Health of the erasure set in a pool for write operations (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});

crates/metrics/src/metrics_type/cluster_health.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static HEALTH_DRIVES_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesOfflineCount,
+        "Count of offline drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});
+
+pub static HEALTH_DRIVES_ONLINE_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesOnlineCount,
+        "Count of online drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});
+
+pub static HEALTH_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesCount,
+        "Count of all drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});

crates/metrics/src/metrics_type/cluster_iam.rs

@@ -0,0 +1,108 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
+use std::sync::LazyLock;
+
+pub static LAST_SYNC_DURATION_MILLIS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::LastSyncDurationMillis,
+        "Last successful IAM data sync duration in milliseconds",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_FAILED_REQUESTS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceFailedRequestsMinute,
+        "When plugin authentication is configured, returns failed requests count in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceLastFailSeconds,
+        "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceLastSuccSeconds,
+        "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceSuccAvgRttMsMinute,
+        "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceSuccMaxRttMsMinute,
+        "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceTotalRequestsMinute,
+        "When plugin authentication is configured, returns total requests count in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SINCE_LAST_SYNC_MILLIS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SinceLastSyncMillis,
+        "Time (in milliseconds) since last successful IAM data sync.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SYNC_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SyncFailures,
+        "Number of failed IAM data syncs since server start.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SYNC_SUCCESSES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SyncSuccesses,
+        "Number of successful IAM data syncs since server start.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});

crates/metrics/src/metrics_type/cluster_notification.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
+use std::sync::LazyLock;
+
+pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationCurrentSendInProgress,
+        "Number of concurrent async Send calls active to all targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsErrorsTotal,
+        "Events that were failed to be sent to the targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsSentTotal,
+        "Total number of events sent to the targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsSkippedTotal,
+        "Events that were skipped to be sent to the targets due to the in-memory queue being full",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});

crates/metrics/src/metrics_type/cluster_usage.rs

@@ -0,0 +1,158 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Bucket labels
+pub const BUCKET_LABEL: &str = "bucket";
+/// Range labels
+pub const RANGE_LABEL: &str = "range";
+
+pub static USAGE_SINCE_LAST_UPDATE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageSinceLastUpdateSeconds,
+        "Time since last update of usage metrics in seconds",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageTotalBytes,
+        "Total cluster usage in bytes",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_OBJECTS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageObjectsCount,
+        "Total cluster objects count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_VERSIONS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageVersionsCount,
+        "Total cluster object versions (including delete markers) count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_DELETE_MARKERS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageDeleteMarkersCount,
+        "Total cluster delete markers count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_BUCKETS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketsCount,
+        "Total cluster buckets count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_OBJECTS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageSizeDistribution,
+        "Cluster object size distribution",
+        &[RANGE_LABEL],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_VERSIONS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageVersionCountDistribution,
+        "Cluster object version count distribution",
+        &[RANGE_LABEL],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketTotalBytes,
+        "Total bucket size in bytes",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectsCount,
+        "Total objects count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_VERSIONS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketVersionsCount,
+        "Total object versions (including delete markers) count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_DELETE_MARKERS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketDeleteMarkersCount,
+        "Total delete markers count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketQuotaTotalBytes,
+        "Total bucket quota in bytes",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectSizeDistribution,
+        "Bucket object size distribution",
+        &[RANGE_LABEL, BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectVersionCountDistribution,
+        "Bucket object version count distribution",
+        &[RANGE_LABEL, BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});

crates/metrics/src/metrics_type/entry/descriptor.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
+use std::collections::HashSet;
+
+/// MetricDescriptor - Metric descriptors
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct MetricDescriptor {
+    pub name: MetricName,
+    pub metric_type: MetricType,
+    pub help: String,
+    pub variable_labels: Vec<String>,
+    pub namespace: MetricNamespace,
+    pub subsystem: MetricSubsystem,
+
+    // Internal management values
+    label_set: Option<HashSet<String>>,
+}
+
+impl MetricDescriptor {
+    /// Create a new metric descriptor
+    pub fn new(
+        name: MetricName,
+        metric_type: MetricType,
+        help: String,
+        variable_labels: Vec<String>,
+        namespace: MetricNamespace,
+        subsystem: impl Into<MetricSubsystem>, // Modify the parameter type
+    ) -> Self {
+        Self {
+            name,
+            metric_type,
+            help,
+            variable_labels,
+            namespace,
+            subsystem: subsystem.into(),
+            label_set: None,
+        }
+    }
+
+    /// Get the full metric name, including the prefix and formatting path
+    #[allow(dead_code)]
+    pub fn get_full_metric_name(&self) -> String {
+        let prefix = self.metric_type.as_prom();
+        let namespace = self.namespace.as_str();
+        let formatted_subsystem = self.subsystem.as_str();
+
+        format!("{}{}_{}_{}", prefix, namespace, formatted_subsystem, self.name.as_str())
+    }
+
+    /// check whether the label is in the label set
+    #[allow(dead_code)]
+    pub fn has_label(&mut self, label: &str) -> bool {
+        self.get_label_set().contains(label)
+    }
+
+    /// Gets a collection of tags and creates them if they don't exist
+    pub fn get_label_set(&mut self) -> &HashSet<String> {
+        if self.label_set.is_none() {
+            let mut set = HashSet::with_capacity(self.variable_labels.len());
+            for label in &self.variable_labels {
+                set.insert(label.clone());
+            }
+            self.label_set = Some(set);
+        }
+        self.label_set.as_ref().unwrap()
+    }
+}

crates/metrics/src/metrics_type/entry/metric_name.rs

@@ -0,0 +1,680 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// The metric name is the individual name of the metric
+#[allow(dead_code)]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MetricName {
+    // The generic metric name
+    AuthTotal,
+    CanceledTotal,
+    ErrorsTotal,
+    HeaderTotal,
+    HealTotal,
+    HitsTotal,
+    InflightTotal,
+    InvalidTotal,
+    LimitTotal,
+    MissedTotal,
+    WaitingTotal,
+    IncomingTotal,
+    ObjectTotal,
+    VersionTotal,
+    DeleteMarkerTotal,
+    OfflineTotal,
+    OnlineTotal,
+    OpenTotal,
+    ReadTotal,
+    TimestampTotal,
+    WriteTotal,
+    Total,
+    FreeInodes,
+
+    // Failure statistical metrics
+    LastMinFailedCount,
+    LastMinFailedBytes,
+    LastHourFailedCount,
+    LastHourFailedBytes,
+    TotalFailedCount,
+    TotalFailedBytes,
+
+    // Worker metrics
+    CurrActiveWorkers,
+    AvgActiveWorkers,
+    MaxActiveWorkers,
+    RecentBacklogCount,
+    CurrInQueueCount,
+    CurrInQueueBytes,
+    ReceivedCount,
+    SentCount,
+    CurrTransferRate,
+    AvgTransferRate,
+    MaxTransferRate,
+    CredentialErrors,
+
+    // Link latency metrics
+    CurrLinkLatency,
+    AvgLinkLatency,
+    MaxLinkLatency,
+
+    // Link status metrics
+    LinkOnline,
+    LinkOfflineDuration,
+    LinkDowntimeTotalDuration,
+
+    // Queue metrics
+    AvgInQueueCount,
+    AvgInQueueBytes,
+    MaxInQueueCount,
+    MaxInQueueBytes,
+
+    // Proxy request metrics
+    ProxiedGetRequestsTotal,
+    ProxiedHeadRequestsTotal,
+    ProxiedPutTaggingRequestsTotal,
+    ProxiedGetTaggingRequestsTotal,
+    ProxiedDeleteTaggingRequestsTotal,
+    ProxiedGetRequestsFailures,
+    ProxiedHeadRequestsFailures,
+    ProxiedPutTaggingRequestFailures,
+    ProxiedGetTaggingRequestFailures,
+    ProxiedDeleteTaggingRequestFailures,
+
+    // Byte-related metrics
+    FreeBytes,
+    ReadBytes,
+    RcharBytes,
+    ReceivedBytes,
+    LatencyMilliSec,
+    SentBytes,
+    TotalBytes,
+    UsedBytes,
+    WriteBytes,
+    WcharBytes,
+
+    // Latency metrics
+    LatencyMicroSec,
+    LatencyNanoSec,
+
+    // Information metrics
+    CommitInfo,
+    UsageInfo,
+    VersionInfo,
+
+    // Distribution metrics
+    SizeDistribution,
+    VersionDistribution,
+    TtfbDistribution,
+    TtlbDistribution,
+
+    // Time metrics
+    LastActivityTime,
+    StartTime,
+    UpTime,
+    Memory,
+    Vmemory,
+    Cpu,
+
+    // Expiration and conversion metrics
+    ExpiryMissedTasks,
+    ExpiryMissedFreeVersions,
+    ExpiryMissedTierJournalTasks,
+    ExpiryNumWorkers,
+    TransitionMissedTasks,
+    TransitionedBytes,
+    TransitionedObjects,
+    TransitionedVersions,
+
+    //Tier request metrics
+    TierRequestsSuccess,
+    TierRequestsFailure,
+
+    // KMS metrics
+    KmsOnline,
+    KmsRequestsSuccess,
+    KmsRequestsError,
+    KmsRequestsFail,
+    KmsUptime,
+
+    // Webhook metrics
+    WebhookOnline,
+
+    // API rejection metrics
+    ApiRejectedAuthTotal,
+    ApiRejectedHeaderTotal,
+    ApiRejectedTimestampTotal,
+    ApiRejectedInvalidTotal,
+
+    //API request metrics
+    ApiRequestsWaitingTotal,
+    ApiRequestsIncomingTotal,
+    ApiRequestsInFlightTotal,
+    ApiRequestsTotal,
+    ApiRequestsErrorsTotal,
+    ApiRequests5xxErrorsTotal,
+    ApiRequests4xxErrorsTotal,
+    ApiRequestsCanceledTotal,
+
+    // API distribution metrics
+    ApiRequestsTTFBSecondsDistribution,
+
+    // API traffic metrics
+    ApiTrafficSentBytes,
+    ApiTrafficRecvBytes,
+
+    // Audit metrics
+    AuditFailedMessages,
+    AuditTargetQueueLength,
+    AuditTotalMessages,
+
+    // Metrics related to cluster configurations
+    ConfigRRSParity,
+    ConfigStandardParity,
+
+    // Erasure coding set related metrics
+    ErasureSetOverallWriteQuorum,
+    ErasureSetOverallHealth,
+    ErasureSetReadQuorum,
+    ErasureSetWriteQuorum,
+    ErasureSetOnlineDrivesCount,
+    ErasureSetHealingDrivesCount,
+    ErasureSetHealth,
+    ErasureSetReadTolerance,
+    ErasureSetWriteTolerance,
+    ErasureSetReadHealth,
+    ErasureSetWriteHealth,
+
+    // Cluster health-related metrics
+    HealthDrivesOfflineCount,
+    HealthDrivesOnlineCount,
+    HealthDrivesCount,
+
+    // IAM-related metrics
+    LastSyncDurationMillis,
+    PluginAuthnServiceFailedRequestsMinute,
+    PluginAuthnServiceLastFailSeconds,
+    PluginAuthnServiceLastSuccSeconds,
+    PluginAuthnServiceSuccAvgRttMsMinute,
+    PluginAuthnServiceSuccMaxRttMsMinute,
+    PluginAuthnServiceTotalRequestsMinute,
+    SinceLastSyncMillis,
+    SyncFailures,
+    SyncSuccesses,
+
+    // Notify relevant metrics
+    NotificationCurrentSendInProgress,
+    NotificationEventsErrorsTotal,
+    NotificationEventsSentTotal,
+    NotificationEventsSkippedTotal,
+
+    // Metrics related to the usage of cluster objects
+    UsageSinceLastUpdateSeconds,
+    UsageTotalBytes,
+    UsageObjectsCount,
+    UsageVersionsCount,
+    UsageDeleteMarkersCount,
+    UsageBucketsCount,
+    UsageSizeDistribution,
+    UsageVersionCountDistribution,
+
+    // Metrics related to bucket usage
+    UsageBucketQuotaTotalBytes,
+    UsageBucketTotalBytes,
+    UsageBucketObjectsCount,
+    UsageBucketVersionsCount,
+    UsageBucketDeleteMarkersCount,
+    UsageBucketObjectSizeDistribution,
+    UsageBucketObjectVersionCountDistribution,
+
+    // ILM-related metrics
+    IlmExpiryPendingTasks,
+    IlmTransitionActiveTasks,
+    IlmTransitionPendingTasks,
+    IlmTransitionMissedImmediateTasks,
+    IlmVersionsScanned,
+
+    // Webhook logs
+    WebhookQueueLength,
+    WebhookTotalMessages,
+    WebhookFailedMessages,
+
+    // Copy the relevant metrics
+    ReplicationAverageActiveWorkers,
+    ReplicationAverageQueuedBytes,
+    ReplicationAverageQueuedCount,
+    ReplicationAverageDataTransferRate,
+    ReplicationCurrentActiveWorkers,
+    ReplicationCurrentDataTransferRate,
+    ReplicationLastMinuteQueuedBytes,
+    ReplicationLastMinuteQueuedCount,
+    ReplicationMaxActiveWorkers,
+    ReplicationMaxQueuedBytes,
+    ReplicationMaxQueuedCount,
+    ReplicationMaxDataTransferRate,
+    ReplicationRecentBacklogCount,
+
+    // Scanner-related metrics
+    ScannerBucketScansFinished,
+    ScannerBucketScansStarted,
+    ScannerDirectoriesScanned,
+    ScannerObjectsScanned,
+    ScannerVersionsScanned,
+    ScannerLastActivitySeconds,
+
+    // CPU system-related metrics
+    SysCPUAvgIdle,
+    SysCPUAvgIOWait,
+    SysCPULoad,
+    SysCPULoadPerc,
+    SysCPUNice,
+    SysCPUSteal,
+    SysCPUSystem,
+    SysCPUUser,
+
+    // Drive-related metrics
+    DriveUsedBytes,
+    DriveFreeBytes,
+    DriveTotalBytes,
+    DriveUsedInodes,
+    DriveFreeInodes,
+    DriveTotalInodes,
+    DriveTimeoutErrorsTotal,
+    DriveIOErrorsTotal,
+    DriveAvailabilityErrorsTotal,
+    DriveWaitingIO,
+    DriveAPILatencyMicros,
+    DriveHealth,
+
+    DriveOfflineCount,
+    DriveOnlineCount,
+    DriveCount,
+
+    // iostat related metrics
+    DriveReadsPerSec,
+    DriveReadsKBPerSec,
+    DriveReadsAwait,
+    DriveWritesPerSec,
+    DriveWritesKBPerSec,
+    DriveWritesAwait,
+    DrivePercUtil,
+
+    // Memory-related metrics
+    MemTotal,
+    MemUsed,
+    MemUsedPerc,
+    MemFree,
+    MemBuffers,
+    MemCache,
+    MemShared,
+    MemAvailable,
+
+    // Network-related metrics
+    InternodeErrorsTotal,
+    InternodeDialErrorsTotal,
+    InternodeDialAvgTimeNanos,
+    InternodeSentBytesTotal,
+    InternodeRecvBytesTotal,
+
+    // Process-related metrics
+    ProcessLocksReadTotal,
+    ProcessLocksWriteTotal,
+    ProcessCPUTotalSeconds,
+    ProcessGoRoutineTotal,
+    ProcessIORCharBytes,
+    ProcessIOReadBytes,
+    ProcessIOWCharBytes,
+    ProcessIOWriteBytes,
+    ProcessStartTimeSeconds,
+    ProcessUptimeSeconds,
+    ProcessFileDescriptorLimitTotal,
+    ProcessFileDescriptorOpenTotal,
+    ProcessSyscallReadTotal,
+    ProcessSyscallWriteTotal,
+    ProcessResidentMemoryBytes,
+    ProcessVirtualMemoryBytes,
+    ProcessVirtualMemoryMaxBytes,
+
+    // Custom metrics
+    Custom(String),
+}
+
+impl MetricName {
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> String {
+        match self {
+            Self::AuthTotal => "auth_total".to_string(),
+            Self::CanceledTotal => "canceled_total".to_string(),
+            Self::ErrorsTotal => "errors_total".to_string(),
+            Self::HeaderTotal => "header_total".to_string(),
+            Self::HealTotal => "heal_total".to_string(),
+            Self::HitsTotal => "hits_total".to_string(),
+            Self::InflightTotal => "inflight_total".to_string(),
+            Self::InvalidTotal => "invalid_total".to_string(),
+            Self::LimitTotal => "limit_total".to_string(),
+            Self::MissedTotal => "missed_total".to_string(),
+            Self::WaitingTotal => "waiting_total".to_string(),
+            Self::IncomingTotal => "incoming_total".to_string(),
+            Self::ObjectTotal => "object_total".to_string(),
+            Self::VersionTotal => "version_total".to_string(),
+            Self::DeleteMarkerTotal => "deletemarker_total".to_string(),
+            Self::OfflineTotal => "offline_total".to_string(),
+            Self::OnlineTotal => "online_total".to_string(),
+            Self::OpenTotal => "open_total".to_string(),
+            Self::ReadTotal => "read_total".to_string(),
+            Self::TimestampTotal => "timestamp_total".to_string(),
+            Self::WriteTotal => "write_total".to_string(),
+            Self::Total => "total".to_string(),
+            Self::FreeInodes => "free_inodes".to_string(),
+
+            Self::LastMinFailedCount => "last_minute_failed_count".to_string(),
+            Self::LastMinFailedBytes => "last_minute_failed_bytes".to_string(),
+            Self::LastHourFailedCount => "last_hour_failed_count".to_string(),
+            Self::LastHourFailedBytes => "last_hour_failed_bytes".to_string(),
+            Self::TotalFailedCount => "total_failed_count".to_string(),
+            Self::TotalFailedBytes => "total_failed_bytes".to_string(),
+
+            Self::CurrActiveWorkers => "current_active_workers".to_string(),
+            Self::AvgActiveWorkers => "average_active_workers".to_string(),
+            Self::MaxActiveWorkers => "max_active_workers".to_string(),
+            Self::RecentBacklogCount => "recent_backlog_count".to_string(),
+            Self::CurrInQueueCount => "last_minute_queued_count".to_string(),
+            Self::CurrInQueueBytes => "last_minute_queued_bytes".to_string(),
+            Self::ReceivedCount => "received_count".to_string(),
+            Self::SentCount => "sent_count".to_string(),
+            Self::CurrTransferRate => "current_transfer_rate".to_string(),
+            Self::AvgTransferRate => "average_transfer_rate".to_string(),
+            Self::MaxTransferRate => "max_transfer_rate".to_string(),
+            Self::CredentialErrors => "credential_errors".to_string(),
+
+            Self::CurrLinkLatency => "current_link_latency_ms".to_string(),
+            Self::AvgLinkLatency => "average_link_latency_ms".to_string(),
+            Self::MaxLinkLatency => "max_link_latency_ms".to_string(),
+
+            Self::LinkOnline => "link_online".to_string(),
+            Self::LinkOfflineDuration => "link_offline_duration_seconds".to_string(),
+            Self::LinkDowntimeTotalDuration => "link_downtime_duration_seconds".to_string(),
+
+            Self::AvgInQueueCount => "average_queued_count".to_string(),
+            Self::AvgInQueueBytes => "average_queued_bytes".to_string(),
+            Self::MaxInQueueCount => "max_queued_count".to_string(),
+            Self::MaxInQueueBytes => "max_queued_bytes".to_string(),
+
+            Self::ProxiedGetRequestsTotal => "proxied_get_requests_total".to_string(),
+            Self::ProxiedHeadRequestsTotal => "proxied_head_requests_total".to_string(),
+            Self::ProxiedPutTaggingRequestsTotal => "proxied_put_tagging_requests_total".to_string(),
+            Self::ProxiedGetTaggingRequestsTotal => "proxied_get_tagging_requests_total".to_string(),
+            Self::ProxiedDeleteTaggingRequestsTotal => "proxied_delete_tagging_requests_total".to_string(),
+            Self::ProxiedGetRequestsFailures => "proxied_get_requests_failures".to_string(),
+            Self::ProxiedHeadRequestsFailures => "proxied_head_requests_failures".to_string(),
+            Self::ProxiedPutTaggingRequestFailures => "proxied_put_tagging_requests_failures".to_string(),
+            Self::ProxiedGetTaggingRequestFailures => "proxied_get_tagging_requests_failures".to_string(),
+            Self::ProxiedDeleteTaggingRequestFailures => "proxied_delete_tagging_requests_failures".to_string(),
+
+            Self::FreeBytes => "free_bytes".to_string(),
+            Self::ReadBytes => "read_bytes".to_string(),
+            Self::RcharBytes => "rchar_bytes".to_string(),
+            Self::ReceivedBytes => "received_bytes".to_string(),
+            Self::LatencyMilliSec => "latency_ms".to_string(),
+            Self::SentBytes => "sent_bytes".to_string(),
+            Self::TotalBytes => "total_bytes".to_string(),
+            Self::UsedBytes => "used_bytes".to_string(),
+            Self::WriteBytes => "write_bytes".to_string(),
+            Self::WcharBytes => "wchar_bytes".to_string(),
+
+            Self::LatencyMicroSec => "latency_us".to_string(),
+            Self::LatencyNanoSec => "latency_ns".to_string(),
+
+            Self::CommitInfo => "commit_info".to_string(),
+            Self::UsageInfo => "usage_info".to_string(),
+            Self::VersionInfo => "version_info".to_string(),
+
+            Self::SizeDistribution => "size_distribution".to_string(),
+            Self::VersionDistribution => "version_distribution".to_string(),
+            Self::TtfbDistribution => "seconds_distribution".to_string(),
+            Self::TtlbDistribution => "ttlb_seconds_distribution".to_string(),
+
+            Self::LastActivityTime => "last_activity_nano_seconds".to_string(),
+            Self::StartTime => "starttime_seconds".to_string(),
+            Self::UpTime => "uptime_seconds".to_string(),
+            Self::Memory => "resident_memory_bytes".to_string(),
+            Self::Vmemory => "virtual_memory_bytes".to_string(),
+            Self::Cpu => "cpu_total_seconds".to_string(),
+
+            Self::ExpiryMissedTasks => "expiry_missed_tasks".to_string(),
+            Self::ExpiryMissedFreeVersions => "expiry_missed_freeversions".to_string(),
+            Self::ExpiryMissedTierJournalTasks => "expiry_missed_tierjournal_tasks".to_string(),
+            Self::ExpiryNumWorkers => "expiry_num_workers".to_string(),
+            Self::TransitionMissedTasks => "transition_missed_immediate_tasks".to_string(),
+
+            Self::TransitionedBytes => "transitioned_bytes".to_string(),
+            Self::TransitionedObjects => "transitioned_objects".to_string(),
+            Self::TransitionedVersions => "transitioned_versions".to_string(),
+
+            Self::TierRequestsSuccess => "requests_success".to_string(),
+            Self::TierRequestsFailure => "requests_failure".to_string(),
+
+            Self::KmsOnline => "online".to_string(),
+            Self::KmsRequestsSuccess => "request_success".to_string(),
+            Self::KmsRequestsError => "request_error".to_string(),
+            Self::KmsRequestsFail => "request_failure".to_string(),
+            Self::KmsUptime => "uptime".to_string(),
+
+            Self::WebhookOnline => "online".to_string(),
+
+            Self::ApiRejectedAuthTotal => "rejected_auth_total".to_string(),
+            Self::ApiRejectedHeaderTotal => "rejected_header_total".to_string(),
+            Self::ApiRejectedTimestampTotal => "rejected_timestamp_total".to_string(),
+            Self::ApiRejectedInvalidTotal => "rejected_invalid_total".to_string(),
+
+            Self::ApiRequestsWaitingTotal => "waiting_total".to_string(),
+            Self::ApiRequestsIncomingTotal => "incoming_total".to_string(),
+            Self::ApiRequestsInFlightTotal => "inflight_total".to_string(),
+            Self::ApiRequestsTotal => "total".to_string(),
+            Self::ApiRequestsErrorsTotal => "errors_total".to_string(),
+            Self::ApiRequests5xxErrorsTotal => "5xx_errors_total".to_string(),
+            Self::ApiRequests4xxErrorsTotal => "4xx_errors_total".to_string(),
+            Self::ApiRequestsCanceledTotal => "canceled_total".to_string(),
+
+            Self::ApiRequestsTTFBSecondsDistribution => "ttfb_seconds_distribution".to_string(),
+
+            Self::ApiTrafficSentBytes => "traffic_sent_bytes".to_string(),
+            Self::ApiTrafficRecvBytes => "traffic_received_bytes".to_string(),
+
+            Self::AuditFailedMessages => "failed_messages".to_string(),
+            Self::AuditTargetQueueLength => "target_queue_length".to_string(),
+            Self::AuditTotalMessages => "total_messages".to_string(),
+
+            // metrics related to cluster configurations
+            Self::ConfigRRSParity => "rrs_parity".to_string(),
+            Self::ConfigStandardParity => "standard_parity".to_string(),
+
+            // Erasure coding set related metrics
+            Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(),
+            Self::ErasureSetOverallHealth => "overall_health".to_string(),
+            Self::ErasureSetReadQuorum => "read_quorum".to_string(),
+            Self::ErasureSetWriteQuorum => "write_quorum".to_string(),
+            Self::ErasureSetOnlineDrivesCount => "online_drives_count".to_string(),
+            Self::ErasureSetHealingDrivesCount => "healing_drives_count".to_string(),
+            Self::ErasureSetHealth => "health".to_string(),
+            Self::ErasureSetReadTolerance => "read_tolerance".to_string(),
+            Self::ErasureSetWriteTolerance => "write_tolerance".to_string(),
+            Self::ErasureSetReadHealth => "read_health".to_string(),
+            Self::ErasureSetWriteHealth => "write_health".to_string(),
+
+            // Cluster health-related metrics
+            Self::HealthDrivesOfflineCount => "drives_offline_count".to_string(),
+            Self::HealthDrivesOnlineCount => "drives_online_count".to_string(),
+            Self::HealthDrivesCount => "drives_count".to_string(),
+
+            // IAM-related metrics
+            Self::LastSyncDurationMillis => "last_sync_duration_millis".to_string(),
+            Self::PluginAuthnServiceFailedRequestsMinute => "plugin_authn_service_failed_requests_minute".to_string(),
+            Self::PluginAuthnServiceLastFailSeconds => "plugin_authn_service_last_fail_seconds".to_string(),
+            Self::PluginAuthnServiceLastSuccSeconds => "plugin_authn_service_last_succ_seconds".to_string(),
+            Self::PluginAuthnServiceSuccAvgRttMsMinute => "plugin_authn_service_succ_avg_rtt_ms_minute".to_string(),
+            Self::PluginAuthnServiceSuccMaxRttMsMinute => "plugin_authn_service_succ_max_rtt_ms_minute".to_string(),
+            Self::PluginAuthnServiceTotalRequestsMinute => "plugin_authn_service_total_requests_minute".to_string(),
+            Self::SinceLastSyncMillis => "since_last_sync_millis".to_string(),
+            Self::SyncFailures => "sync_failures".to_string(),
+            Self::SyncSuccesses => "sync_successes".to_string(),
+
+            // Notify relevant metrics
+            Self::NotificationCurrentSendInProgress => "current_send_in_progress".to_string(),
+            Self::NotificationEventsErrorsTotal => "events_errors_total".to_string(),
+            Self::NotificationEventsSentTotal => "events_sent_total".to_string(),
+            Self::NotificationEventsSkippedTotal => "events_skipped_total".to_string(),
+
+            // Metrics related to the usage of cluster objects
+            Self::UsageSinceLastUpdateSeconds => "since_last_update_seconds".to_string(),
+            Self::UsageTotalBytes => "total_bytes".to_string(),
+            Self::UsageObjectsCount => "count".to_string(),
+            Self::UsageVersionsCount => "versions_count".to_string(),
+            Self::UsageDeleteMarkersCount => "delete_markers_count".to_string(),
+            Self::UsageBucketsCount => "buckets_count".to_string(),
+            Self::UsageSizeDistribution => "size_distribution".to_string(),
+            Self::UsageVersionCountDistribution => "version_count_distribution".to_string(),
+
+            // Metrics related to bucket usage
+            Self::UsageBucketQuotaTotalBytes => "quota_total_bytes".to_string(),
+            Self::UsageBucketTotalBytes => "total_bytes".to_string(),
+            Self::UsageBucketObjectsCount => "objects_count".to_string(),
+            Self::UsageBucketVersionsCount => "versions_count".to_string(),
+            Self::UsageBucketDeleteMarkersCount => "delete_markers_count".to_string(),
+            Self::UsageBucketObjectSizeDistribution => "object_size_distribution".to_string(),
+            Self::UsageBucketObjectVersionCountDistribution => "object_version_count_distribution".to_string(),
+
+            // ILM-related metrics
+            Self::IlmExpiryPendingTasks => "expiry_pending_tasks".to_string(),
+            Self::IlmTransitionActiveTasks => "transition_active_tasks".to_string(),
+            Self::IlmTransitionPendingTasks => "transition_pending_tasks".to_string(),
+            Self::IlmTransitionMissedImmediateTasks => "transition_missed_immediate_tasks".to_string(),
+            Self::IlmVersionsScanned => "versions_scanned".to_string(),
+
+            // Webhook logs
+            Self::WebhookQueueLength => "queue_length".to_string(),
+            Self::WebhookTotalMessages => "total_messages".to_string(),
+            Self::WebhookFailedMessages => "failed_messages".to_string(),
+
+            // Copy the relevant metrics
+            Self::ReplicationAverageActiveWorkers => "average_active_workers".to_string(),
+            Self::ReplicationAverageQueuedBytes => "average_queued_bytes".to_string(),
+            Self::ReplicationAverageQueuedCount => "average_queued_count".to_string(),
+            Self::ReplicationAverageDataTransferRate => "average_data_transfer_rate".to_string(),
+            Self::ReplicationCurrentActiveWorkers => "current_active_workers".to_string(),
+            Self::ReplicationCurrentDataTransferRate => "current_data_transfer_rate".to_string(),
+            Self::ReplicationLastMinuteQueuedBytes => "last_minute_queued_bytes".to_string(),
+            Self::ReplicationLastMinuteQueuedCount => "last_minute_queued_count".to_string(),
+            Self::ReplicationMaxActiveWorkers => "max_active_workers".to_string(),
+            Self::ReplicationMaxQueuedBytes => "max_queued_bytes".to_string(),
+            Self::ReplicationMaxQueuedCount => "max_queued_count".to_string(),
+            Self::ReplicationMaxDataTransferRate => "max_data_transfer_rate".to_string(),
+            Self::ReplicationRecentBacklogCount => "recent_backlog_count".to_string(),
+
+            // Scanner-related metrics
+            Self::ScannerBucketScansFinished => "bucket_scans_finished".to_string(),
+            Self::ScannerBucketScansStarted => "bucket_scans_started".to_string(),
+            Self::ScannerDirectoriesScanned => "directories_scanned".to_string(),
+            Self::ScannerObjectsScanned => "objects_scanned".to_string(),
+            Self::ScannerVersionsScanned => "versions_scanned".to_string(),
+            Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
+
+            // CPU system-related metrics
+            Self::SysCPUAvgIdle => "avg_idle".to_string(),
+            Self::SysCPUAvgIOWait => "avg_iowait".to_string(),
+            Self::SysCPULoad => "load".to_string(),
+            Self::SysCPULoadPerc => "load_perc".to_string(),
+            Self::SysCPUNice => "nice".to_string(),
+            Self::SysCPUSteal => "steal".to_string(),
+            Self::SysCPUSystem => "system".to_string(),
+            Self::SysCPUUser => "user".to_string(),
+
+            // Drive-related metrics
+            Self::DriveUsedBytes => "used_bytes".to_string(),
+            Self::DriveFreeBytes => "free_bytes".to_string(),
+            Self::DriveTotalBytes => "total_bytes".to_string(),
+            Self::DriveUsedInodes => "used_inodes".to_string(),
+            Self::DriveFreeInodes => "free_inodes".to_string(),
+            Self::DriveTotalInodes => "total_inodes".to_string(),
+            Self::DriveTimeoutErrorsTotal => "timeout_errors_total".to_string(),
+            Self::DriveIOErrorsTotal => "io_errors_total".to_string(),
+            Self::DriveAvailabilityErrorsTotal => "availability_errors_total".to_string(),
+            Self::DriveWaitingIO => "waiting_io".to_string(),
+            Self::DriveAPILatencyMicros => "api_latency_micros".to_string(),
+            Self::DriveHealth => "health".to_string(),
+
+            Self::DriveOfflineCount => "offline_count".to_string(),
+            Self::DriveOnlineCount => "online_count".to_string(),
+            Self::DriveCount => "count".to_string(),
+
+            // iostat related metrics
+            Self::DriveReadsPerSec => "reads_per_sec".to_string(),
+            Self::DriveReadsKBPerSec => "reads_kb_per_sec".to_string(),
+            Self::DriveReadsAwait => "reads_await".to_string(),
+            Self::DriveWritesPerSec => "writes_per_sec".to_string(),
+            Self::DriveWritesKBPerSec => "writes_kb_per_sec".to_string(),
+            Self::DriveWritesAwait => "writes_await".to_string(),
+            Self::DrivePercUtil => "perc_util".to_string(),
+
+            // Memory-related metrics
+            Self::MemTotal => "total".to_string(),
+            Self::MemUsed => "used".to_string(),
+            Self::MemUsedPerc => "used_perc".to_string(),
+            Self::MemFree => "free".to_string(),
+            Self::MemBuffers => "buffers".to_string(),
+            Self::MemCache => "cache".to_string(),
+            Self::MemShared => "shared".to_string(),
+            Self::MemAvailable => "available".to_string(),
+
+            // Network-related metrics
+            Self::InternodeErrorsTotal => "errors_total".to_string(),
+            Self::InternodeDialErrorsTotal => "dial_errors_total".to_string(),
+            Self::InternodeDialAvgTimeNanos => "dial_avg_time_nanos".to_string(),
+            Self::InternodeSentBytesTotal => "sent_bytes_total".to_string(),
+            Self::InternodeRecvBytesTotal => "recv_bytes_total".to_string(),
+
+            // Process-related metrics
+            Self::ProcessLocksReadTotal => "locks_read_total".to_string(),
+            Self::ProcessLocksWriteTotal => "locks_write_total".to_string(),
+            Self::ProcessCPUTotalSeconds => "cpu_total_seconds".to_string(),
+            Self::ProcessGoRoutineTotal => "go_routine_total".to_string(),
+            Self::ProcessIORCharBytes => "io_rchar_bytes".to_string(),
+            Self::ProcessIOReadBytes => "io_read_bytes".to_string(),
+            Self::ProcessIOWCharBytes => "io_wchar_bytes".to_string(),
+            Self::ProcessIOWriteBytes => "io_write_bytes".to_string(),
+            Self::ProcessStartTimeSeconds => "start_time_seconds".to_string(),
+            Self::ProcessUptimeSeconds => "uptime_seconds".to_string(),
+            Self::ProcessFileDescriptorLimitTotal => "file_descriptor_limit_total".to_string(),
+            Self::ProcessFileDescriptorOpenTotal => "file_descriptor_open_total".to_string(),
+            Self::ProcessSyscallReadTotal => "syscall_read_total".to_string(),
+            Self::ProcessSyscallWriteTotal => "syscall_write_total".to_string(),
+            Self::ProcessResidentMemoryBytes => "resident_memory_bytes".to_string(),
+            Self::ProcessVirtualMemoryBytes => "virtual_memory_bytes".to_string(),
+            Self::ProcessVirtualMemoryMaxBytes => "virtual_memory_max_bytes".to_string(),
+
+            Self::Custom(name) => name.clone(),
+        }
+    }
+}
+
+impl From<String> for MetricName {
+    fn from(s: String) -> Self {
+        Self::Custom(s)
+    }
+}
+
+impl From<&str> for MetricName {
+    fn from(s: &str) -> Self {
+        Self::Custom(s.to_string())
+    }
+}

crates/metrics/src/metrics_type/entry/metric_type.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// MetricType - Indicates the type of indicator
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MetricType {
+    Counter,
+    Gauge,
+    Histogram,
+}
+
+impl MetricType {
+    /// convert the metric type to a string representation
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Counter => "counter",
+            Self::Gauge => "gauge",
+            Self::Histogram => "histogram",
+        }
+    }
+
+    /// Convert the metric type to the Prometheus value type
+    /// In a Rust implementation, this might return the corresponding Prometheus Rust client type
+    #[allow(dead_code)]
+    pub fn as_prom(&self) -> &'static str {
+        match self {
+            Self::Counter => "counter.",
+            Self::Gauge => "gauge.",
+            Self::Histogram => "histogram.", // Histograms still use the counter value in Prometheus
+        }
+    }
+}

crates/metrics/src/metrics_type/entry/mod.rs

@@ -0,0 +1,129 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{MetricDescriptor, MetricName, MetricNamespace, MetricSubsystem, MetricType};
+
+pub mod descriptor;
+pub mod metric_name;
+pub mod metric_type;
+pub mod namespace;
+mod path_utils;
+pub mod subsystem;
+
+/// Create a new counter metric descriptor
+pub fn new_counter_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Counter,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+/// create a new dashboard metric descriptor
+pub fn new_gauge_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Gauge,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+/// create a new histogram indicator descriptor
+#[allow(dead_code)]
+pub fn new_histogram_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Histogram,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{MetricName, MetricNamespace, MetricSubsystem, subsystems};
+
+    #[test]
+    fn test_new_histogram_md() {
+        // create a histogram indicator descriptor
+        let histogram_md = new_histogram_md(
+            MetricName::TtfbDistribution,
+            "test the response time distribution",
+            &["api", "method", "le"],
+            subsystems::API_REQUESTS,
+        );
+
+        // verify that the metric type is correct
+        assert_eq!(histogram_md.metric_type, MetricType::Histogram);
+
+        // verify that the metric name is correct
+        assert_eq!(histogram_md.name.as_str(), "seconds_distribution");
+
+        // verify that the help information is correct
+        assert_eq!(histogram_md.help, "test the response time distribution");
+
+        // Verify that the label is correct
+        assert_eq!(histogram_md.variable_labels.len(), 3);
+        assert!(histogram_md.variable_labels.contains(&"api".to_string()));
+        assert!(histogram_md.variable_labels.contains(&"method".to_string()));
+        assert!(histogram_md.variable_labels.contains(&"le".to_string()));
+
+        // Verify that the namespace is correct
+        assert_eq!(histogram_md.namespace, MetricNamespace::RustFS);
+
+        // Verify that the subsystem is correct
+        assert_eq!(histogram_md.subsystem, MetricSubsystem::ApiRequests);
+
+        // Verify that the full metric name generated is formatted correctly
+        assert_eq!(histogram_md.get_full_metric_name(), "histogram.rustfs_api_requests_seconds_distribution");
+
+        // Tests use custom subsystems
+        let custom_histogram_md = new_histogram_md(
+            "custom_latency_distribution",
+            "custom latency distribution",
+            &["endpoint", "le"],
+            MetricSubsystem::new("/custom/path-metrics"),
+        );
+
+        // Verify the custom name and subsystem
+        assert_eq!(
+            custom_histogram_md.get_full_metric_name(),
+            "histogram.rustfs_custom_path_metrics_custom_latency_distribution"
+        );
+    }
+}

crates/metrics/src/metrics_type/entry/namespace.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// The metric namespace, which represents the top-level grouping of the metric
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MetricNamespace {
+    RustFS,
+}
+
+impl MetricNamespace {
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::RustFS => "rustfs",
+        }
+    }
+}

crates/metrics/src/metrics_type/entry/path_utils.rs

@@ -0,0 +1,33 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Format the path to the metric name format
+/// Replace '/' and '-' with '_'
+#[allow(dead_code)]
+pub fn format_path_to_metric_name(path: &str) -> String {
+    path.trim_start_matches('/').replace(['/', '-'], "_")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_path_to_metric_name() {
+        assert_eq!(format_path_to_metric_name("/api/requests"), "api_requests");
+        assert_eq!(format_path_to_metric_name("/system/network/internode"), "system_network_internode");
+        assert_eq!(format_path_to_metric_name("/bucket-api"), "bucket_api");
+        assert_eq!(format_path_to_metric_name("cluster/health"), "cluster_health");
+    }
+}

crates/metrics/src/metrics_type/entry/subsystem.rs

@@ -0,0 +1,245 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::entry::path_utils::format_path_to_metric_name;
+
+/// The metrics subsystem is a subgroup of metrics within a namespace
+/// The metrics subsystem, which represents a subgroup of metrics within a namespace
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum MetricSubsystem {
+    // API related subsystems
+    ApiRequests,
+
+    // bucket related subsystems
+    BucketApi,
+    BucketReplication,
+
+    // system related subsystems
+    SystemNetworkInternode,
+    SystemDrive,
+    SystemMemory,
+    SystemCpu,
+    SystemProcess,
+
+    // debug related subsystems
+    DebugGo,
+
+    // cluster related subsystems
+    ClusterHealth,
+    ClusterUsageObjects,
+    ClusterUsageBuckets,
+    ClusterErasureSet,
+    ClusterIam,
+    ClusterConfig,
+
+    // other service related subsystems
+    Ilm,
+    Audit,
+    LoggerWebhook,
+    Replication,
+    Notification,
+    Scanner,
+
+    // Custom paths
+    Custom(String),
+}
+
+impl MetricSubsystem {
+    /// Gets the original path string
+    pub fn path(&self) -> &str {
+        match self {
+            // api related subsystems
+            Self::ApiRequests => "/api/requests",
+
+            // bucket related subsystems
+            Self::BucketApi => "/bucket/api",
+            Self::BucketReplication => "/bucket/replication",
+
+            // system related subsystems
+            Self::SystemNetworkInternode => "/system/network/internode",
+            Self::SystemDrive => "/system/drive",
+            Self::SystemMemory => "/system/memory",
+            Self::SystemCpu => "/system/cpu",
+            Self::SystemProcess => "/system/process",
+
+            // debug related subsystems
+            Self::DebugGo => "/debug/go",
+
+            // cluster related subsystems
+            Self::ClusterHealth => "/cluster/health",
+            Self::ClusterUsageObjects => "/cluster/usage/objects",
+            Self::ClusterUsageBuckets => "/cluster/usage/buckets",
+            Self::ClusterErasureSet => "/cluster/erasure-set",
+            Self::ClusterIam => "/cluster/iam",
+            Self::ClusterConfig => "/cluster/config",
+
+            // other service related subsystems
+            Self::Ilm => "/ilm",
+            Self::Audit => "/audit",
+            Self::LoggerWebhook => "/logger/webhook",
+            Self::Replication => "/replication",
+            Self::Notification => "/notification",
+            Self::Scanner => "/scanner",
+
+            // Custom paths
+            Self::Custom(path) => path,
+        }
+    }
+
+    /// Get the formatted metric name format string
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> String {
+        format_path_to_metric_name(self.path())
+    }
+
+    /// Create a subsystem enumeration from a path string
+    pub fn from_path(path: &str) -> Self {
+        match path {
+            // API-related subsystems
+            "/api/requests" => Self::ApiRequests,
+
+            // Bucket-related subsystems
+            "/bucket/api" => Self::BucketApi,
+            "/bucket/replication" => Self::BucketReplication,
+
+            // System-related subsystems
+            "/system/network/internode" => Self::SystemNetworkInternode,
+            "/system/drive" => Self::SystemDrive,
+            "/system/memory" => Self::SystemMemory,
+            "/system/cpu" => Self::SystemCpu,
+            "/system/process" => Self::SystemProcess,
+
+            // Debug related subsystems
+            "/debug/go" => Self::DebugGo,
+
+            // Cluster-related subsystems
+            "/cluster/health" => Self::ClusterHealth,
+            "/cluster/usage/objects" => Self::ClusterUsageObjects,
+            "/cluster/usage/buckets" => Self::ClusterUsageBuckets,
+            "/cluster/erasure-set" => Self::ClusterErasureSet,
+            "/cluster/iam" => Self::ClusterIam,
+            "/cluster/config" => Self::ClusterConfig,
+
+            // Other service-related subsystems
+            "/ilm" => Self::Ilm,
+            "/audit" => Self::Audit,
+            "/logger/webhook" => Self::LoggerWebhook,
+            "/replication" => Self::Replication,
+            "/notification" => Self::Notification,
+            "/scanner" => Self::Scanner,
+
+            // Treat other paths as custom subsystems
+            _ => Self::Custom(path.to_string()),
+        }
+    }
+
+    /// A convenient way to create custom subsystems directly
+    #[allow(dead_code)]
+    pub fn new(path: impl Into<String>) -> Self {
+        Self::Custom(path.into())
+    }
+}
+
+/// Implementations that facilitate conversion to and from strings
+impl From<&str> for MetricSubsystem {
+    fn from(s: &str) -> Self {
+        Self::from_path(s)
+    }
+}
+
+impl From<String> for MetricSubsystem {
+    fn from(s: String) -> Self {
+        Self::from_path(&s)
+    }
+}
+
+impl std::fmt::Display for MetricSubsystem {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.path())
+    }
+}
+
+#[allow(dead_code)]
+pub mod subsystems {
+    use super::MetricSubsystem;
+
+    // cluster base path constant
+    pub const CLUSTER_BASE_PATH: &str = "/cluster";
+
+    // Quick access to constants for each subsystem
+    pub const API_REQUESTS: MetricSubsystem = MetricSubsystem::ApiRequests;
+    pub const BUCKET_API: MetricSubsystem = MetricSubsystem::BucketApi;
+    pub const BUCKET_REPLICATION: MetricSubsystem = MetricSubsystem::BucketReplication;
+    pub const SYSTEM_NETWORK_INTERNODE: MetricSubsystem = MetricSubsystem::SystemNetworkInternode;
+    pub const SYSTEM_DRIVE: MetricSubsystem = MetricSubsystem::SystemDrive;
+    pub const SYSTEM_MEMORY: MetricSubsystem = MetricSubsystem::SystemMemory;
+    pub const SYSTEM_CPU: MetricSubsystem = MetricSubsystem::SystemCpu;
+    pub const SYSTEM_PROCESS: MetricSubsystem = MetricSubsystem::SystemProcess;
+    pub const DEBUG_GO: MetricSubsystem = MetricSubsystem::DebugGo;
+    pub const CLUSTER_HEALTH: MetricSubsystem = MetricSubsystem::ClusterHealth;
+    pub const CLUSTER_USAGE_OBJECTS: MetricSubsystem = MetricSubsystem::ClusterUsageObjects;
+    pub const CLUSTER_USAGE_BUCKETS: MetricSubsystem = MetricSubsystem::ClusterUsageBuckets;
+    pub const CLUSTER_ERASURE_SET: MetricSubsystem = MetricSubsystem::ClusterErasureSet;
+    pub const CLUSTER_IAM: MetricSubsystem = MetricSubsystem::ClusterIam;
+    pub const CLUSTER_CONFIG: MetricSubsystem = MetricSubsystem::ClusterConfig;
+    pub const ILM: MetricSubsystem = MetricSubsystem::Ilm;
+    pub const AUDIT: MetricSubsystem = MetricSubsystem::Audit;
+    pub const LOGGER_WEBHOOK: MetricSubsystem = MetricSubsystem::LoggerWebhook;
+    pub const REPLICATION: MetricSubsystem = MetricSubsystem::Replication;
+    pub const NOTIFICATION: MetricSubsystem = MetricSubsystem::Notification;
+    pub const SCANNER: MetricSubsystem = MetricSubsystem::Scanner;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{MetricDescriptor, MetricName, MetricNamespace, MetricType};
+
+    #[test]
+    fn test_metric_subsystem_formatting() {
+        assert_eq!(MetricSubsystem::ApiRequests.as_str(), "api_requests");
+        assert_eq!(MetricSubsystem::SystemNetworkInternode.as_str(), "system_network_internode");
+        assert_eq!(MetricSubsystem::BucketApi.as_str(), "bucket_api");
+        assert_eq!(MetricSubsystem::ClusterHealth.as_str(), "cluster_health");
+
+        // Test custom paths
+        let custom = MetricSubsystem::new("/custom/path-test");
+        assert_eq!(custom.as_str(), "custom_path_test");
+    }
+
+    #[test]
+    fn test_metric_descriptor_name_generation() {
+        let md = MetricDescriptor::new(
+            MetricName::ApiRequestsTotal,
+            MetricType::Counter,
+            "Test help".to_string(),
+            vec!["label1".to_string(), "label2".to_string()],
+            MetricNamespace::RustFS,
+            MetricSubsystem::ApiRequests,
+        );
+
+        assert_eq!(md.get_full_metric_name(), "counter.rustfs_api_requests_total");
+
+        let custom_md = MetricDescriptor::new(
+            MetricName::Custom("test_metric".to_string()),
+            MetricType::Gauge,
+            "Test help".to_string(),
+            vec!["label1".to_string()],
+            MetricNamespace::RustFS,
+            MetricSubsystem::new("/custom/path-with-dash"),
+        );
+
+        assert_eq!(custom_md.get_full_metric_name(), "gauge.rustfs_custom_path_with_dash_test_metric");
+    }
+}

crates/metrics/src/metrics_type/ilm.rs

@@ -0,0 +1,63 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmExpiryPendingTasks,
+        "Number of pending ILM expiry tasks in the queue",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_ACTIVE_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmTransitionActiveTasks,
+        "Number of active ILM transition tasks",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmTransitionPendingTasks,
+        "Number of pending ILM transition tasks in the queue",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmTransitionMissedImmediateTasks,
+        "Number of missed immediate ILM transition tasks",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmVersionsScanned,
+        "Total number of object versions checked for ILM actions since server start",
+        &[],
+        subsystems::ILM,
+    )
+});

crates/metrics/src/metrics_type/logger_webhook.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Define label constants for webhook metrics
+/// name label
+pub const NAME_LABEL: &str = "name";
+/// endpoint label
+pub const ENDPOINT_LABEL: &str = "endpoint";
+
+// The label used by all webhook metrics
+const ALL_WEBHOOK_LABELS: [&str; 2] = [NAME_LABEL, ENDPOINT_LABEL];
+
+pub static WEBHOOK_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::WebhookFailedMessages,
+        "Number of messages that failed to send",
+        &ALL_WEBHOOK_LABELS[..],
+        subsystems::LOGGER_WEBHOOK,
+    )
+});
+
+pub static WEBHOOK_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::WebhookQueueLength,
+        "Webhook queue length",
+        &ALL_WEBHOOK_LABELS[..],
+        subsystems::LOGGER_WEBHOOK,
+    )
+});
+
+pub static WEBHOOK_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::WebhookTotalMessages,
+        "Total number of messages sent to this target",
+        &ALL_WEBHOOK_LABELS[..],
+        subsystems::LOGGER_WEBHOOK,
+    )
+});

crates/metrics/src/metrics_type/mod.rs

@@ -0,0 +1,42 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod audit;
+pub mod bucket;
+pub mod bucket_replication;
+pub mod cluster_config;
+pub mod cluster_erasure_set;
+pub mod cluster_health;
+pub mod cluster_iam;
+pub mod cluster_notification;
+pub mod cluster_usage;
+pub mod entry;
+pub mod ilm;
+pub mod logger_webhook;
+pub mod replication;
+pub mod request;
+pub mod scanner;
+pub mod system_cpu;
+pub mod system_drive;
+pub mod system_memory;
+pub mod system_network;
+pub mod system_process;
+
+pub use entry::descriptor::MetricDescriptor;
+pub use entry::metric_name::MetricName;
+pub use entry::metric_type::MetricType;
+pub use entry::namespace::MetricNamespace;
+pub use entry::subsystem::MetricSubsystem;
+pub use entry::subsystem::subsystems;
+pub use entry::{new_counter_md, new_gauge_md, new_histogram_md};

crates/metrics/src/metrics_type/replication.rs

@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageActiveWorkers,
+        "Average number of active replication workers",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageQueuedBytes,
+        "Average number of bytes queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageQueuedCount,
+        "Average number of objects queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageDataTransferRate,
+        "Average replication data transfer rate in bytes/sec",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationCurrentActiveWorkers,
+        "Total number of active replication workers",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationCurrentDataTransferRate,
+        "Current replication data transfer rate in bytes/sec",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationLastMinuteQueuedBytes,
+        "Number of bytes queued for replication in the last full minute",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationLastMinuteQueuedCount,
+        "Number of objects queued for replication in the last full minute",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxActiveWorkers,
+        "Maximum number of active replication workers seen since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxQueuedBytes,
+        "Maximum number of bytes queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxQueuedCount,
+        "Maximum number of objects queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxDataTransferRate,
+        "Maximum replication data transfer rate in bytes/sec seen since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationRecentBacklogCount,
+        "Total number of objects seen in replication backlog in the last 5 minutes",
+        &[],
+        subsystems::REPLICATION,
+    )
+});

crates/metrics/src/metrics_type/request.rs

@@ -0,0 +1,153 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedAuthTotal,
+        "Total number of requests rejected for auth failure",
+        &["type"],
+        subsystems::API_REQUESTS,
+    )
+});
+
+pub static API_REJECTED_HEADER_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedHeaderTotal,
+        "Total number of requests rejected for invalid header",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REJECTED_TIMESTAMP_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedTimestampTotal,
+        "Total number of requests rejected for invalid timestamp",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REJECTED_INVALID_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedInvalidTotal,
+        "Total number of invalid requests",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_WAITING_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsWaitingTotal,
+        "Total number of requests in the waiting queue",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_INCOMING_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsIncomingTotal,
+        "Total number of incoming requests",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsInFlightTotal,
+        "Total number of requests currently in flight",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTotal,
+        "Total number of requests",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsErrorsTotal,
+        "Total number of requests with (4xx and 5xx) errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests5xxErrorsTotal,
+        "Total number of requests with 5xx errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests4xxErrorsTotal,
+        "Total number of requests with 4xx errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsCanceledTotal,
+        "Total number of requests canceled by the client",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTTFBSecondsDistribution,
+        "Distribution of time to first byte across API calls",
+        &["name", "type", "le"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficSentBytes,
+        "Total number of bytes sent",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficRecvBytes,
+        "Total number of bytes received",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});

crates/metrics/src/metrics_type/scanner.rs

@@ -0,0 +1,72 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerBucketScansFinished,
+        "Total number of bucket scans finished since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerBucketScansStarted,
+        "Total number of bucket scans started since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_DIRECTORIES_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerDirectoriesScanned,
+        "Total number of directories scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_OBJECTS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerObjectsScanned,
+        "Total number of unique objects scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerVersionsScanned,
+        "Total number of object versions scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_ACTIVITY_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastActivitySeconds,
+        "Time elapsed (in seconds) since last scan activity.",
+        &[],
+        subsystems::SCANNER,
+    )
+});

crates/metrics/src/metrics_type/system_cpu.rs

@@ -0,0 +1,49 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+/// CPU system-related metric descriptors
+use std::sync::LazyLock;
+
+pub static SYS_CPU_AVG_IDLE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIdle, "Average CPU idle time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIOWait, "Average CPU IOWait time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_LOAD_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_LOAD_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::SysCPULoadPerc,
+        "CPU load average 1min (percentage)",
+        &[],
+        subsystems::SYSTEM_CPU,
+    )
+});
+
+pub static SYS_CPU_NICE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_STEAL_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_SYSTEM_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_USER_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[], subsystems::SYSTEM_CPU));

crates/metrics/src/metrics_type/system_drive.rs

@@ -0,0 +1,212 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// drive related labels
+pub const DRIVE_LABEL: &str = "drive";
+/// pool index label
+pub const POOL_INDEX_LABEL: &str = "pool_index";
+/// set index label
+pub const SET_INDEX_LABEL: &str = "set_index";
+/// drive index label
+pub const DRIVE_INDEX_LABEL: &str = "drive_index";
+/// API label
+pub const API_LABEL: &str = "api";
+
+/// All drive-related labels
+pub const ALL_DRIVE_LABELS: [&str; 4] = [DRIVE_LABEL, POOL_INDEX_LABEL, SET_INDEX_LABEL, DRIVE_INDEX_LABEL];
+
+pub static DRIVE_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveUsedBytes,
+        "Total storage used on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveFreeBytes,
+        "Total storage free on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveTotalBytes,
+        "Total storage available on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_USED_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveUsedInodes,
+        "Total used inodes on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_FREE_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveFreeInodes,
+        "Total free inodes on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TOTAL_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveTotalInodes,
+        "Total inodes available on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TIMEOUT_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveTimeoutErrorsTotal,
+        "Total timeout errors on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_IO_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveIOErrorsTotal,
+        "Total I/O errors on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_AVAILABILITY_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveAvailabilityErrorsTotal,
+        "Total availability errors (I/O errors, timeouts) on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WAITING_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWaitingIO,
+        "Total waiting I/O operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_API_LATENCY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveAPILatencyMicros,
+        "Average last minute latency in µs for drive API storage operations",
+        &[&ALL_DRIVE_LABELS[..], &[API_LABEL]].concat(),
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveHealth,
+        "Drive health (0 = offline, 1 = healthy, 2 = healing)",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveOfflineCount, "Count of offline drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_ONLINE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveOnlineCount, "Count of online drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveCount, "Count of all drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_READS_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsPerSec,
+        "Reads per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_READS_KB_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsKBPerSec,
+        "Kilobytes read per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_READS_AWAIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsAwait,
+        "Average time for read requests served on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesPerSec,
+        "Writes per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_KB_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesKBPerSec,
+        "Kilobytes written per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_AWAIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesAwait,
+        "Average time for write requests served on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_PERC_UTIL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DrivePercUtil,
+        "Percentage of time the disk was busy",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});

crates/metrics/src/metrics_type/system_memory.rs

@@ -0,0 +1,56 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Total memory available on the node
+pub static MEM_TOTAL_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemTotal, "Total memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory currently in use on the node
+pub static MEM_USED_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Percentage of total memory currently in use
+pub static MEM_USED_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::MemUsedPerc,
+        "Used memory percentage on the node",
+        &[],
+        subsystems::SYSTEM_MEMORY,
+    )
+});
+
+/// Memory not currently in use and available for allocation
+pub static MEM_FREE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory used for file buffers by the kernel
+pub static MEM_BUFFERS_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemBuffers, "Buffers memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory used for caching file data by the kernel
+pub static MEM_CACHE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemCache, "Cache memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory shared between multiple processes
+pub static MEM_SHARED_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemShared, "Shared memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Estimate of memory available for new applications without swapping
+pub static MEM_AVAILABLE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemAvailable, "Available memory on the node", &[], subsystems::SYSTEM_MEMORY));

crates/metrics/src/metrics_type/system_network.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Total number of failed internode calls counter
+pub static INTERNODE_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeErrorsTotal,
+        "Total number of failed internode calls",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// TCP dial timeouts and errors counter
+pub static INTERNODE_DIAL_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeDialErrorsTotal,
+        "Total number of internode TCP dial timeouts and errors",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Average dial time gauge in nanoseconds
+pub static INTERNODE_DIAL_AVG_TIME_NANOS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::InternodeDialAvgTimeNanos,
+        "Average dial time of internode TCP calls in nanoseconds",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Outbound network traffic counter in bytes
+pub static INTERNODE_SENT_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeSentBytesTotal,
+        "Total number of bytes sent to other peer nodes",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Inbound network traffic counter in bytes
+pub static INTERNODE_RECV_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeRecvBytesTotal,
+        "Total number of bytes received from other peer nodes",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});

crates/metrics/src/metrics_type/system_process.rs

@@ -0,0 +1,188 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Number of current READ locks on this peer
+pub static PROCESS_LOCKS_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessLocksReadTotal,
+        "Number of current READ locks on this peer",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Number of current WRITE locks on this peer
+pub static PROCESS_LOCKS_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessLocksWriteTotal,
+        "Number of current WRITE locks on this peer",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total user and system CPU time spent in seconds
+pub static PROCESS_CPU_TOTAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessCPUTotalSeconds,
+        "Total user and system CPU time spent in seconds",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total number of go routines running
+pub static PROCESS_GO_ROUTINE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessGoRoutineTotal,
+        "Total number of go routines running",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes read by the process from the underlying storage system including cache
+pub static PROCESS_IO_RCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIORCharBytes,
+        "Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes read by the process from the underlying storage system
+pub static PROCESS_IO_READ_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOReadBytes,
+        "Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes written by the process to the underlying storage system including page cache
+pub static PROCESS_IO_WCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOWCharBytes,
+        "Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes written by the process to the underlying storage system
+pub static PROCESS_IO_WRITE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOWriteBytes,
+        "Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Start time for RustFS process in seconds since Unix epoch
+pub static PROCESS_START_TIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessStartTimeSeconds,
+        "Start time for RustFS process in seconds since Unix epoch",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Uptime for RustFS process in seconds
+pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessUptimeSeconds,
+        "Uptime for RustFS process in seconds",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Limit on total number of open file descriptors for the RustFS Server process
+pub static PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessFileDescriptorLimitTotal,
+        "Limit on total number of open file descriptors for the RustFS Server process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total number of open file descriptors by the RustFS Server process
+pub static PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessFileDescriptorOpenTotal,
+        "Total number of open file descriptors by the RustFS Server process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total read SysCalls to the kernel
+pub static PROCESS_SYSCALL_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessSyscallReadTotal,
+        "Total read SysCalls to the kernel. /proc/[pid]/io syscr",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total write SysCalls to the kernel
+pub static PROCESS_SYSCALL_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessSyscallWriteTotal,
+        "Total write SysCalls to the kernel. /proc/[pid]/io syscw",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Resident memory size in bytes
+pub static PROCESS_RESIDENT_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessResidentMemoryBytes,
+        "Resident memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Virtual memory size in bytes
+pub static PROCESS_VIRTUAL_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessVirtualMemoryBytes,
+        "Virtual memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Maximum virtual memory size in bytes
+pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessVirtualMemoryMaxBytes,
+        "Maximum virtual memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});