refactor(obs): migrate metrics runtime/schema and tighten migration guards (#2584)

crates/obs/src/global.rs

@@ -116,8 +116,7 @@ pub async fn init_obs(endpoint: Option<String>) -> Result<OtelGuard, GlobalError
 /// ```
 pub async fn init_obs_with_config(config: &OtelConfig) -> Result<OtelGuard, GlobalError> {
     let otel_guard = init_telemetry(config)?;
-    // Note: System monitoring has been migrated to rustfs-metrics
-    // Use rustfs_metrics::init_metrics_collectors() for system metrics
+    // Metrics runtime scheduling is exposed by rustfs_obs::init_metrics_runtime().
     Ok(otel_guard)
 }
 

crates/obs/src/lib.rs

@@ -47,28 +47,30 @@
 //! # }
 //! ```
 //!
-//! ## System Monitoring Migration
+//! ## Metrics Runtime
 //!
-//! The system monitoring functionality has been migrated to `rustfs-metrics`.
-//! Use `rustfs_metrics::init_metrics_collectors()` for system metrics collection.
+//! Start metrics scheduling with `rustfs_obs::init_metrics_runtime()`.
 //!
 //! ```ignore
 //! use tokio_util::sync::CancellationToken;
-//! use rustfs_metrics::init_metrics_collectors;
+//! use rustfs_obs::init_metrics_runtime;
 //!
 //! let token = CancellationToken::new();
-//! init_metrics_collectors(token.clone());
+//! init_metrics_runtime(token.clone());
 //! ```
 mod cleaner;
 mod config;
 mod error;
 mod global;
+pub mod metrics;
 mod telemetry;
 
 pub use cleaner::*;
 pub use config::*;
 pub use error::*;
 pub use global::*;
+pub use metrics::schema::*;
+pub use metrics::{init_metrics_collectors, init_metrics_runtime};
 pub use telemetry::{OtelGuard, Recorder};
 
 // Dial9 Tokio runtime telemetry

crates/obs/src/metrics/collectors/audit.rs

@@ -0,0 +1,109 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Audit metrics collector.
+//!
+//! Collects audit log metrics including failed messages, queue length,
+//! and total messages per target.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::audit`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::audit::*;
+use std::borrow::Cow;
+
+/// Audit target statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct AuditTargetStats {
+    /// Target identifier
+    pub target_id: String,
+    /// Number of messages that failed to send
+    pub failed_messages: u64,
+    /// Number of unsent messages in queue
+    pub queue_length: u64,
+    /// Total number of messages sent
+    pub total_messages: u64,
+}
+
+/// Collects audit metrics from the provided audit target statistics.
+///
+/// Uses the metric descriptors from `metrics_type::audit` module.
+/// Returns a vector of Prometheus metrics for audit statistics.
+pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
+    for stat in stats {
+        let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
+                .with_label("target_id", target_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
+                .with_label("target_id", target_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
+                .with_label("target_id", target_id_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_audit_metrics() {
+        let stats = vec![
+            AuditTargetStats {
+                target_id: "target-1".to_string(),
+                failed_messages: 5,
+                queue_length: 10,
+                total_messages: 1000,
+            },
+            AuditTargetStats {
+                target_id: "target-2".to_string(),
+                failed_messages: 2,
+                queue_length: 5,
+                total_messages: 500,
+            },
+        ];
+
+        let metrics = collect_audit_metrics(&stats);
+
+        assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each
+
+        let failed = metrics
+            .iter()
+            .find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1"));
+        assert!(failed.is_some());
+    }
+
+    #[test]
+    fn test_collect_audit_metrics_empty() {
+        let stats: Vec<AuditTargetStats> = vec![];
+        let metrics = collect_audit_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/obs/src/metrics/collectors/bucket.rs

@@ -0,0 +1,176 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket metrics collector.
+//!
+//! Collects usage metrics for each bucket in the cluster, including
+//! size, object counts, and quota information.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::node_bucket`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::node_bucket::*;
+use std::borrow::Cow;
+
+/// Bucket statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting bucket metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct BucketStats {
+    /// Name of the bucket
+    pub name: String,
+    /// Total size of all objects in the bucket (bytes)
+    pub size_bytes: u64,
+    /// Number of objects in the bucket
+    pub objects_count: u64,
+    /// Quota limit for the bucket (bytes), 0 if no quota
+    pub quota_bytes: u64,
+}
+
+/// Collects per-bucket metrics from the provided bucket statistics.
+///
+/// Uses the metric descriptors from `metrics_type::node_bucket` module.
+/// Returns a vector of Prometheus metrics for all buckets.
+pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
+    if buckets.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(buckets.len() * 3);
+    for bucket in buckets {
+        let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
+                .with_label("bucket", bucket_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_bucket_metrics() {
+        let buckets = vec![
+            BucketStats {
+                name: "test-bucket".to_string(),
+                size_bytes: 1000,
+                objects_count: 100,
+                quota_bytes: 0,
+            },
+            BucketStats {
+                name: "another-bucket".to_string(),
+                size_bytes: 2000,
+                objects_count: 200,
+                quota_bytes: 0,
+            },
+        ];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
+        assert_eq!(metrics.len(), 6);
+
+        // Verify test-bucket metrics have correct labels
+        let test_bucket_size_name = BUCKET_USAGE_BYTES_MD.get_full_metric_name();
+        let test_bucket_size = metrics
+            .iter()
+            .find(|m| m.name == test_bucket_size_name && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
+        assert!(test_bucket_size.is_some());
+        assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_with_quotas() {
+        let buckets = vec![BucketStats {
+            name: "quota-bucket".to_string(),
+            size_bytes: 500,
+            objects_count: 10,
+            quota_bytes: 10000,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // 1 bucket * 3 metrics (size, objects, quota) = 3 metrics
+        assert_eq!(metrics.len(), 3);
+
+        // Verify quota metric exists
+        let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
+        let quota_metric = metrics.iter().find(|m| {
+            m.name == quota_metric_name
+                && m.value == 10000.0
+                && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "quota-bucket")
+        });
+        assert!(quota_metric.is_some());
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_empty() {
+        let buckets: Vec<BucketStats> = vec![];
+        let metrics = collect_bucket_metrics(&buckets);
+        assert!(metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_zero_quota_always_reported() {
+        let buckets = vec![BucketStats {
+            name: "no-quota-bucket".to_string(),
+            size_bytes: 100,
+            objects_count: 5,
+            quota_bytes: 0,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
+        assert_eq!(metrics.len(), 3);
+        let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
+        let quota_metric = metrics.iter().find(|m| {
+            m.name == quota_metric_name
+                && m.value == 0.0
+                && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "no-quota-bucket")
+        });
+        assert!(quota_metric.is_some());
+    }
+
+    #[test]
+    fn test_bucket_stats_default() {
+        let stats = BucketStats::default();
+        assert!(stats.name.is_empty());
+        assert_eq!(stats.size_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.quota_bytes, 0);
+    }
+}

crates/obs/src/metrics/collectors/bucket_replication.rs

@@ -0,0 +1,114 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Bucket replication bandwidth metrics collector.
+//!
+//! Collects bandwidth metrics for bucket replication targets.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
+use std::borrow::Cow;
+
+/// Bucket replication bandwidth statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct BucketReplicationBandwidthStats {
+    /// Name of the bucket
+    pub bucket: String,
+    /// Target ARN for replication
+    pub target_arn: String,
+    /// Configured bandwidth limit in bytes per second
+    pub limit_bytes_per_sec: u64,
+    /// Current bandwidth in bytes per second (EWMA)
+    pub current_bandwidth_bytes_per_sec: f64,
+}
+
+/// Collects bucket replication bandwidth metrics from the provided statistics.
+///
+/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
+/// Returns a vector of Prometheus metrics for replication bandwidth.
+pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 2);
+    for stat in stats {
+        let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+        let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
+                .with_label("bucket", bucket_label.clone())
+                .with_label("targetArn", target_arn_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
+                .with_label("bucket", bucket_label)
+                .with_label("targetArn", target_arn_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_bucket_replication_bandwidth_metrics() {
+        let stats = vec![BucketReplicationBandwidthStats {
+            bucket: "b1".to_string(),
+            target_arn: "arn:rustfs:replication:us-east-1:1:test-2".to_string(),
+            limit_bytes_per_sec: 1_048_576,
+            current_bandwidth_bytes_per_sec: 204_800.0,
+        }];
+
+        let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+        assert_eq!(metrics.len(), 2);
+
+        let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
+        let limit_metric = metrics.iter().find(|m| {
+            m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        });
+        assert!(limit_metric.is_some());
+        assert!(
+            limit_metric
+                .and_then(|m| {
+                    m.labels
+                        .iter()
+                        .find(|(k, _)| *k == "targetArn")
+                        .map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
+                })
+                .unwrap_or(false)
+        );
+
+        let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
+        let current_metric = metrics.iter().find(|m| {
+            m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        });
+        assert!(current_metric.is_some());
+    }
+
+    #[test]
+    fn test_collect_bucket_replication_bandwidth_metrics_empty() {
+        let stats: Vec<BucketReplicationBandwidthStats> = Vec::new();
+        let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/obs/src/metrics/collectors/cluster.rs

@@ -0,0 +1,130 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Cluster-wide metrics collector.
+//!
+//! Collects aggregate metrics across the entire RustFS cluster including
+//! total capacity, usage, and object counts.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster::*;
+
+/// Cluster capacity and usage statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting cluster metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterStats {
+    /// Total raw storage capacity across all disks in bytes
+    pub raw_capacity_bytes: u64,
+    /// Usable capacity after erasure coding overhead in bytes
+    pub usable_capacity_bytes: u64,
+    /// Currently used storage in bytes
+    pub used_bytes: u64,
+    /// Available free storage in bytes
+    pub free_bytes: u64,
+    /// Total number of objects in the cluster
+    pub objects_count: u64,
+    /// Total number of buckets in the cluster
+    pub buckets_count: u64,
+}
+
+/// Collects cluster-wide metrics from the provided cluster statistics.
+///
+/// Uses the metric descriptors from `metrics_type::cluster` module.
+/// Returns a vector of Prometheus metrics for cluster statistics.
+pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
+        PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_cluster_metrics() {
+        let stats = ClusterStats {
+            raw_capacity_bytes: 3000,
+            usable_capacity_bytes: 2500,
+            used_bytes: 1200,
+            free_bytes: 1300,
+            objects_count: 100,
+            buckets_count: 5,
+        };
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // Verify raw capacity
+        let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name();
+        let raw_capacity = metrics.iter().find(|m| m.name == raw_capacity_name && m.value == 3000.0);
+        assert!(raw_capacity.is_some());
+
+        // Verify used capacity
+        let used_name = CLUSTER_CAPACITY_USED_BYTES_MD.get_full_metric_name();
+        let used = metrics.iter().find(|m| m.name == used_name && m.value == 1200.0);
+        assert!(used.is_some());
+
+        // Verify object count
+        let objects_name = CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name();
+        let objects = metrics.iter().find(|m| m.name == objects_name && m.value == 100.0);
+        assert!(objects.is_some());
+
+        // Verify bucket count
+        let buckets_name = CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name();
+        let buckets = metrics.iter().find(|m| m.name == buckets_name && m.value == 5.0);
+        assert!(buckets.is_some());
+    }
+
+    #[test]
+    fn test_collect_cluster_metrics_empty() {
+        let stats = ClusterStats::default();
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // All values should be zero
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_cluster_stats_default() {
+        let stats = ClusterStats::default();
+        assert_eq!(stats.raw_capacity_bytes, 0);
+        assert_eq!(stats.usable_capacity_bytes, 0);
+        assert_eq!(stats.used_bytes, 0);
+        assert_eq!(stats.free_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.buckets_count, 0);
+    }
+}

crates/obs/src/metrics/collectors/cluster_config.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster config metrics collector.
+//!
+//! Collects cluster configuration metrics including storage class
+//! parity settings.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster_config`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_config::*;
+
+/// Cluster configuration statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterConfigStats {
+    /// Parity for reduced redundancy storage (RRS) class
+    pub rrs_parity: u32,
+    /// Parity for standard storage class
+    pub standard_parity: u32,
+}
+
+/// Collects cluster config metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_config` module.
+/// Returns a vector of Prometheus metrics for cluster configuration.
+pub fn collect_cluster_config_metrics(stats: &ClusterConfigStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&CONFIG_RRS_PARITY_MD, stats.rrs_parity as f64),
+        PrometheusMetric::from_descriptor(&CONFIG_STANDARD_PARITY_MD, stats.standard_parity as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_cluster_config_metrics() {
+        let stats = ClusterConfigStats {
+            rrs_parity: 2,
+            standard_parity: 4,
+        };
+
+        let metrics = collect_cluster_config_metrics(&stats);
+
+        assert_eq!(metrics.len(), 2);
+
+        let rrs = metrics.iter().find(|m| m.value == 2.0);
+        assert!(rrs.is_some());
+
+        let standard = metrics.iter().find(|m| m.value == 4.0);
+        assert!(standard.is_some());
+    }
+
+    #[test]
+    fn test_collect_cluster_config_metrics_default() {
+        let stats = ClusterConfigStats::default();
+        let metrics = collect_cluster_config_metrics(&stats);
+
+        assert_eq!(metrics.len(), 2);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/cluster_erasure_set.rs

@@ -0,0 +1,179 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster erasure set metrics collector.
+//!
+//! Collects erasure coding set metrics including parity, quorum,
+//! drive counts, and health status.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_erasure_set::*;
+
+/// Erasure set statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ErasureSetStats {
+    /// Pool ID
+    pub pool_id: u32,
+    /// Set ID within the pool
+    pub set_id: u32,
+    /// Total number of drives in the set
+    pub size: u32,
+    /// Number of parity drives
+    pub parity: u32,
+    /// Number of data shards
+    pub data_shards: u32,
+    /// Read quorum
+    pub read_quorum: u32,
+    /// Write quorum
+    pub write_quorum: u32,
+    /// Number of online drives
+    pub online_drives_count: u32,
+    /// Number of healing drives
+    pub healing_drives_count: u32,
+    /// Health status (1=healthy, 0=unhealthy)
+    pub health: u8,
+    /// Read tolerance (number of drive failures tolerated for reads)
+    pub read_tolerance: u32,
+    /// Write tolerance (number of drive failures tolerated for writes)
+    pub write_tolerance: u32,
+    /// Read health status (1=healthy, 0=unhealthy)
+    pub read_health: u8,
+    /// Write health status (1=healthy, 0=unhealthy)
+    pub write_health: u8,
+}
+
+/// Collects erasure set metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for erasure sets.
+pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(stats.len() * 12);
+
+    for stat in stats {
+        let pool_id_label = stat.pool_id.to_string();
+        let set_id_label = stat.set_id.to_string();
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_SIZE_MD, stat.size as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_PARITY_MD, stat.parity as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_DATA_SHARDS_MD, stat.data_shards as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_QUORUM_MD, stat.read_quorum as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_QUORUM_MD, stat.write_quorum as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_ONLINE_DRIVES_COUNT_MD, stat.online_drives_count as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_HEALING_DRIVES_COUNT_MD, stat.healing_drives_count as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_HEALTH_MD, stat.health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_TOLERANCE_MD, stat.read_tolerance as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_TOLERANCE_MD, stat.write_tolerance as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_READ_HEALTH_MD, stat.read_health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
+                .with_label_owned(POOL_ID_L, pool_id_label.clone())
+                .with_label_owned(SET_ID_L, set_id_label.clone()),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_erasure_set_metrics() {
+        let stats = vec![ErasureSetStats {
+            pool_id: 1,
+            set_id: 0,
+            size: 16,
+            parity: 4,
+            data_shards: 12,
+            read_quorum: 13,
+            write_quorum: 13,
+            online_drives_count: 16,
+            healing_drives_count: 0,
+            health: 1,
+            read_tolerance: 4,
+            write_tolerance: 4,
+            read_health: 1,
+            write_health: 1,
+        }];
+
+        let metrics = collect_erasure_set_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 12);
+
+        let size_name = ERASURE_SET_SIZE_MD.get_full_metric_name();
+        let size = metrics.iter().find(|m| m.name == size_name);
+        assert!(size.is_some());
+        assert_eq!(size.map(|m| m.value), Some(16.0));
+
+        let health_name = ERASURE_SET_HEALTH_MD.get_full_metric_name();
+        let health = metrics.iter().find(|m| m.name == health_name);
+        assert!(health.is_some());
+        assert_eq!(health.map(|m| m.value), Some(1.0));
+    }
+
+    #[test]
+    fn test_collect_erasure_set_metrics_empty() {
+        let stats: Vec<ErasureSetStats> = vec![];
+        let metrics = collect_erasure_set_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/obs/src/metrics/collectors/cluster_health.rs

@@ -0,0 +1,85 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster health metrics collector.
+//!
+//! Collects cluster-wide health metrics including drive counts
+//! (offline, online, total).
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::cluster_health`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_health::*;
+
+/// Cluster health statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterHealthStats {
+    /// Number of offline drives in the cluster
+    pub drives_offline_count: u64,
+    /// Number of online drives in the cluster
+    pub drives_online_count: u64,
+    /// Total number of drives in the cluster
+    pub drives_count: u64,
+}
+
+/// Collects cluster health metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_health` module.
+/// Returns a vector of Prometheus metrics for cluster health.
+pub fn collect_cluster_health_metrics(stats: &ClusterHealthStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_OFFLINE_COUNT_MD, stats.drives_offline_count as f64),
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_ONLINE_COUNT_MD, stats.drives_online_count as f64),
+        PrometheusMetric::from_descriptor(&HEALTH_DRIVES_COUNT_MD, stats.drives_count as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_cluster_health_metrics() {
+        let stats = ClusterHealthStats {
+            drives_offline_count: 2,
+            drives_online_count: 18,
+            drives_count: 20,
+        };
+
+        let metrics = collect_cluster_health_metrics(&stats);
+
+        assert_eq!(metrics.len(), 3);
+
+        let offline = metrics.iter().find(|m| m.value == 2.0);
+        assert!(offline.is_some());
+
+        let online = metrics.iter().find(|m| m.value == 18.0);
+        assert!(online.is_some());
+    }
+
+    #[test]
+    fn test_collect_cluster_health_metrics_default() {
+        let stats = ClusterHealthStats::default();
+        let metrics = collect_cluster_health_metrics(&stats);
+
+        assert_eq!(metrics.len(), 3);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/cluster_iam.rs

@@ -0,0 +1,117 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster IAM metrics collector.
+//!
+//! Collects IAM (Identity and Access Management) metrics including
+//! plugin authentication service stats and sync statistics.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_iam::*;
+
+/// IAM statistics.
+#[derive(Debug, Clone, Default)]
+pub struct IamStats {
+    /// Time in seconds since last failed authn service request
+    pub plugin_authn_service_last_fail_seconds: u64,
+    /// Time in seconds since last successful authn service request
+    pub plugin_authn_service_last_succ_seconds: u64,
+    /// Average RTT of successful requests in the last minute (ms)
+    pub plugin_authn_service_succ_avg_rtt_ms_minute: u64,
+    /// Maximum RTT of successful requests in the last minute (ms)
+    pub plugin_authn_service_succ_max_rtt_ms_minute: u64,
+    /// Total requests count in the last full minute
+    pub plugin_authn_service_total_requests_minute: u64,
+    /// Time in milliseconds since last successful IAM data sync
+    pub since_last_sync_millis: u64,
+    /// Number of failed IAM data syncs since server start
+    pub sync_failures: u64,
+    /// Number of successful IAM data syncs since server start
+    pub sync_successes: u64,
+}
+
+/// Collects IAM metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for IAM statistics.
+pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
+            stats.plugin_authn_service_last_fail_seconds as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD,
+            stats.plugin_authn_service_last_succ_seconds as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD,
+            stats.plugin_authn_service_succ_avg_rtt_ms_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD,
+            stats.plugin_authn_service_succ_max_rtt_ms_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD,
+            stats.plugin_authn_service_total_requests_minute as f64,
+        ),
+        PrometheusMetric::from_descriptor(&SINCE_LAST_SYNC_MILLIS_MD, stats.since_last_sync_millis as f64),
+        PrometheusMetric::from_descriptor(&SYNC_FAILURES_MD, stats.sync_failures as f64),
+        PrometheusMetric::from_descriptor(&SYNC_SUCCESSES_MD, stats.sync_successes as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_iam_metrics() {
+        let stats = IamStats {
+            plugin_authn_service_last_fail_seconds: 3600,
+            plugin_authn_service_last_succ_seconds: 10,
+            plugin_authn_service_succ_avg_rtt_ms_minute: 50,
+            plugin_authn_service_succ_max_rtt_ms_minute: 200,
+            plugin_authn_service_total_requests_minute: 1000,
+            since_last_sync_millis: 5000,
+            sync_failures: 5,
+            sync_successes: 1000,
+        };
+
+        let metrics = collect_iam_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 8);
+
+        let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name();
+        let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name);
+        assert!(sync_successes.is_some());
+        assert_eq!(sync_successes.map(|m| m.value), Some(1000.0));
+    }
+
+    #[test]
+    fn test_collect_iam_metrics_default() {
+        let stats = IamStats::default();
+        let metrics = collect_iam_metrics(&stats);
+
+        assert_eq!(metrics.len(), 8);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/cluster_usage.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Cluster usage metrics collector.
+//!
+//! Collects cluster-wide and per-bucket usage metrics including
+//! object counts, sizes, versions, and distributions.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_usage::*;
+
+/// Cluster-wide usage statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterUsageStats {
+    /// Total bytes used in the cluster
+    pub total_bytes: u64,
+    /// Total number of objects
+    pub objects_count: u64,
+    /// Total number of object versions (including delete markers)
+    pub versions_count: u64,
+    /// Total number of delete markers
+    pub delete_markers_count: u64,
+    /// Object size distribution by range
+    pub object_size_distribution: Vec<(String, u64)>,
+    /// Version count distribution by range
+    pub versions_distribution: Vec<(String, u64)>,
+}
+
+/// Per-bucket usage statistics.
+#[derive(Debug, Clone, Default)]
+pub struct BucketUsageStats {
+    /// Bucket name
+    pub bucket: String,
+    /// Total bytes used in the bucket
+    pub total_bytes: u64,
+    /// Total number of objects in the bucket
+    pub objects_count: u64,
+    /// Total number of object versions (including delete markers)
+    pub versions_count: u64,
+    /// Total number of delete markers
+    pub delete_markers_count: u64,
+    /// Bucket quota in bytes (0 if no quota)
+    pub quota_bytes: u64,
+    /// Object size distribution by range
+    pub object_size_distribution: Vec<(String, u64)>,
+    /// Version count distribution by range
+    pub version_count_distribution: Vec<(String, u64)>,
+}
+
+/// Collects cluster-wide usage metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for cluster usage.
+pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len());
+
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64));
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64));
+    metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64));
+    metrics.push(PrometheusMetric::from_descriptor(
+        &USAGE_DELETE_MARKERS_COUNT_MD,
+        stats.delete_markers_count as f64,
+    ));
+
+    // Object size distribution
+    for (range, count) in &stats.object_size_distribution {
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_OBJECTS_DISTRIBUTION_MD, *count as f64)
+                .with_label_owned(RANGE_LABEL, range.clone()),
+        );
+    }
+
+    // Version distribution
+    for (range, count) in &stats.versions_distribution {
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_VERSIONS_DISTRIBUTION_MD, *count as f64)
+                .with_label_owned(RANGE_LABEL, range.clone()),
+        );
+    }
+
+    metrics
+}
+
+/// Collects per-bucket usage metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for bucket usage.
+pub fn collect_bucket_usage_metrics(stats: &[BucketUsageStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::new();
+
+    for stat in stats {
+        let bucket_label = stat.bucket.clone();
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, stat.total_bytes as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, stat.objects_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, stat.versions_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, stat.delete_markers_count as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, stat.quota_bytes as f64)
+                .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+        );
+
+        // Object size distribution per bucket
+        for (range, count) in &stat.object_size_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, *count as f64)
+                    .with_label_owned(RANGE_LABEL, range.clone())
+                    .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+            );
+        }
+
+        // Version count distribution per bucket
+        for (range, count) in &stat.version_count_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, *count as f64)
+                    .with_label_owned(RANGE_LABEL, range.clone())
+                    .with_label_owned(BUCKET_LABEL, bucket_label.clone()),
+            );
+        }
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_cluster_usage_metrics() {
+        let stats = ClusterUsageStats {
+            total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
+            objects_count: 10000,
+            versions_count: 15000,
+            delete_markers_count: 500,
+            object_size_distribution: vec![
+                ("0-1KB".to_string(), 5000),
+                ("1KB-1MB".to_string(), 3000),
+                ("1MB-100MB".to_string(), 1500),
+                ("100MB+".to_string(), 500),
+            ],
+            versions_distribution: vec![("1".to_string(), 8000), ("2-5".to_string(), 1500), ("6+".to_string(), 500)],
+        };
+
+        let metrics = collect_cluster_usage_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 4 base metrics + 4 size distribution + 3 version distribution = 11
+        assert_eq!(metrics.len(), 11);
+
+        let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
+        let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
+        assert!(total_bytes.is_some());
+    }
+
+    #[test]
+    fn test_collect_bucket_usage_metrics() {
+        let stats = vec![BucketUsageStats {
+            bucket: "test-bucket".to_string(),
+            total_bytes: 1024 * 1024 * 1024 * 10, // 10 GB
+            objects_count: 1000,
+            versions_count: 1200,
+            delete_markers_count: 50,
+            quota_bytes: 1024 * 1024 * 1024 * 100, // 100 GB quota
+            object_size_distribution: vec![("0-1KB".to_string(), 500)],
+            version_count_distribution: vec![("1".to_string(), 800)],
+        }];
+
+        let metrics = collect_bucket_usage_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 5 base metrics + 1 size distribution + 1 version distribution = 7
+        assert_eq!(metrics.len(), 7);
+    }
+}

crates/obs/src/metrics/collectors/dial9.rs

@@ -0,0 +1,181 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! dial9 Tokio runtime telemetry metrics collector.
+//!
+//! This module provides metrics for monitoring the health and performance
+//! of the dial9 telemetry system itself.
+
+#![allow(dead_code)]
+
+use crate::MetricType;
+use crate::metrics::report::PrometheusMetric;
+use rustfs_config::{DEFAULT_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_ENABLED};
+use rustfs_utils::get_env_bool;
+
+/// Dial9 telemetry system statistics.
+#[derive(Debug, Clone, Default)]
+pub struct Dial9Stats {
+    /// Total number of telemetry events recorded
+    pub events_total: u64,
+
+    /// Total bytes written to trace files
+    pub bytes_written: u64,
+
+    /// Number of file rotations that have occurred
+    pub rotation_count: u64,
+
+    /// Total number of dial9 errors
+    pub errors_total: u64,
+
+    /// Estimated CPU overhead percentage (if available)
+    pub cpu_overhead_percent: f64,
+
+    /// Current disk usage by trace files in bytes
+    pub disk_usage_bytes: u64,
+
+    /// Number of active sessions
+    pub active_sessions: u64,
+}
+
+/// Collect dial9 telemetry metrics.
+///
+/// This function converts dial9 statistics into Prometheus metrics format.
+///
+/// # Arguments
+///
+/// * `stats` - Dial9 statistics to report
+///
+/// # Returns
+///
+/// A vector of Prometheus metrics for dial9 telemetry statistics.
+pub fn collect_dial9_metrics(stats: &Dial9Stats) -> Vec<PrometheusMetric> {
+    let enabled = is_dial9_enabled();
+    let enabled_value = if enabled { 1.0 } else { 0.0 };
+
+    let mut metrics = vec![PrometheusMetric::new(
+        "rustfs_dial9_enabled",
+        MetricType::Gauge,
+        "Whether dial9 telemetry is enabled (1) or disabled (0)",
+        enabled_value,
+    )];
+
+    // If dial9 is disabled, return just the enabled flag
+    if !enabled {
+        return metrics;
+    }
+
+    // Add detailed metrics when enabled
+    metrics.extend(vec![
+        PrometheusMetric::new(
+            "rustfs_dial9_events_total",
+            MetricType::Counter,
+            "Total number of Tokio runtime events recorded by dial9",
+            stats.events_total as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_bytes_written_total",
+            MetricType::Counter,
+            "Total bytes written to dial9 trace files",
+            stats.bytes_written as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_rotations_total",
+            MetricType::Counter,
+            "Total number of trace file rotations",
+            stats.rotation_count as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_errors_total",
+            MetricType::Counter,
+            "Total number of dial9 telemetry errors",
+            stats.errors_total as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_cpu_overhead_percent",
+            MetricType::Gauge,
+            "Estimated CPU overhead percentage from dial9 telemetry",
+            stats.cpu_overhead_percent,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_disk_usage_bytes",
+            MetricType::Gauge,
+            "Current disk usage by dial9 trace files",
+            stats.disk_usage_bytes as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_active_sessions",
+            MetricType::Gauge,
+            "Number of active dial9 telemetry sessions",
+            stats.active_sessions as f64,
+        ),
+    ]);
+
+    metrics
+}
+
+/// Check if dial9 telemetry is enabled via environment variable.
+pub fn is_dial9_enabled() -> bool {
+    get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_dial9_stats_default() {
+        let stats = Dial9Stats::default();
+        assert_eq!(stats.events_total, 0);
+        assert_eq!(stats.bytes_written, 0);
+        assert_eq!(stats.rotation_count, 0);
+        assert_eq!(stats.errors_total, 0);
+        assert_eq!(stats.cpu_overhead_percent, 0.0);
+        assert_eq!(stats.disk_usage_bytes, 0);
+        assert_eq!(stats.active_sessions, 0);
+    }
+
+    #[test]
+    fn test_collect_dial9_metrics() {
+        let stats = Dial9Stats {
+            events_total: 100,
+            bytes_written: 1024,
+            ..Default::default()
+        };
+        let metrics = collect_dial9_metrics(&stats);
+
+        // Should always have at least the enabled flag
+        assert!(!metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_dial9_metrics_with_values() {
+        let stats = Dial9Stats {
+            events_total: 10000,
+            bytes_written: 1024000,
+            rotation_count: 5,
+            errors_total: 0,
+            cpu_overhead_percent: 2.5,
+            disk_usage_bytes: 2048000,
+            active_sessions: 1,
+        };
+
+        let metrics = collect_dial9_metrics(&stats);
+
+        // When dial9 is enabled, should have all metrics
+        // Note: This test assumes dial9 is enabled in the test environment
+        // If disabled, only the enabled flag metric will be present
+        assert!(!metrics.is_empty());
+    }
+}

crates/obs/src/metrics/collectors/ilm.rs

@@ -0,0 +1,96 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! ILM (Information Lifecycle Management) metrics collector.
+//!
+//! Collects ILM metrics including pending tasks, active tasks,
+//! and scanned versions.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::ilm`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::ilm::*;
+
+/// ILM statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct IlmStats {
+    /// Number of pending ILM expiry tasks
+    pub expiry_pending_tasks: u64,
+    /// Number of active ILM transition tasks
+    pub transition_active_tasks: u64,
+    /// Number of pending ILM transition tasks
+    pub transition_pending_tasks: u64,
+    /// Number of missed immediate ILM transition tasks
+    pub transition_missed_immediate_tasks: u64,
+    /// Total number of object versions scanned for ILM
+    pub versions_scanned: u64,
+}
+
+/// Collects ILM metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::ilm` module.
+/// Returns a vector of Prometheus metrics for ILM statistics.
+pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&ILM_EXPIRY_PENDING_TASKS_MD, stats.expiry_pending_tasks as f64),
+        PrometheusMetric::from_descriptor(&ILM_TRANSITION_ACTIVE_TASKS_MD, stats.transition_active_tasks as f64),
+        PrometheusMetric::from_descriptor(&ILM_TRANSITION_PENDING_TASKS_MD, stats.transition_pending_tasks as f64),
+        PrometheusMetric::from_descriptor(
+            &ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD,
+            stats.transition_missed_immediate_tasks as f64,
+        ),
+        PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_ilm_metrics() {
+        let stats = IlmStats {
+            expiry_pending_tasks: 100,
+            transition_active_tasks: 5,
+            transition_pending_tasks: 50,
+            transition_missed_immediate_tasks: 10,
+            versions_scanned: 1000000,
+        };
+
+        let metrics = collect_ilm_metrics(&stats);
+
+        assert_eq!(metrics.len(), 5);
+
+        let pending = metrics.iter().find(|m| m.value == 100.0);
+        assert!(pending.is_some());
+
+        let scanned = metrics.iter().find(|m| m.value == 1000000.0);
+        assert!(scanned.is_some());
+    }
+
+    #[test]
+    fn test_collect_ilm_metrics_default() {
+        let stats = IlmStats::default();
+        let metrics = collect_ilm_metrics(&stats);
+
+        assert_eq!(metrics.len(), 5);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/mod.rs

@@ -0,0 +1,71 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod audit;
+pub mod bucket;
+pub mod bucket_replication;
+pub mod cluster;
+pub mod cluster_config;
+pub mod cluster_erasure_set;
+pub mod cluster_health;
+pub mod cluster_iam;
+pub mod cluster_usage;
+pub mod dial9;
+pub mod ilm;
+pub mod node;
+pub mod notification;
+pub mod notification_target;
+pub mod replication;
+pub mod request;
+pub mod resource;
+pub mod scanner;
+pub mod system_cpu;
+pub mod system_drive;
+#[cfg(feature = "gpu")]
+pub mod system_gpu;
+pub mod system_memory;
+pub mod system_network;
+pub mod system_process;
+
+pub use audit::{AuditTargetStats, collect_audit_metrics};
+pub use bucket::{BucketStats, collect_bucket_metrics};
+pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
+pub use cluster::{ClusterStats, collect_cluster_metrics};
+pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
+pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
+pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics};
+pub use cluster_iam::{IamStats, collect_iam_metrics};
+pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
+pub use dial9::{Dial9Stats, collect_dial9_metrics, is_dial9_enabled};
+pub use ilm::{IlmStats, collect_ilm_metrics};
+pub use node::{DiskStats, collect_node_metrics};
+pub use notification::{NotificationStats, collect_notification_metrics};
+pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
+pub use replication::{ReplicationStats, collect_replication_metrics};
+pub use request::{ApiRequestStats, collect_request_metrics};
+pub use resource::{ResourceStats, collect_resource_metrics};
+pub use scanner::{ScannerStats, collect_scanner_metrics};
+pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
+pub use system_drive::{
+    DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
+    collect_process_disk_metrics,
+};
+#[cfg(feature = "gpu")]
+pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
+pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
+pub use system_network::{NetworkStats, ProcessNetworkStats, collect_network_metrics, collect_process_network_metrics};
+pub use system_process::{
+    ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes,
+    collect_process_metrics,
+};

crates/obs/src/metrics/collectors/node.rs

@@ -0,0 +1,166 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-node and per-disk metrics collector.
+//!
+//! Collects storage metrics for each disk/drive in the cluster,
+//! including capacity, usage, and health status.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::node_disk`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::node_disk::*;
+use std::borrow::Cow;
+
+/// Disk statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting disk metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct DiskStats {
+    /// Server endpoint (e.g., "node1:9000")
+    pub server: String,
+    /// Drive path (e.g., "/data/disk1")
+    pub drive: String,
+    /// Total disk capacity in bytes
+    pub total_bytes: u64,
+    /// Used disk space in bytes
+    pub used_bytes: u64,
+    /// Free disk space in bytes
+    pub free_bytes: u64,
+}
+
+/// Collects per-disk metrics from the provided disk statistics.
+///
+/// Uses the metric descriptors from `metrics_type::node_disk` module.
+/// Returns a vector of Prometheus metrics for all disks.
+pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec<PrometheusMetric> {
+    if disks.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(disks.len() * 3);
+    for disk in disks {
+        let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone());
+        let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NODE_DISK_TOTAL_BYTES_MD, disk.total_bytes as f64)
+                .with_label("server", server_label.clone())
+                .with_label("drive", drive_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NODE_DISK_USED_BYTES_MD, disk.used_bytes as f64)
+                .with_label("server", server_label.clone())
+                .with_label("drive", drive_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NODE_DISK_FREE_BYTES_MD, disk.free_bytes as f64)
+                .with_label("server", server_label)
+                .with_label("drive", drive_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_node_metrics() {
+        let disks = vec![
+            DiskStats {
+                server: "node1:9000".to_string(),
+                drive: "/data/disk1".to_string(),
+                total_bytes: 1000000,
+                used_bytes: 400000,
+                free_bytes: 600000,
+            },
+            DiskStats {
+                server: "node2:9000".to_string(),
+                drive: "/data/disk2".to_string(),
+                total_bytes: 2000000,
+                used_bytes: 800000,
+                free_bytes: 1200000,
+            },
+        ];
+
+        let metrics = collect_node_metrics(&disks);
+
+        // 2 disks * 3 metrics each = 6 metrics
+        assert_eq!(metrics.len(), 6);
+
+        // Verify node1 disk1 total bytes
+        let node1_total_name = NODE_DISK_TOTAL_BYTES_MD.get_full_metric_name();
+        let node1_total = metrics.iter().find(|m| {
+            m.name == node1_total_name
+                && m.value == 1000000.0
+                && m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000")
+                && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1")
+        });
+        assert!(node1_total.is_some());
+
+        // Verify node2 disk2 used bytes
+        let node2_used_name = NODE_DISK_USED_BYTES_MD.get_full_metric_name();
+        let node2_used = metrics.iter().find(|m| {
+            m.name == node2_used_name
+                && m.value == 800000.0
+                && m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000")
+                && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2")
+        });
+        assert!(node2_used.is_some());
+    }
+
+    #[test]
+    fn test_collect_node_metrics_empty() {
+        let disks: Vec<DiskStats> = vec![];
+        let metrics = collect_node_metrics(&disks);
+        assert!(metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_node_metrics_labels() {
+        let disks = vec![DiskStats {
+            server: "localhost:9000".to_string(),
+            drive: "/mnt/data".to_string(),
+            total_bytes: 500,
+            used_bytes: 200,
+            free_bytes: 300,
+        }];
+
+        let metrics = collect_node_metrics(&disks);
+
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 2);
+            assert!(metric.labels.iter().any(|(k, _)| *k == "server"));
+            assert!(metric.labels.iter().any(|(k, _)| *k == "drive"));
+        }
+    }
+
+    #[test]
+    fn test_disk_stats_default() {
+        let stats = DiskStats::default();
+        assert!(stats.server.is_empty());
+        assert!(stats.drive.is_empty());
+        assert_eq!(stats.total_bytes, 0);
+        assert_eq!(stats.used_bytes, 0);
+        assert_eq!(stats.free_bytes, 0);
+    }
+}

crates/obs/src/metrics/collectors/notification.rs

@@ -0,0 +1,91 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Notification metrics collector.
+//!
+//! Collects notification system metrics including events sent,
+//! errors, and skipped events.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::cluster_notification::{
+    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
+    NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
+};
+
+/// Notification statistics.
+#[derive(Debug, Clone, Default)]
+pub struct NotificationStats {
+    /// Number of concurrent send operations in progress
+    pub current_send_in_progress: u64,
+    /// Total number of events that encountered errors
+    pub events_errors_total: u64,
+    /// Total number of events successfully sent
+    pub events_sent_total: u64,
+    /// Total number of events skipped
+    pub events_skipped_total: u64,
+}
+
+/// Collects notification metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::cluster_notification` module.
+/// Returns a vector of Prometheus metrics for notification statistics.
+pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, stats.current_send_in_progress as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, stats.events_errors_total as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_MD, stats.events_sent_total as f64),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, stats.events_skipped_total as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_notification_metrics() {
+        let stats = NotificationStats {
+            current_send_in_progress: 5,
+            events_errors_total: 10,
+            events_sent_total: 10000,
+            events_skipped_total: 50,
+        };
+
+        let metrics = collect_notification_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 4);
+
+        let sent = metrics.iter().find(|m| m.value == 10000.0);
+        assert!(sent.is_some());
+
+        let errors = metrics.iter().find(|m| m.value == 10.0);
+        assert!(errors.is_some());
+    }
+
+    #[test]
+    fn test_collect_notification_metrics_default() {
+        let stats = NotificationStats::default();
+        let metrics = collect_notification_metrics(&stats);
+
+        assert_eq!(metrics.len(), 4);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/notification_target.rs

@@ -0,0 +1,92 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::notification_target::{
+    NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD,
+    TARGET_ID, TARGET_TYPE,
+};
+use std::borrow::Cow;
+
+#[derive(Debug, Clone, Default)]
+pub struct NotificationTargetStats {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub target_type: String,
+    pub total_messages: u64,
+}
+
+pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
+    for stat in stats {
+        let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
+        let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_MESSAGES_MD, stat.failed_messages as f64)
+                .with_label(TARGET_ID, target_id.clone())
+                .with_label(TARGET_TYPE, target_type.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
+                .with_label(TARGET_ID, target_id.clone())
+                .with_label(TARGET_TYPE, target_type.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, stat.total_messages as f64)
+                .with_label(TARGET_ID, target_id)
+                .with_label(TARGET_TYPE, target_type),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_notification_target_metrics() {
+        let stats = vec![NotificationTargetStats {
+            failed_messages: 2,
+            queue_length: 4,
+            target_id: "primary:webhook".to_string(),
+            target_type: "webhook".to_string(),
+            total_messages: 42,
+        }];
+
+        let metrics = collect_notification_target_metrics(&stats);
+
+        assert_eq!(metrics.len(), 3);
+        assert!(metrics.iter().any(|metric| {
+            metric.value == 42.0
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ID && value == "primary:webhook")
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
+        }));
+    }
+}

crates/obs/src/metrics/collectors/replication.rs

@@ -0,0 +1,107 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Replication metrics collector.
+//!
+//! Collects cluster-wide replication metrics including queue stats,
+//! data transfer rates, and worker information.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::replication::*;
+
+/// Replication statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ReplicationStats {
+    /// Number of active replication workers
+    pub active_workers: u64,
+    /// Current data transfer rate in bytes/sec
+    pub current_data_transfer_rate: f64,
+    /// Bytes queued in the last full minute
+    pub last_minute_queued_bytes: u64,
+    /// Objects queued in the last full minute
+    pub last_minute_queued_count: u64,
+    /// Maximum active workers seen since server start
+    pub max_active_workers: u64,
+    /// Maximum bytes queued since server start
+    pub max_queued_bytes: u64,
+    /// Maximum objects queued since server start
+    pub max_queued_count: u64,
+    /// Maximum data transfer rate seen since server start
+    pub max_data_transfer_rate: f64,
+    /// Objects in replication backlog in the last 5 minutes
+    pub recent_backlog_count: u64,
+}
+
+/// Collects replication metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for replication statistics.
+pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
+        PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD, stats.last_minute_queued_count as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_MD, stats.max_active_workers as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_MD, stats.max_queued_bytes as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_MD, stats.max_queued_count as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_MAX_DATA_TRANSFER_RATE_MD, stats.max_data_transfer_rate),
+        PrometheusMetric::from_descriptor(&REPLICATION_RECENT_BACKLOG_COUNT_MD, stats.recent_backlog_count as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_replication_metrics() {
+        let stats = ReplicationStats {
+            active_workers: 10,
+            current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
+            last_minute_queued_bytes: 1024 * 1024 * 100,       // 100 MB
+            last_minute_queued_count: 500,
+            max_active_workers: 20,
+            max_queued_bytes: 1024 * 1024 * 500, // 500 MB
+            max_queued_count: 2000,
+            max_data_transfer_rate: 1024.0 * 1024.0 * 10.0, // 10 MB/s
+            recent_backlog_count: 1500,
+        };
+
+        let metrics = collect_replication_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 9);
+
+        // Verify active workers
+        let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
+        let active = metrics.iter().find(|m| m.name == active_name);
+        assert!(active.is_some());
+        assert_eq!(active.map(|m| m.value), Some(10.0));
+    }
+
+    #[test]
+    fn test_collect_replication_metrics_default() {
+        let stats = ReplicationStats::default();
+        let metrics = collect_replication_metrics(&stats);
+
+        assert_eq!(metrics.len(), 9);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/request.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! API request metrics collector.
+//!
+//! Collects API request metrics including request counts, errors,
+//! latency, and traffic statistics.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::request::*;
+
+/// API request statistics for a specific API endpoint.
+#[derive(Debug, Clone, Default)]
+pub struct ApiRequestStats {
+    /// API name (e.g., "GetObject", "PutObject")
+    pub name: String,
+    /// Request type (e.g., "s3", "admin")
+    pub req_type: String,
+    /// Number of requests currently in flight
+    pub in_flight: u64,
+    /// Total number of requests
+    pub total: u64,
+    /// Total number of errors (4xx + 5xx)
+    pub errors_total: u64,
+    /// Total number of 5xx errors
+    pub errors_5xx: u64,
+    /// Total number of 4xx errors
+    pub errors_4xx: u64,
+    /// Total number of canceled requests
+    pub canceled: u64,
+    /// TTFB distribution by bucket (le label)
+    pub ttfb_distribution: Vec<(String, f64)>,
+    /// Bytes sent
+    pub sent_bytes: u64,
+    /// Bytes received
+    pub recv_bytes: u64,
+}
+
+/// Collects API request metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for API request statistics.
+pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::new();
+
+    for stat in stats {
+        // In-flight requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Total requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Total errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // 5xx errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // 4xx errors
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // Canceled requests
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
+                .with_label_owned(NAME_LABEL, stat.name.clone())
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+
+        // TTFB distribution (histogram buckets)
+        for (le, value) in &stat.ttfb_distribution {
+            metrics.push(
+                PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
+                    .with_label_owned(NAME_LABEL, stat.name.clone())
+                    .with_label_owned(TYPE_LABEL, stat.req_type.clone())
+                    .with_label_owned(LE_LABEL, le.clone()),
+            );
+        }
+
+        // Traffic metrics
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_MD, stat.sent_bytes as f64)
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_MD, stat.recv_bytes as f64)
+                .with_label_owned(TYPE_LABEL, stat.req_type.clone()),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_request_metrics() {
+        let stats = vec![ApiRequestStats {
+            name: "GetObject".to_string(),
+            req_type: "s3".to_string(),
+            in_flight: 10,
+            total: 10000,
+            errors_total: 50,
+            errors_5xx: 10,
+            errors_4xx: 40,
+            canceled: 5,
+            ttfb_distribution: vec![
+                ("0.1".to_string(), 5000.0),
+                ("0.5".to_string(), 8000.0),
+                ("1.0".to_string(), 9500.0),
+                ("+Inf".to_string(), 10000.0),
+            ],
+            sent_bytes: 1024 * 1024 * 500, // 500 MB
+            recv_bytes: 1024 * 1024 * 100, // 100 MB
+        }];
+
+        let metrics = collect_request_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
+        assert_eq!(metrics.len(), 12);
+
+        let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
+        let total = metrics.iter().find(|m| m.name == total_name);
+        assert!(total.is_some());
+        assert_eq!(total.map(|m| m.value), Some(10000.0));
+
+        let in_flight_name = API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name();
+        let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
+        assert!(in_flight.is_some());
+        assert_eq!(in_flight.map(|m| m.value), Some(10.0));
+    }
+
+    #[test]
+    fn test_collect_request_metrics_empty() {
+        let stats: Vec<ApiRequestStats> = vec![];
+        let metrics = collect_request_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/obs/src/metrics/collectors/resource.rs

@@ -0,0 +1,127 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! System resource metrics collector.
+//!
+//! Collects system-level metrics for the RustFS process including
+//! CPU usage, memory consumption, and process uptime.
+//!
+//! This collector reuses the metric descriptors defined in `metrics_type::process_resource`
+//! to avoid duplication of metric names, types, and help text.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::process_resource::*;
+
+/// Resource statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting resource metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct ResourceStats {
+    /// CPU usage as a percentage (can exceed 100% on multi-core systems)
+    pub cpu_percent: f64,
+    /// Resident memory usage in bytes
+    pub memory_bytes: u64,
+    /// Process uptime in seconds
+    pub uptime_seconds: u64,
+}
+
+/// Collects resource metrics from the provided resource statistics.
+///
+/// Uses the metric descriptors from `metrics_type::process_resource` module.
+/// Returns a vector of Prometheus metrics for resource statistics.
+pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent),
+        PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_resource_metrics() {
+        let stats = ResourceStats {
+            cpu_percent: 45.5,
+            memory_bytes: 1024 * 1024 * 256,
+            uptime_seconds: 7200,
+        };
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        // Verify CPU metric
+        let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
+        let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 45.5);
+        assert!(cpu.is_some());
+
+        // Verify memory metric
+        let memory_metric_name = PROCESS_MEMORY_BYTES_MD.get_full_metric_name();
+        let memory = metrics
+            .iter()
+            .find(|m| m.name == memory_metric_name && m.value == (1024 * 1024 * 256) as f64);
+        assert!(memory.is_some());
+
+        // Verify uptime metric
+        let uptime_metric_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
+        let uptime = metrics.iter().find(|m| m.name == uptime_metric_name && m.value == 7200.0);
+        assert!(uptime.is_some());
+    }
+
+    #[test]
+    fn test_collect_resource_metrics_zero_values() {
+        let stats = ResourceStats::default();
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_collect_resource_metrics_high_cpu() {
+        let stats = ResourceStats {
+            cpu_percent: 150.0, // Can exceed 100% on multi-core systems
+            memory_bytes: 0,
+            uptime_seconds: 0,
+        };
+
+        let metrics = collect_resource_metrics(&stats);
+        report_metrics(&metrics);
+
+        let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
+        let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 150.0);
+        assert!(cpu.is_some());
+    }
+
+    #[test]
+    fn test_resource_stats_default() {
+        let stats = ResourceStats::default();
+        assert_eq!(stats.cpu_percent, 0.0);
+        assert_eq!(stats.memory_bytes, 0);
+        assert_eq!(stats.uptime_seconds, 0);
+    }
+}

crates/obs/src/metrics/collectors/scanner.rs

@@ -0,0 +1,99 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Scanner metrics collector.
+//!
+//! Collects background scanner metrics including bucket scans,
+//! directory scans, and object scans.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::scanner::{
+    SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
+};
+
+/// Scanner statistics.
+#[derive(Debug, Clone, Default)]
+pub struct ScannerStats {
+    /// Number of bucket scans finished
+    pub bucket_scans_finished: u64,
+    /// Number of bucket scans started
+    pub bucket_scans_started: u64,
+    /// Number of directories scanned
+    pub directories_scanned: u64,
+    /// Number of objects scanned
+    pub objects_scanned: u64,
+    /// Number of object versions scanned
+    pub versions_scanned: u64,
+    /// Seconds since last scanner activity
+    pub last_activity_seconds: u64,
+}
+
+/// Collects scanner metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::scanner` module.
+/// Returns a vector of Prometheus metrics for scanner statistics.
+pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
+    ]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_scanner_metrics() {
+        let stats = ScannerStats {
+            bucket_scans_finished: 100,
+            bucket_scans_started: 100,
+            directories_scanned: 50000,
+            objects_scanned: 1000000,
+            versions_scanned: 1500000,
+            last_activity_seconds: 30,
+        };
+
+        let metrics = collect_scanner_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        let objects = metrics.iter().find(|m| m.value == 1000000.0);
+        assert!(objects.is_some());
+
+        let last_activity = metrics.iter().find(|m| m.value == 30.0);
+        assert!(last_activity.is_some());
+    }
+
+    #[test]
+    fn test_collect_scanner_metrics_default() {
+        let stats = ScannerStats::default();
+        let metrics = collect_scanner_metrics(&stats);
+
+        assert_eq!(metrics.len(), 6);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/system_cpu.rs

@@ -0,0 +1,184 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System CPU metrics collector.
+//!
+//! Collects CPU metrics including load average, CPU time distribution,
+//! and process-level CPU usage.
+//!
+//! This module provides both system-level and process-level CPU metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::system_cpu::*;
+use crate::metrics::schema::system_process::{PROCESS_CPU_USAGE_MD, PROCESS_CPU_UTILIZATION_MD};
+use std::borrow::Cow;
+
+/// System CPU statistics.
+#[derive(Debug, Clone, Default)]
+pub struct CpuStats {
+    /// Average CPU idle time (percentage, 0-100)
+    pub avg_idle: f64,
+    /// Average CPU I/O wait time (percentage, 0-100)
+    pub avg_iowait: f64,
+    /// CPU load average over 1 minute
+    pub load_avg: f64,
+    /// CPU load average as percentage
+    pub load_avg_perc: f64,
+    /// CPU nice time (percentage, 0-100)
+    pub nice: f64,
+    /// CPU steal time (percentage, 0-100)
+    pub steal: f64,
+    /// CPU system time (percentage, 0-100)
+    pub system: f64,
+    /// CPU user time (percentage, 0-100)
+    pub user: f64,
+}
+
+/// Process CPU statistics.
+///
+/// Contains CPU usage metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessCpuStats {
+    /// CPU usage percentage (0-100)
+    pub usage: f64,
+    /// CPU utilization percentage (considering multiple cores, can exceed 100)
+    pub utilization: f64,
+}
+
+/// Collects CPU metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_cpu` module.
+/// Returns a vector of Prometheus metrics for CPU statistics.
+pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle),
+        PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IOWAIT_MD, stats.avg_iowait),
+        PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg),
+        PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc),
+        PrometheusMetric::from_descriptor(&SYS_CPU_NICE_MD, stats.nice),
+        PrometheusMetric::from_descriptor(&SYS_CPU_STEAL_MD, stats.steal),
+        PrometheusMetric::from_descriptor(&SYS_CPU_SYSTEM_MD, stats.system),
+        PrometheusMetric::from_descriptor(&SYS_CPU_USER_MD, stats.user),
+    ]
+}
+
+/// Collects process CPU metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_process` module.
+/// Returns a vector of Prometheus metrics for process CPU statistics.
+///
+/// # Arguments
+///
+/// * `stats` - Process CPU statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_cpu_metrics(
+    stats: &ProcessCpuStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut usage_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_USAGE_MD, stats.usage);
+    let mut utilization_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_UTILIZATION_MD, stats.utilization);
+
+    if let Some(l) = labels {
+        usage_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        utilization_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![usage_metric, utilization_metric]
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_cpu_metrics() {
+        let stats = CpuStats {
+            avg_idle: 75.5,
+            avg_iowait: 2.3,
+            load_avg: 1.5,
+            load_avg_perc: 37.5,
+            nice: 0.5,
+            steal: 0.1,
+            system: 10.0,
+            user: 15.0,
+        };
+
+        let metrics = collect_cpu_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 8);
+
+        // Verify that metric names are properly generated from descriptors
+        assert!(metrics.iter().all(|m| m.name.starts_with("rustfs_system_cpu_")));
+    }
+
+    #[test]
+    fn test_collect_cpu_metrics_default() {
+        let stats = CpuStats::default();
+        let metrics = collect_cpu_metrics(&stats);
+
+        assert_eq!(metrics.len(), 8);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_collect_process_cpu_metrics() {
+        let stats = ProcessCpuStats {
+            usage: 45.5,
+            utilization: 182.0, // 4 cores at ~45% each
+        };
+
+        let metrics = collect_process_cpu_metrics(&stats, None);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 2);
+
+        // Verify usage metric
+        let usage_metric = metrics.iter().find(|m| m.name.contains("cpu_usage"));
+        assert!(usage_metric.is_some());
+        assert_eq!(usage_metric.map(|m| m.value), Some(45.5));
+
+        // Verify utilization metric
+        let util_metric = metrics.iter().find(|m| m.name.contains("cpu_utilization"));
+        assert!(util_metric.is_some());
+        assert_eq!(util_metric.map(|m| m.value), Some(182.0));
+    }
+
+    #[test]
+    fn test_collect_process_cpu_metrics_with_labels() {
+        let stats = ProcessCpuStats {
+            usage: 25.0,
+            utilization: 100.0,
+        };
+
+        let labels = vec![
+            ("process_pid", Cow::Borrowed("12345")),
+            ("process_executable_name", Cow::Borrowed("rustfs")),
+        ];
+
+        let metrics = collect_process_cpu_metrics(&stats, Some(&labels));
+        assert_eq!(metrics.len(), 2);
+
+        // All metrics should have the labels
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 2);
+        }
+    }
+}

crates/obs/src/metrics/collectors/system_drive.rs

@@ -0,0 +1,271 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System drive metrics collector.
+//!
+//! Collects detailed drive/disk metrics including capacity, I/O statistics,
+//! error counts, and health status.
+//!
+//! This module provides both system-level and process-level disk metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::system_drive::*;
+use crate::metrics::schema::system_process::PROCESS_DISK_IO_MD;
+use std::borrow::Cow;
+
+/// Detailed drive statistics for a single drive.
+#[derive(Debug, Clone, Default)]
+pub struct DriveDetailedStats {
+    /// Server identifier (e.g., "node1:9000")
+    pub server: String,
+    /// Drive path (e.g., "/data/disk1")
+    pub drive: String,
+    /// Total capacity in bytes
+    pub total_bytes: u64,
+    /// Used capacity in bytes
+    pub used_bytes: u64,
+    /// Free capacity in bytes
+    pub free_bytes: u64,
+    /// Used inodes
+    pub used_inodes: u64,
+    /// Free inodes
+    pub free_inodes: u64,
+    /// Total inodes
+    pub total_inodes: u64,
+    /// Total timeout errors
+    pub timeout_errors_total: u64,
+    /// Total I/O errors
+    pub io_errors_total: u64,
+    /// Total availability errors
+    pub availability_errors_total: u64,
+    /// Number of I/O operations waiting
+    pub waiting_io: u64,
+    /// API latency in microseconds
+    pub api_latency_micros: u64,
+    /// Health status (1=healthy, 0=unhealthy)
+    pub health: u8,
+    /// Reads per second
+    pub reads_per_sec: f64,
+    /// Kilobytes read per second
+    pub reads_kb_per_sec: f64,
+    /// Average read await time
+    pub reads_await: f64,
+    /// Writes per second
+    pub writes_per_sec: f64,
+    /// Kilobytes written per second
+    pub writes_kb_per_sec: f64,
+    /// Average write await time
+    pub writes_await: f64,
+    /// Percentage utilization
+    pub perc_util: f64,
+}
+
+/// Aggregate drive count statistics.
+#[derive(Debug, Clone, Default)]
+pub struct DriveCountStats {
+    /// Number of offline Drives
+    pub offline_count: u64,
+    /// Number of online drives
+    pub online_count: u64,
+    /// Total number of drives
+    pub total_count: u64,
+}
+
+/// Collects detailed drive metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for each drive.
+pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
+    fn push_drive_metric(
+        metrics: &mut Vec<PrometheusMetric>,
+        descriptor: &'static crate::metrics::schema::MetricDescriptor,
+        value: f64,
+        server_label: &str,
+        drive_label: &str,
+    ) {
+        metrics.push(
+            PrometheusMetric::from_descriptor(descriptor, value)
+                .with_label_owned(DRIVE_LABEL, drive_label.to_string())
+                .with_label_owned(SERVER_LABEL, server_label.to_string()),
+        );
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 19);
+
+    for stat in stats {
+        let server_label = stat.server.as_str();
+        let drive_label = stat.drive.as_str();
+
+        push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_TIMEOUT_ERRORS_MD,
+            stat.timeout_errors_total as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, stat.io_errors_total as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_AVAILABILITY_ERRORS_MD,
+            stat.availability_errors_total as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, stat.waiting_io as f64, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_API_LATENCY_MD,
+            stat.api_latency_micros as f64,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, stat.reads_per_sec, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, stat.reads_kb_per_sec, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, stat.reads_await, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, stat.writes_per_sec, server_label, drive_label);
+        push_drive_metric(
+            &mut metrics,
+            &DRIVE_WRITES_KB_PER_SEC_MD,
+            stat.writes_kb_per_sec,
+            server_label,
+            drive_label,
+        );
+        push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, stat.writes_await, server_label, drive_label);
+        push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, stat.perc_util, server_label, drive_label);
+    }
+
+    metrics
+}
+
+/// Collects drive count metrics (offline, online, total).
+///
+/// Returns a vector of Prometheus metrics for drive counts.
+pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&DRIVE_OFFLINE_COUNT_MD, stats.offline_count as f64),
+        PrometheusMetric::from_descriptor(&DRIVE_ONLINE_COUNT_MD, stats.online_count as f64),
+        PrometheusMetric::from_descriptor(&DRIVE_COUNT_MD, stats.total_count as f64),
+    ]
+}
+
+/// Process disk I/O statistics.
+///
+/// Contains disk I/O metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessDiskStats {
+    /// Bytes read from disk
+    pub read_bytes: u64,
+    /// Bytes written to disk
+    pub written_bytes: u64,
+}
+
+/// Collects process disk I/O metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process disk I/O statistics.
+/// Each metric includes a `direction` label ("read" or "write").
+///
+/// # Arguments
+///
+/// * `stats` - Process disk I/O statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_disk_metrics(
+    stats: &ProcessDiskStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut read_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.read_bytes as f64);
+    let mut write_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.written_bytes as f64);
+
+    read_metric.labels.push(("direction", Cow::Borrowed("read")));
+    write_metric.labels.push(("direction", Cow::Borrowed("write")));
+
+    if let Some(l) = labels {
+        read_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        write_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![read_metric, write_metric]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_drive_detailed_metrics() {
+        let stats = vec![DriveDetailedStats {
+            server: "node1:9000".to_string(),
+            drive: "/data/disk1".to_string(),
+            total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
+            used_bytes: 1024 * 1024 * 1024 * 50,   // 50 GB
+            free_bytes: 1024 * 1024 * 1024 * 50,   // 50 GB
+            used_inodes: 100000,
+            free_inodes: 900000,
+            total_inodes: 1000000,
+            timeout_errors_total: 5,
+            io_errors_total: 10,
+            availability_errors_total: 2,
+            waiting_io: 3,
+            api_latency_micros: 1500,
+            health: 1,
+            reads_per_sec: 100.0,
+            reads_kb_per_sec: 1024.0,
+            reads_await: 5.5,
+            writes_per_sec: 50.0,
+            writes_kb_per_sec: 512.0,
+            writes_await: 10.2,
+            perc_util: 75.5,
+        }];
+
+        let metrics = collect_drive_detailed_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 19);
+
+        // Verify total bytes metric
+        let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
+        let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
+        assert!(total_bytes.is_some());
+        assert_eq!(total_bytes.map(|m| m.value), Some(1024.0 * 1024.0 * 1024.0 * 100.0));
+    }
+
+    #[test]
+    fn test_collect_drive_count_metrics() {
+        let stats = DriveCountStats {
+            offline_count: 2,
+            online_count: 8,
+            total_count: 10,
+        };
+
+        let metrics = collect_drive_count_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 3);
+
+        // Verify offline count
+        let offline_name = DRIVE_OFFLINE_COUNT_MD.get_full_metric_name();
+        let offline = metrics.iter().find(|m| m.name == offline_name);
+        assert!(offline.is_some());
+        assert_eq!(offline.map(|m| m.value), Some(2.0));
+    }
+}

crates/obs/src/metrics/collectors/system_gpu.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System GPU metrics collector.
+//!
+//! Collects GPU memory usage metrics using NVML library.
+//! This module is only available when `gpu` feature is enabled.
+//!
+//! # Example
+//!
+//! ```ignore
+//! use rustfs_obs::metrics::collectors::{GpuCollector, collect_gpu_metrics};
+//! use sysinfo::Pid;
+//!
+//! let pid = sysinfo::get_current_pid().unwrap();
+//! let collector = GpuCollector::new(pid)?;
+//! let stats = collector.collect()?;
+//! let metrics = collect_gpu_metrics(&stats, &labels);
+//! ```
+
+use crate::metrics::report::PrometheusMetric;
+
+use crate::metrics::schema::system_gpu::PROCESS_GPU_MEMORY_USAGE_MD;
+
+use nvml_wrapper::Nvml;
+
+use nvml_wrapper::enums::device::UsedGpuMemory;
+
+use sysinfo::Pid;
+
+use std::borrow::Cow;
+
+use thiserror::Error;
+
+use tracing::warn;
+
+/// GPU statistics.
+///
+/// Contains GPU memory usage metrics for the monitored process.
+
+#[derive(Debug, Clone, Default)]
+pub struct GpuStats {
+    /// GPU memory usage in bytes
+    pub memory_usage: u64,
+}
+
+/// GPU collector error types.
+
+#[derive(Debug, Error)]
+pub enum GpuError {
+    /// GPU initialization failed
+    #[error("GPU initialization failed: {0}")]
+    InitError(String),
+
+    /// GPU device access error
+    #[error("GPU device error: {0}")]
+    DeviceError(String),
+
+    /// Process not found in GPU process list
+    #[error("Process not found in GPU process list")]
+    ProcessNotFound,
+}
+
+/// GPU metrics collector.
+///
+/// Collects GPU memory usage metrics for a specific process using NVML.
+pub struct GpuCollector {
+    /// NVML instance for GPU access
+    nvml: Nvml,
+    /// Process ID to monitor
+    pid: Pid,
+}
+
+impl GpuCollector {
+    /// Creates a new GPU collector for the specified process.
+    ///
+    /// # Arguments
+    ///
+    /// * `pid` - The process ID to monitor
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if NVML initialization fails.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// use rustfs_obs::metrics::collectors::GpuCollector;
+    /// use sysinfo::Pid;
+    ///
+    /// let pid = sysinfo::get_current_pid().unwrap();
+    /// let collector = GpuCollector::new(pid)?;
+    /// ```
+    pub fn new(pid: Pid) -> Result<Self, GpuError> {
+        let nvml = Nvml::init().map_err(|e| GpuError::InitError(e.to_string()))?;
+        Ok(GpuCollector { nvml, pid })
+    }
+
+    /// Collects GPU metrics for the monitored process.
+    ///
+    /// Returns GPU memory usage statistics for the process.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if GPU device access fails.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// let stats = collector.collect()?;
+    /// println!("GPU memory usage: {} bytes", stats.memory_usage);
+    /// ```
+    pub fn collect(&self) -> Result<GpuStats, GpuError> {
+        if let Ok(device) = self.nvml.device_by_index(0) {
+            if let Ok(gpu_stats) = device.running_compute_processes() {
+                for stat in gpu_stats.iter() {
+                    if stat.pid == self.pid.as_u32() {
+                        let memory_used = match stat.used_gpu_memory {
+                            UsedGpuMemory::Used(bytes) => bytes,
+                            UsedGpuMemory::Unavailable => 0,
+                        };
+                        return Ok(GpuStats {
+                            memory_usage: memory_used,
+                        });
+                    }
+                }
+            } else {
+                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
+            }
+        } else {
+            return Err(GpuError::DeviceError("No GPU device found".to_string()));
+        }
+
+        // Process not found in GPU process list, return 0 usage
+        Ok(GpuStats { memory_usage: 0 })
+    }
+}
+
+/// Converts GPU stats to Prometheus metrics.
+///
+/// # Arguments
+///
+/// * `stats` - GPU statistics to convert
+/// * `labels` - Metric labels (typically from ProcessAttributes)
+///
+/// # Returns
+///
+/// A vector of Prometheus metrics.
+///
+/// # Example
+///
+/// ```ignore
+/// use rustfs_obs::metrics::collectors::{GpuStats, collect_gpu_metrics};
+///
+/// let stats = GpuStats { memory_usage: 1024 };
+/// let labels = vec![("process_pid", Cow::Borrowed("1234"))];
+/// let metrics = collect_gpu_metrics(&stats, &labels);
+/// ```
+pub fn collect_gpu_metrics(stats: &GpuStats, labels: &[(&'static str, Cow<'static, str>)]) -> Vec<PrometheusMetric> {
+    let mut metric = PrometheusMetric::from_descriptor(&PROCESS_GPU_MEMORY_USAGE_MD, stats.memory_usage as f64);
+    metric.labels.extend(labels.iter().map(|(k, v)| (*k, v.clone())));
+    vec![metric]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_gpu_stats_default() {
+        let stats = GpuStats::default();
+        assert_eq!(stats.memory_usage, 0);
+    }
+
+    #[test]
+    fn test_gpu_error_display() {
+        let err = GpuError::InitError("test error".to_string());
+        assert!(err.to_string().contains("test error"));
+
+        let err = GpuError::DeviceError("device error".to_string());
+        assert!(err.to_string().contains("device error"));
+
+        let err = GpuError::ProcessNotFound;
+        assert!(err.to_string().contains("Process not found"));
+    }
+}

crates/obs/src/metrics/collectors/system_memory.rs

@@ -0,0 +1,169 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System memory metrics collector.
+//!
+//! Collects memory-related metrics including total, used, free,
+//! buffers, cache, shared, and available memory.
+//!
+//! This module provides both system-level and process-level memory metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::system_memory::*;
+use crate::metrics::schema::system_process::{PROCESS_RESIDENT_MEMORY_BYTES_MD, PROCESS_VIRTUAL_MEMORY_BYTES_MD};
+use std::borrow::Cow;
+
+/// System memory statistics.
+#[derive(Debug, Clone, Default)]
+pub struct MemoryStats {
+    /// Total memory in bytes
+    pub total: u64,
+    /// Used memory in bytes
+    pub used: u64,
+    /// Used memory percentage (0-100)
+    pub used_perc: f64,
+    /// Free memory in bytes
+    pub free: u64,
+    /// Buffer memory in bytes
+    pub buffers: u64,
+    /// Cache memory in bytes
+    pub cache: u64,
+    /// Shared memory in bytes
+    pub shared: u64,
+    /// Available memory in bytes
+    pub available: u64,
+}
+
+/// Process memory statistics.
+///
+/// Contains memory usage metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessMemoryStats {
+    /// Resident memory size in bytes
+    pub resident: u64,
+    /// Virtual memory size in bytes
+    pub virtual_mem: u64,
+}
+
+/// Collects memory metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_memory` module.
+/// Returns a vector of Prometheus metrics for memory statistics.
+pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64),
+        PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64),
+        PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc),
+        PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64),
+        PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64),
+        PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64),
+        PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64),
+        PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64),
+    ]
+}
+
+/// Collects process memory metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_process` module.
+/// Returns a vector of Prometheus metrics for process memory statistics.
+///
+/// # Arguments
+///
+/// * `stats` - Process memory statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_memory_metrics(
+    stats: &ProcessMemoryStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut resident_metric = PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident as f64);
+    let mut virtual_metric = PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_mem as f64);
+
+    if let Some(l) = labels {
+        resident_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        virtual_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    vec![resident_metric, virtual_metric]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_memory_metrics() {
+        let stats = MemoryStats {
+            total: 16 * 1024 * 1024 * 1024, // 16 GB
+            used: 8 * 1024 * 1024 * 1024,   // 8 GB
+            used_perc: 50.0,
+            free: 4 * 1024 * 1024 * 1024,      // 4 GB
+            buffers: 1024 * 1024 * 512,        // 512 MB
+            cache: 2 * 1024 * 1024 * 1024,     // 2 GB
+            shared: 1024 * 1024 * 256,         // 256 MB
+            available: 6 * 1024 * 1024 * 1024, // 6 GB
+        };
+
+        let metrics = collect_memory_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 8);
+        assert!(metrics.iter().all(|m| m.name.starts_with("rustfs_system_memory_")));
+    }
+
+    #[test]
+    fn test_collect_memory_metrics_default() {
+        let stats = MemoryStats::default();
+        let metrics = collect_memory_metrics(&stats);
+
+        assert_eq!(metrics.len(), 8);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_collect_process_memory_metrics() {
+        let stats = ProcessMemoryStats {
+            resident: 512 * 1024 * 1024,         // 512 MB
+            virtual_mem: 2 * 1024 * 1024 * 1024, // 2 GB
+        };
+
+        let metrics = collect_process_memory_metrics(&stats, None);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 2);
+    }
+
+    #[test]
+    fn test_collect_process_memory_metrics_with_labels() {
+        let stats = ProcessMemoryStats {
+            resident: 256 * 1024 * 1024,
+            virtual_mem: 1024 * 1024 * 1024,
+        };
+
+        let labels = vec![("process_pid", Cow::Borrowed("12345"))];
+
+        let metrics = collect_process_memory_metrics(&stats, Some(&labels));
+        assert_eq!(metrics.len(), 2);
+
+        for metric in &metrics {
+            assert_eq!(metric.labels.len(), 1);
+        }
+    }
+}

crates/obs/src/metrics/collectors/system_network.rs

@@ -0,0 +1,159 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System network metrics collector.
+//!
+//! Collects internode network metrics including errors, dial times,
+//! and bytes sent/received.
+//!
+//! This module provides both system-level and process-level network metrics,
+//! with process-level metrics migrated from `rustfs-obs::system`.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::system_network::*;
+use crate::metrics::schema::system_process::{PROCESS_NETWORK_IO_MD, PROCESS_NETWORK_IO_PER_INTERFACE_MD};
+use std::borrow::Cow;
+
+/// Network statistics for internode communication.
+#[derive(Debug, Clone, Default)]
+pub struct NetworkStats {
+    /// Total number of failed internode calls
+    pub internode_errors_total: u64,
+    /// Total number of TCP dial timeouts and errors
+    pub internode_dial_errors_total: u64,
+    /// Average dial time in nanoseconds
+    pub internode_dial_avg_time_nanos: u64,
+    /// Total bytes sent to other nodes
+    pub internode_sent_bytes_total: u64,
+    /// Total bytes received from other nodes
+    pub internode_recv_bytes_total: u64,
+}
+
+/// Process network I/O statistics.
+///
+/// Contains network I/O metrics for a specific process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessNetworkStats {
+    /// Total bytes received
+    pub total_received: u64,
+    /// Total bytes transmitted
+    pub total_transmitted: u64,
+    /// Per-interface statistics: (interface_name, received_bytes, transmitted_bytes)
+    pub per_interface: Vec<(String, u64, u64)>,
+}
+
+/// Collects network metrics from the given stats.
+///
+/// Uses the metric descriptors from `metrics_type::system_network` module.
+/// Returns a vector of Prometheus metrics for network statistics.
+pub fn collect_network_metrics(stats: &NetworkStats) -> Vec<PrometheusMetric> {
+    vec![
+        PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64),
+        PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64),
+        PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64),
+        PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64),
+        PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64),
+    ]
+}
+
+/// Collects process network I/O metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process network I/O statistics.
+/// Each metric includes a `direction` label ("received" or "transmitted").
+/// Per-interface metrics also include an `interface` label.
+///
+/// # Arguments
+///
+/// * `stats` - Process network I/O statistics
+/// * `labels` - Optional additional labels (e.g., process attributes)
+pub fn collect_process_network_metrics(
+    stats: &ProcessNetworkStats,
+    labels: Option<&[(&'static str, Cow<'static, str>)]>,
+) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(2 + stats.per_interface.len() * 2);
+
+    // Total network I/O
+    let mut received_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_received as f64);
+    let mut transmitted_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_transmitted as f64);
+
+    received_metric.labels.push(("direction", Cow::Borrowed("received")));
+    transmitted_metric.labels.push(("direction", Cow::Borrowed("transmitted")));
+
+    if let Some(l) = labels {
+        received_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        transmitted_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+    }
+
+    metrics.push(received_metric);
+    metrics.push(transmitted_metric);
+
+    // Per-interface network I/O
+    for (interface, received, transmitted) in &stats.per_interface {
+        let mut iface_received = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *received as f64);
+        let mut iface_transmitted = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *transmitted as f64);
+
+        iface_received.labels.push(("interface", Cow::Owned(interface.clone())));
+        iface_received.labels.push(("direction", Cow::Borrowed("received")));
+
+        iface_transmitted.labels.push(("interface", Cow::Owned(interface.clone())));
+        iface_transmitted.labels.push(("direction", Cow::Borrowed("transmitted")));
+
+        if let Some(l) = labels {
+            iface_received.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+            iface_transmitted.labels.extend(l.iter().map(|(k, v)| (*k, v.clone())));
+        }
+
+        metrics.push(iface_received);
+        metrics.push(iface_transmitted);
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_network_metrics() {
+        let stats = NetworkStats {
+            internode_errors_total: 10,
+            internode_dial_errors_total: 5,
+            internode_dial_avg_time_nanos: 1_500_000,      // 1.5ms
+            internode_sent_bytes_total: 1024 * 1024 * 100, // 100 MB
+            internode_recv_bytes_total: 1024 * 1024 * 200, // 200 MB
+        };
+
+        let metrics = collect_network_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 5);
+        assert!(metrics.iter().all(|m| m.name.contains("internode")));
+    }
+
+    #[test]
+    fn test_collect_network_metrics_default() {
+        let stats = NetworkStats::default();
+        let metrics = collect_network_metrics(&stats);
+
+        assert_eq!(metrics.len(), 5);
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+}

crates/obs/src/metrics/collectors/system_process.rs

@@ -0,0 +1,325 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! System process metrics collector.
+//!
+//! Collects process-level metrics including file descriptors, memory,
+//! syscalls, and runtime statistics.
+//!
+//! This module also provides process attribute collection for use as
+//! metric labels, migrated from `rustfs-obs::system`.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::system_process::*;
+use std::borrow::Cow;
+use sysinfo::{Pid, ProcessStatus, System};
+
+/// Process attributes used as metric labels.
+///
+/// Contains identifying information about the process being monitored.
+#[derive(Debug, Clone)]
+pub struct ProcessAttributes {
+    /// Process ID
+    pub pid: u32,
+    /// Executable name (e.g., "rustfs")
+    pub executable_name: String,
+    /// Full path to the executable
+    pub executable_path: String,
+    /// Full command line with arguments
+    pub command: String,
+}
+
+impl ProcessAttributes {
+    /// Creates a new instance by reading from the current process.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the current process PID cannot be determined
+    /// or if process information cannot be retrieved.
+    pub fn current() -> Result<Self, ProcessAttributeError> {
+        let pid = sysinfo::get_current_pid().map_err(|e| ProcessAttributeError::PidError(e.to_string()))?;
+        Self::from_pid(pid)
+    }
+
+    /// Creates a new instance for a specific PID.
+    ///
+    /// # Arguments
+    ///
+    /// * `pid` - The process ID to query
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the process does not exist or information
+    /// cannot be retrieved.
+    pub fn from_pid(pid: Pid) -> Result<Self, ProcessAttributeError> {
+        let mut system = System::new();
+        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+        let process = system
+            .process(pid)
+            .ok_or_else(|| ProcessAttributeError::ProcessNotFound(pid.as_u32()))?;
+
+        Ok(ProcessAttributes {
+            pid: pid.as_u32(),
+            executable_name: process.name().to_string_lossy().to_string(),
+            executable_path: process.exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_default(),
+            command: process
+                .cmd()
+                .iter()
+                .map(|s| s.to_string_lossy().into_owned())
+                .collect::<Vec<_>>()
+                .join(" "),
+        })
+    }
+
+    /// Converts attributes to Prometheus metric labels.
+    pub fn to_labels(&self) -> Vec<(&'static str, Cow<'static, str>)> {
+        vec![
+            ("process_pid", Cow::Owned(self.pid.to_string())),
+            ("process_executable_name", Cow::Owned(self.executable_name.clone())),
+            ("process_executable_path", Cow::Owned(self.executable_path.clone())),
+            ("process_command", Cow::Owned(self.command.clone())),
+        ]
+    }
+}
+
+/// Errors that can occur when collecting process attributes.
+#[derive(Debug, Clone)]
+pub enum ProcessAttributeError {
+    /// Failed to get current process PID
+    PidError(String),
+    /// Process not found
+    ProcessNotFound(u32),
+}
+
+impl std::fmt::Display for ProcessAttributeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::PidError(e) => write!(f, "Failed to get current PID: {}", e),
+            Self::ProcessNotFound(pid) => write!(f, "Process not found: {}", pid),
+        }
+    }
+}
+
+impl std::error::Error for ProcessAttributeError {}
+
+/// Process status enumeration.
+///
+/// Maps `sysinfo::ProcessStatus` to a simpler representation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ProcessStatusType {
+    /// Process is currently running
+    Running = 0,
+    /// Process is sleeping (waiting for I/O or event)
+    Sleeping = 1,
+    /// Process is a zombie (terminated but not reaped)
+    Zombie = 2,
+    /// Process is in some other state
+    #[default]
+    Other = 3,
+}
+
+impl From<ProcessStatus> for ProcessStatusType {
+    fn from(status: ProcessStatus) -> Self {
+        match status {
+            ProcessStatus::Run => ProcessStatusType::Running,
+            ProcessStatus::Sleep => ProcessStatusType::Sleeping,
+            ProcessStatus::Zombie => ProcessStatusType::Zombie,
+            _ => ProcessStatusType::Other,
+        }
+    }
+}
+
+/// Process statistics for the RustFS server process.
+#[derive(Debug, Clone, Default)]
+pub struct ProcessStats {
+    /// Total read locks held
+    pub locks_read_total: u64,
+    /// Total write locks held
+    pub locks_write_total: u64,
+    /// Total CPU time in seconds
+    pub cpu_total_seconds: f64,
+    /// Total number of async tasks (goroutines equivalent)
+    pub go_routine_total: u64,
+    /// Total bytes read via read syscalls (rchar)
+    pub io_rchar_bytes: u64,
+    /// Total bytes actually read from storage
+    pub io_read_bytes: u64,
+    /// Total bytes written via write syscalls (wchar)
+    pub io_wchar_bytes: u64,
+    /// Total bytes actually written to storage
+    pub io_write_bytes: u64,
+    /// Process start time in seconds since Unix epoch
+    pub start_time_seconds: u64,
+    /// Process uptime in seconds
+    pub uptime_seconds: u64,
+    /// File descriptor limit
+    pub file_descriptor_limit_total: u64,
+    /// Open file descriptors count
+    pub file_descriptor_open_total: u64,
+    /// Total read syscalls
+    pub syscall_read_total: u64,
+    /// Total write syscalls
+    pub syscall_write_total: u64,
+    /// Resident memory size in bytes
+    pub resident_memory_bytes: u64,
+    /// Virtual memory size in bytes
+    pub virtual_memory_bytes: u64,
+    /// Maximum virtual memory size in bytes
+    pub virtual_memory_max_bytes: u64,
+    /// Process status
+    pub status: ProcessStatusType,
+    /// Process status value (numeric)
+    pub status_value: i64,
+}
+
+/// Collects process metrics from the given stats.
+///
+/// Returns a vector of Prometheus metrics for process statistics.
+pub fn collect_process_metrics(stats: &ProcessStats) -> Vec<PrometheusMetric> {
+    let mut metrics = vec![
+        PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds),
+        PrometheusMetric::from_descriptor(&PROCESS_GO_ROUTINE_TOTAL_MD, stats.go_routine_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_IO_RCHAR_BYTES_MD, stats.io_rchar_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_IO_READ_BYTES_MD, stats.io_read_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_IO_WCHAR_BYTES_MD, stats.io_wchar_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_IO_WRITE_BYTES_MD, stats.io_write_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_START_TIME_SECONDS_MD, stats.start_time_seconds as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD, stats.file_descriptor_limit_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD, stats.file_descriptor_open_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_READ_TOTAL_MD, stats.syscall_read_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_WRITE_TOTAL_MD, stats.syscall_write_total as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64),
+        PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64),
+    ];
+
+    // Add process status metric
+    let mut status_metric = PrometheusMetric::from_descriptor(&PROCESS_STATUS_MD, stats.status_value as f64);
+    status_metric
+        .labels
+        .push(("status", Cow::Owned(format!("{:?}", stats.status))));
+    metrics.push(status_metric);
+
+    metrics
+}
+
+/// Collects process attributes for the current process.
+///
+/// This is a convenience function that wraps `ProcessAttributes::current()`.
+pub fn collect_process_attributes() -> Result<ProcessAttributes, ProcessAttributeError> {
+    ProcessAttributes::current()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::report::report_metrics;
+
+    #[test]
+    fn test_collect_process_metrics() {
+        let stats = ProcessStats {
+            locks_read_total: 100,
+            locks_write_total: 50,
+            cpu_total_seconds: 1234.56,
+            go_routine_total: 200,
+            io_rchar_bytes: 1024 * 1024 * 500,
+            io_read_bytes: 1024 * 1024 * 400,
+            io_wchar_bytes: 1024 * 1024 * 300,
+            io_write_bytes: 1024 * 1024 * 250,
+            start_time_seconds: 1700000000,
+            uptime_seconds: 86400,
+            file_descriptor_limit_total: 65536,
+            file_descriptor_open_total: 1500,
+            syscall_read_total: 100000,
+            syscall_write_total: 50000,
+            resident_memory_bytes: 1024 * 1024 * 512,
+            virtual_memory_bytes: 1024 * 1024 * 1024,
+            virtual_memory_max_bytes: 1024 * 1024 * 2048,
+            status: ProcessStatusType::Running,
+            status_value: 0,
+        };
+
+        let metrics = collect_process_metrics(&stats);
+        report_metrics(&metrics);
+
+        // 17 original metrics + 1 status metric = 18
+        assert_eq!(metrics.len(), 18);
+
+        // Verify uptime
+        let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
+        let uptime = metrics.iter().find(|m| m.name == uptime_name);
+        assert!(uptime.is_some());
+        assert_eq!(uptime.map(|m| m.value), Some(86400.0));
+
+        // Verify file descriptors
+        let fd_open_name = PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD.get_full_metric_name();
+        let fd_open = metrics.iter().find(|m| m.name == fd_open_name);
+        assert!(fd_open.is_some());
+        assert_eq!(fd_open.map(|m| m.value), Some(1500.0));
+
+        // Verify status metric
+        let status_name = PROCESS_STATUS_MD.get_full_metric_name();
+        let status_metric = metrics.iter().find(|m| m.name == status_name);
+        assert!(status_metric.is_some());
+        assert_eq!(status_metric.map(|m| m.value), Some(0.0));
+    }
+
+    #[test]
+    fn test_collect_process_metrics_default() {
+        let stats = ProcessStats::default();
+        let metrics = collect_process_metrics(&stats);
+
+        // 17 original metrics + 1 status metric = 18
+        assert_eq!(metrics.len(), 18);
+    }
+
+    #[test]
+    fn test_process_attributes_current() {
+        // This test should succeed as we're querying the current process
+        let result = collect_process_attributes();
+        assert!(result.is_ok());
+
+        let attrs = result.unwrap();
+        assert!(attrs.pid > 0);
+        assert!(!attrs.executable_name.is_empty());
+    }
+
+    #[test]
+    fn test_process_status_conversion() {
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Run), ProcessStatusType::Running);
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Sleep), ProcessStatusType::Sleeping);
+        assert_eq!(ProcessStatusType::from(ProcessStatus::Zombie), ProcessStatusType::Zombie);
+    }
+
+    #[test]
+    fn test_process_attributes_to_labels() {
+        let attrs = ProcessAttributes {
+            pid: 12345,
+            executable_name: "rustfs".to_string(),
+            executable_path: "/usr/bin/rustfs".to_string(),
+            command: "rustfs server /data".to_string(),
+        };
+
+        let labels = attrs.to_labels();
+        assert_eq!(labels.len(), 4);
+        assert_eq!(labels[0].0, "process_pid");
+        assert_eq!(labels[0].1, "12345");
+    }
+}

crates/obs/src/metrics/config.rs

@@ -0,0 +1,57 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+/// Environment variable key for the global default metrics interval (seconds).
+pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
+/// Default interval for metrics collection if not specified otherwise.
+#[allow(dead_code)]
+pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for cluster metrics interval (seconds).
+pub const ENV_CLUSTER_METRICS_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL_SEC";
+/// Default interval for collecting cluster-wide metrics (capacity, object counts).
+pub const DEFAULT_CLUSTER_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for bucket metrics interval (seconds).
+pub const ENV_BUCKET_METRICS_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL_SEC";
+/// Default interval for collecting per-bucket metrics (usage, quotas).
+/// This can be expensive if there are many buckets, so a longer interval is recommended.
+pub const DEFAULT_BUCKET_METRICS_INTERVAL: Duration = Duration::from_secs(300);
+
+/// Environment variable key for node metrics interval (seconds).
+pub const ENV_NODE_METRICS_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL_SEC";
+/// Default interval for collecting node/disk metrics.
+pub const DEFAULT_NODE_METRICS_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Environment variable key for resource metrics interval (seconds).
+pub const ENV_RESOURCE_METRICS_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL_SEC";
+/// Default interval for collecting system resource metrics (CPU, memory).
+pub const DEFAULT_RESOURCE_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Environment variable key for audit target metrics interval (seconds).
+pub const ENV_AUDIT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_AUDIT_INTERVAL_SEC";
+/// Default interval for collecting audit target delivery metrics.
+pub const DEFAULT_AUDIT_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Environment variable key for notification metrics interval (seconds).
+pub const ENV_NOTIFICATION_METRICS_INTERVAL: &str = "RUSTFS_METRICS_NOTIFICATION_INTERVAL_SEC";
+/// Default interval for collecting notification delivery metrics.
+pub const DEFAULT_NOTIFICATION_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Environment variable key for replication bandwidth metrics interval (seconds).
+pub const ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC";
+/// Default interval for collecting replication bandwidth metrics.
+pub const DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL: Duration = Duration::from_secs(30);

crates/obs/src/metrics/mod.rs

@@ -0,0 +1,25 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod collectors;
+pub mod config;
+pub mod report;
+pub mod scheduler;
+pub mod schema;
+pub mod stats_collector;
+
+pub use collectors::*;
+pub use config::*;
+pub use report::{PrometheusMetric, report_metrics};
+pub use scheduler::{init_metrics_collectors, init_metrics_runtime};

crates/obs/src/metrics/report.rs

@@ -0,0 +1,171 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::metrics::schema::{MetricDescriptor, MetricType};
+use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge};
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+
+static NAME_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
+static HELP_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
+
+fn intern_string(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
+    let cache = cache.get_or_init(Default::default);
+    let mut cache = cache.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+
+    if let Some(existing) = cache.get(value) {
+        existing
+    } else {
+        let value = Box::leak(value.to_string().into_boxed_str());
+        cache.insert(value.to_string(), value);
+        value
+    }
+}
+
+fn into_static_str(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
+    intern_string(cache, value)
+}
+
+pub fn report_metrics(metrics: &[PrometheusMetric]) {
+    for metric in metrics {
+        let name = into_static_str(&NAME_CACHE, &metric.name);
+        let help = into_static_str(&HELP_CACHE, &metric.help);
+
+        match metric.metric_type {
+            MetricType::Counter => describe_counter!(name, help),
+            MetricType::Gauge => describe_gauge!(name, help),
+            MetricType::Histogram => describe_histogram!(name, help),
+        }
+
+        let labels: Vec<(String, String)> = metric.labels.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
+
+        match metric.metric_type {
+            MetricType::Counter => {
+                let counter = counter!(name, &labels);
+                counter.absolute(metric.value as u64);
+            }
+            MetricType::Gauge => {
+                let gauge = gauge!(name, &labels);
+                gauge.set(metric.value);
+            }
+            MetricType::Histogram => {
+                let histogram = metrics::histogram!(name, &labels);
+                histogram.record(metric.value);
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrometheusMetric {
+    pub name: Cow<'static, str>,
+    pub metric_type: MetricType,
+    pub help: Cow<'static, str>,
+    pub labels: Vec<(&'static str, Cow<'static, str>)>,
+    pub value: f64,
+}
+
+impl PrometheusMetric {
+    #[inline]
+    pub const fn new(name: &'static str, metric_type: MetricType, help: &'static str, value: f64) -> Self {
+        Self {
+            name: Cow::Borrowed(name),
+            metric_type,
+            help: Cow::Borrowed(help),
+            labels: Vec::new(),
+            value,
+        }
+    }
+
+    #[inline]
+    pub fn new_owned(name: String, metric_type: MetricType, help: String, value: f64) -> Self {
+        Self {
+            name: Cow::Owned(name),
+            metric_type,
+            help: Cow::Owned(help),
+            labels: Vec::new(),
+            value,
+        }
+    }
+
+    #[inline]
+    pub fn from_descriptor(descriptor: &MetricDescriptor, value: f64) -> Self {
+        let help = intern_string(&HELP_CACHE, &descriptor.help);
+        Self {
+            name: Cow::Owned(descriptor.get_full_metric_name()),
+            metric_type: descriptor.metric_type,
+            help: Cow::Borrowed(help),
+            labels: Vec::new(),
+            value,
+        }
+    }
+
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
+        self.labels.push((key, value.into()));
+        self
+    }
+
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
+        self.labels.push((key, Cow::Owned(value)));
+        self
+    }
+
+    #[inline]
+    #[allow(dead_code)]
+    pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
+        self.labels = labels;
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::schema::{MetricName, MetricNamespace, MetricSubsystem};
+
+    #[test]
+    fn from_descriptor_uses_prometheus_metric_names_for_all_types() {
+        let cases = [
+            (MetricType::Counter, "rustfs_api_requests_total"),
+            (MetricType::Gauge, "rustfs_system_memory_used_bytes"),
+            (MetricType::Histogram, "rustfs_custom_path_latency_seconds"),
+        ];
+
+        for (metric_type, expected_name) in cases {
+            let subsystem = match metric_type {
+                MetricType::Counter => MetricSubsystem::ApiRequests,
+                MetricType::Gauge => MetricSubsystem::SystemMemory,
+                MetricType::Histogram => MetricSubsystem::new("/custom/path"),
+            };
+            let name = match metric_type {
+                MetricType::Counter => MetricName::ApiRequestsTotal,
+                MetricType::Gauge => MetricName::Custom("used_bytes".to_string()),
+                MetricType::Histogram => MetricName::Custom("latency_seconds".to_string()),
+            };
+
+            let metric = PrometheusMetric::from_descriptor(
+                &MetricDescriptor::new(name, metric_type, "test help".to_string(), vec![], MetricNamespace::RustFS, subsystem),
+                1.0,
+            );
+
+            assert_eq!(metric.name, expected_name);
+            assert_eq!(metric.metric_type, metric_type);
+        }
+    }
+}

crates/obs/src/metrics/scheduler.rs

@@ -0,0 +1,440 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Global metrics collector initialization.
+//!
+//! This module provides the entry point for initializing all metrics collectors.
+//! The actual statistics collection functions are in `stats_collector.rs`.
+//!
+//! System monitoring collectors (migrated from `rustfs-obs::system`):
+//! - Process CPU metrics
+//! - Process memory metrics
+//! - Process disk I/O metrics
+//! - Process network I/O metrics
+
+use crate::metrics::collectors::{
+    AuditTargetStats,
+    NotificationStats,
+    NotificationTargetStats,
+    // System monitoring collectors (migrated from rustfs-obs::system)
+    ProcessCpuStats,
+    ProcessDiskStats,
+    ProcessMemoryStats,
+    ProcessNetworkStats,
+    collect_audit_metrics,
+    collect_bucket_metrics,
+    collect_bucket_replication_bandwidth_metrics,
+    collect_cluster_metrics,
+    collect_node_metrics,
+    collect_notification_metrics,
+    collect_notification_target_metrics,
+    collect_process_cpu_metrics,
+    collect_process_disk_metrics,
+    collect_process_memory_metrics,
+    collect_process_metrics,
+    collect_process_network_metrics,
+    collect_resource_metrics,
+};
+use crate::metrics::config::{
+    DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+    DEFAULT_CLUSTER_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL, DEFAULT_NOTIFICATION_METRICS_INTERVAL,
+    DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_AUDIT_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL,
+    ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
+    ENV_NODE_METRICS_INTERVAL, ENV_NOTIFICATION_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
+};
+use crate::metrics::report::report_metrics;
+use crate::metrics::stats_collector::{
+    collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats,
+    collect_process_resource_and_system_stats,
+};
+use rustfs_audit::audit_target_metrics;
+use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
+use rustfs_utils::get_env_opt_u64;
+use std::borrow::Cow;
+use std::time::Duration;
+use sysinfo::{Pid, System};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+
+/// Default interval for system monitoring metrics (15 seconds)
+const DEFAULT_SYSTEM_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+/// Environment variable for system monitoring interval
+const ENV_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_METRICS_SYSTEM_INTERVAL_SEC";
+/// Legacy environment variable for system monitoring interval
+const LEGACY_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_OBS_METRICS_SYSTEM_INTERVAL_MS";
+
+/// Initialize all metrics collectors.
+///
+/// This function spawns background tasks that periodically collect metrics
+/// from various sources and report them to the metrics system.
+///
+/// # Arguments
+/// * `token` - A `CancellationToken` that can be used to gracefully shut down
+///   all metrics collection tasks.
+///
+/// # Environment Variables
+/// The collection intervals can be configured via environment variables:
+/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60)
+/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300)
+/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60)
+/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30)
+/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15)
+/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds.
+///
+/// Legacy interval names without `_SEC` are still accepted for backward compatibility:
+/// - `RUSTFS_METRICS_CLUSTER_INTERVAL`
+/// - `RUSTFS_METRICS_BUCKET_INTERVAL`
+/// - `RUSTFS_METRICS_NODE_INTERVAL`
+/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL`
+/// - `RUSTFS_METRICS_RESOURCE_INTERVAL`
+pub fn init_metrics_runtime(token: CancellationToken) {
+    const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL";
+    const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL";
+    const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL";
+    const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL";
+    const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
+    const LEGACY_AUDIT_INTERVAL: &str = "RUSTFS_METRICS_AUDIT_INTERVAL";
+    const LEGACY_NOTIFICATION_INTERVAL: &str = "RUSTFS_METRICS_NOTIFICATION_INTERVAL";
+    const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
+
+    /// Parse metrics interval from environment variables with fallback to default.
+    ///
+    /// Priority: primary_env > legacy_env > default_env > legacy_default > default_value
+    fn parse_metrics_interval(primary_env: &str, legacy_env: &str, default_interval: Duration) -> Duration {
+        get_env_opt_u64(primary_env)
+            .or_else(|| get_env_opt_u64(legacy_env))
+            .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
+            .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
+            .filter(|&v| v > 0)
+            .map(Duration::from_secs)
+            .unwrap_or(default_interval)
+    }
+
+    // Read intervals from environment or use defaults
+    let cluster_interval =
+        parse_metrics_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
+
+    let bucket_interval =
+        parse_metrics_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
+
+    let bucket_replication_bandwidth_interval = parse_metrics_interval(
+        ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+        LEGACY_REPLICATION_BANDWIDTH_INTERVAL,
+        DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+    );
+
+    let node_interval = parse_metrics_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
+
+    let resource_interval =
+        parse_metrics_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
+    let audit_interval =
+        parse_metrics_interval(ENV_AUDIT_METRICS_INTERVAL, LEGACY_AUDIT_INTERVAL, DEFAULT_AUDIT_METRICS_INTERVAL);
+    let notification_interval = parse_metrics_interval(
+        ENV_NOTIFICATION_METRICS_INTERVAL,
+        LEGACY_NOTIFICATION_INTERVAL,
+        DEFAULT_NOTIFICATION_METRICS_INTERVAL,
+    );
+
+    // Spawn task for cluster metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(cluster_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_cluster_stats().await;
+                    let metrics = collect_cluster_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for cluster stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for bucket metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(bucket_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_bucket_stats().await;
+                    let metrics = collect_bucket_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for bucket stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for node/disk metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(node_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_disk_stats().await;
+                    let metrics = collect_node_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for node/disk stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for bucket replication bandwidth metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(bucket_replication_bandwidth_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_bucket_replication_bandwidth_stats();
+                    let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for bucket replication bandwidth stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for resource metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(resource_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let (resource_stats, process_stats) = collect_process_resource_and_system_stats();
+                    let mut metrics = collect_resource_metrics(&resource_stats);
+                    metrics.extend(collect_process_metrics(&process_stats));
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for resource stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for audit target delivery metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(audit_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = audit_target_metrics().await
+                        .into_iter()
+                        .map(|snapshot| AuditTargetStats {
+                            failed_messages: snapshot.failed_messages,
+                            queue_length: snapshot.queue_length,
+                            target_id: snapshot.target_id,
+                            total_messages: snapshot.total_messages,
+                        })
+                        .collect::<Vec<_>>();
+                    let metrics = collect_audit_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for audit target stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for notification delivery metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(notification_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let snapshot = notification_metrics_snapshot();
+                    let mut metrics = collect_notification_metrics(&NotificationStats {
+                        current_send_in_progress: snapshot.current_send_in_progress,
+                        events_errors_total: snapshot.events_errors_total,
+                        events_sent_total: snapshot.events_sent_total,
+                        events_skipped_total: snapshot.events_skipped_total,
+                    });
+
+                    let target_stats = notification_target_metrics().await
+                        .into_iter()
+                        .map(|snapshot| NotificationTargetStats {
+                            failed_messages: snapshot.failed_messages,
+                            queue_length: snapshot.queue_length,
+                            target_id: snapshot.target_id,
+                            target_type: snapshot.target_type,
+                            total_messages: snapshot.total_messages,
+                        })
+                        .collect::<Vec<_>>();
+                    metrics.extend(collect_notification_target_metrics(&target_stats));
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for notification stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for system monitoring metrics (migrated from rustfs-obs::system)
+    let system_interval = get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL)
+        .or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| ms / 1000)) // Convert ms to seconds
+        .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
+        .filter(|&v| v > 0)
+        .map(Duration::from_secs)
+        .unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL);
+
+    let token_clone = token;
+    tokio::spawn(async move {
+        // Get current process PID
+        let pid = match sysinfo::get_current_pid() {
+            Ok(p) => p,
+            Err(e) => {
+                warn!("Failed to get current PID for system monitoring: {}", e);
+                return;
+            }
+        };
+
+        let mut interval = tokio::time::interval(system_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    // Collect system monitoring metrics
+                    let metrics = collect_system_monitoring_metrics(pid);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("System monitoring metrics collection cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+}
+
+/// Backward-compatible alias kept during migration.
+pub fn init_metrics_collectors(token: CancellationToken) {
+    init_metrics_runtime(token);
+}
+
+/// Collect all system monitoring metrics for a process.
+///
+/// This function collects CPU, memory, disk I/O, and network I/O metrics
+/// for the specified process PID.
+///
+/// # Arguments
+/// * `pid` - The process ID to monitor
+///
+/// # Returns
+/// A vector of Prometheus metrics for the process.
+fn collect_system_monitoring_metrics(pid: Pid) -> Vec<crate::metrics::report::PrometheusMetric> {
+    let mut metrics = Vec::new();
+    let mut system = System::new();
+
+    // Refresh process information
+    system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+    if let Some(process) = system.process(pid) {
+        // Create labels with process attributes
+        let labels: Vec<(&'static str, Cow<'static, str>)> = vec![
+            ("process_pid", Cow::Owned(pid.as_u32().to_string())),
+            ("process_executable_name", Cow::Owned(process.name().to_string_lossy().to_string())),
+        ];
+
+        // Collect CPU metrics
+        let cpu_stats = ProcessCpuStats {
+            usage: process.cpu_usage() as f64,
+            utilization: process.cpu_usage() as f64, // Same as usage for single process
+        };
+        metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(&labels)));
+
+        // Collect memory metrics
+        let memory_stats = ProcessMemoryStats {
+            resident: process.memory(),
+            virtual_mem: process.virtual_memory(),
+        };
+        metrics.extend(collect_process_memory_metrics(&memory_stats, Some(&labels)));
+
+        // Collect disk I/O metrics
+        let disk_usage = process.disk_usage();
+        let disk_stats = ProcessDiskStats {
+            read_bytes: disk_usage.read_bytes,
+            written_bytes: disk_usage.written_bytes,
+        };
+        metrics.extend(collect_process_disk_metrics(&disk_stats, Some(&labels)));
+
+        // Collect network I/O metrics
+        // Note: sysinfo 0.38.x provides network info via Networks new type
+        // We use Networks::new_with_refreshed_list() to get network interfaces
+        let networks = sysinfo::Networks::new_with_refreshed_list();
+        let mut total_received = 0u64;
+        let mut total_transmitted = 0u64;
+        let mut per_interface = Vec::new();
+
+        for (interface_name, data) in networks.iter() {
+            let received = data.received();
+            let transmitted = data.transmitted();
+            total_received += received;
+            total_transmitted += transmitted;
+            per_interface.push((interface_name.to_string(), received, transmitted));
+        }
+
+        let network_stats = ProcessNetworkStats {
+            total_received,
+            total_transmitted,
+            per_interface,
+        };
+        metrics.extend(collect_process_network_metrics(&network_stats, Some(&labels)));
+
+        // Collect GPU metrics (if gpu feature is enabled)
+        #[cfg(feature = "gpu")]
+        {
+            use crate::metrics::collectors::{GpuCollector, collect_gpu_metrics};
+
+            match GpuCollector::new(pid) {
+                Ok(collector) => match collector.collect() {
+                    Ok(gpu_stats) => {
+                        metrics.extend(collect_gpu_metrics(&gpu_stats, &labels));
+                    }
+                    Err(e) => {
+                        warn!("GPU metrics collection failed: {}", e);
+                    }
+                },
+                Err(e) => {
+                    warn!("GPU collector initialization failed: {}", e);
+                }
+            }
+        }
+    }
+
+    metrics
+}

crates/obs/src/metrics/schema/audit.rs

@@ -0,0 +1,52 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+const TARGET_ID: &str = "target_id";
+pub const RESULT: &str = "result"; // success / failure
+pub const STATUS: &str = "status"; // success / failure
+
+pub const SUCCESS: &str = "success";
+pub const FAILURE: &str = "failure";
+
+pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::AuditFailedMessages,
+        "Total number of messages that failed to send since start",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});
+
+pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::AuditTargetQueueLength,
+        "Number of unsent messages in queue for target",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});
+
+pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::AuditTotalMessages,
+        "Total number of messages sent since start",
+        &[TARGET_ID],
+        subsystems::AUDIT,
+    )
+});

crates/obs/src/metrics/schema/bucket.rs

@@ -0,0 +1,90 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, new_histogram_md, subsystems};
+use std::sync::LazyLock;
+
+pub static BUCKET_API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficSentBytes,
+        "Total number of bytes received for a bucket",
+        &["bucket", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficRecvBytes,
+        "Total number of bytes sent for a bucket",
+        &["bucket", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_IN_FLIGHT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsInFlightTotal,
+        "Total number of requests currently in flight for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTotal,
+        "Total number of requests for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_CANCELED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsCanceledTotal,
+        "Total number of requests canceled by the client for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_4XX_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests4xxErrorsTotal,
+        "Total number of requests with 4xx errors for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_5XX_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests5xxErrorsTotal,
+        "Total number of requests with 5xx errors for a bucket",
+        &["bucket", "name", "type"],
+        subsystems::BUCKET_API,
+    )
+});
+
+pub static BUCKET_API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_histogram_md(
+        MetricName::ApiRequestsTTFBSecondsDistribution,
+        "Distribution of time to first byte across API calls for a bucket",
+        &["bucket", "name", "le", "type"],
+        subsystems::BUCKET_API,
+    )
+});

crates/obs/src/metrics/schema/bucket_replication.rs

@@ -0,0 +1,219 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Bucket level replication metric descriptor
+pub const BUCKET_L: &str = "bucket";
+/// Replication operation
+pub const OPERATION_L: &str = "operation";
+/// Replication target ARN
+pub const TARGET_ARN_L: &str = "targetArn";
+/// Replication range
+pub const RANGE_L: &str = "range";
+
+pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastHourFailedBytes,
+        "Total number of bytes failed at least once to replicate in the last hour on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastHourFailedCount,
+        "Total number of objects which failed replication in the last hour on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastMinFailedBytes,
+        "Total number of bytes failed at least once to replicate in the last full minute on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LastMinFailedCount,
+        "Total number of objects which failed replication in the last full minute on a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::LatencyMilliSec,
+        "Replication latency on a bucket in milliseconds",
+        &[BUCKET_L, OPERATION_L, RANGE_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedDeleteTaggingRequestsTotal,
+        "Number of DELETE tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetRequestsFailures,
+        "Number of failures in GET requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetRequestsTotal,
+        "Number of GET requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of PUT requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetTaggingRequestFailures,
+        "Number of failures in GET tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedGetTaggingRequestsTotal,
+        "Number of GET tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedHeadRequestsFailures,
+        "Number of failures in HEAD requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedHeadRequestsTotal,
+        "Number of HEAD requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of PUT requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedPutTaggingRequestFailures,
+        "Number of failures in PUT tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedPutTaggingRequestsTotal,
+        "Number of PUT tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SentBytes,
+        "Total number of bytes replicated to the target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SentCount,
+        "Total number of objects replicated to the target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TotalFailedBytes,
+        "Total number of bytes failed at least once to replicate since server start",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TotalFailedCount,
+        "Total number of objects which failed replication since server start",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::BandwidthLimitBytesPerSecond,
+        "Configured bandwidth limit for replication in bytes per second",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_BANDWIDTH_CURRENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::BandwidthCurrentBytesPerSecond,
+        "Current replication bandwidth in bytes per second (EWMA)",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+// TODO - add a metric for the number of DELETE requests proxied to replication target
+pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProxiedDeleteTaggingRequestFailures,
+        "Number of failures in DELETE tagging requests proxied to replication target",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});

crates/obs/src/metrics/schema/cluster.rs

@@ -0,0 +1,78 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Total raw storage capacity across all disks in bytes
+pub static CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("capacity_raw_total_bytes".to_string()),
+        "Total raw storage capacity in bytes across all disks",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});
+
+/// Total usable storage capacity in bytes (accounting for erasure coding)
+pub static CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("capacity_usable_total_bytes".to_string()),
+        "Total usable storage capacity in bytes (accounting for erasure coding)",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});
+
+/// Total used storage capacity in bytes
+pub static CLUSTER_CAPACITY_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("capacity_used_bytes".to_string()),
+        "Total used storage capacity in bytes",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});
+
+/// Total free storage capacity in bytes
+pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("capacity_free_bytes".to_string()),
+        "Total free storage capacity in bytes",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});
+
+/// Total number of objects in the cluster
+pub static CLUSTER_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("objects_total".to_string()),
+        "Total number of objects in the cluster",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});
+
+/// Total number of buckets in the cluster
+pub static CLUSTER_BUCKETS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("buckets_total".to_string()),
+        "Total number of buckets in the cluster",
+        &[],
+        subsystems::CLUSTER_BASE_PATH,
+    )
+});

crates/obs/src/metrics/schema/cluster_config.rs

@@ -0,0 +1,36 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static CONFIG_RRS_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ConfigRRSParity,
+        "Reduced redundancy storage class parity",
+        &[],
+        subsystems::CLUSTER_CONFIG,
+    )
+});
+
+pub static CONFIG_STANDARD_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ConfigStandardParity,
+        "Standard storage class parity",
+        &[],
+        subsystems::CLUSTER_CONFIG,
+    )
+});

crates/obs/src/metrics/schema/cluster_erasure_set.rs

@@ -0,0 +1,149 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// The label for the pool ID
+pub const POOL_ID_L: &str = "pool_id";
+/// The label for the pool ID
+pub const SET_ID_L: &str = "set_id";
+
+pub static ERASURE_SET_SIZE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetSize,
+        "Total number of drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetParity,
+        "Parity drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_DATA_SHARDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetDataShards,
+        "Data shards in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_OVERALL_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOverallWriteQuorum,
+        "Overall write quorum across pools and sets",
+        &[],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_OVERALL_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOverallHealth,
+        "Overall health across pools and sets (1=healthy, 0=unhealthy)",
+        &[],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadQuorum,
+        "Read quorum for the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteQuorum,
+        "Write quorum for the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_ONLINE_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetOnlineDrivesCount,
+        "Count of online drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_HEALING_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetHealingDrivesCount,
+        "Count of healing drives in the erasure set in a pool",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetHealth,
+        "Health of the erasure set in a pool (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_TOLERANCE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadTolerance,
+        "No of drive failures that can be tolerated without disrupting read operations",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_TOLERANCE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteTolerance,
+        "No of drive failures that can be tolerated without disrupting write operations",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_READ_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetReadHealth,
+        "Health of the erasure set in a pool for read operations (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});
+
+pub static ERASURE_SET_WRITE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ErasureSetWriteHealth,
+        "Health of the erasure set in a pool for write operations (1=healthy, 0=unhealthy)",
+        &[POOL_ID_L, SET_ID_L],
+        subsystems::CLUSTER_ERASURE_SET,
+    )
+});

crates/obs/src/metrics/schema/cluster_health.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static HEALTH_DRIVES_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesOfflineCount,
+        "Count of offline drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});
+
+pub static HEALTH_DRIVES_ONLINE_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesOnlineCount,
+        "Count of online drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});
+
+pub static HEALTH_DRIVES_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::HealthDrivesCount,
+        "Count of all drives in the cluster",
+        &[],
+        subsystems::CLUSTER_HEALTH,
+    )
+});

crates/obs/src/metrics/schema/cluster_iam.rs

@@ -0,0 +1,108 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
+use std::sync::LazyLock;
+
+pub static LAST_SYNC_DURATION_MILLIS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::LastSyncDurationMillis,
+        "Last successful IAM data sync duration in milliseconds",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_FAILED_REQUESTS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceFailedRequestsMinute,
+        "When plugin authentication is configured, returns failed requests count in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceLastFailSeconds,
+        "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceLastSuccSeconds,
+        "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceSuccAvgRttMsMinute,
+        "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceSuccMaxRttMsMinute,
+        "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::PluginAuthnServiceTotalRequestsMinute,
+        "When plugin authentication is configured, returns total requests count in the last full minute",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SINCE_LAST_SYNC_MILLIS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SinceLastSyncMillis,
+        "Time (in milliseconds) since last successful IAM data sync.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SYNC_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SyncFailures,
+        "Number of failed IAM data syncs since server start.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});
+
+pub static SYNC_SUCCESSES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::SyncSuccesses,
+        "Number of successful IAM data syncs since server start.",
+        &[],
+        subsystems::CLUSTER_IAM,
+    )
+});

crates/obs/src/metrics/schema/cluster_notification.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::NotificationCurrentSendInProgress,
+        "Number of concurrent async Send calls active to all targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsErrorsTotal,
+        "Events that were failed to be sent to the targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsSentTotal,
+        "Total number of events sent to the targets",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationEventsSkippedTotal,
+        "Notification dispatch attempts skipped before delivery",
+        &[],
+        subsystems::NOTIFICATION,
+    )
+});

crates/obs/src/metrics/schema/cluster_usage.rs

@@ -0,0 +1,158 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Bucket labels
+pub const BUCKET_LABEL: &str = "bucket";
+/// Range labels
+pub const RANGE_LABEL: &str = "range";
+
+pub static USAGE_SINCE_LAST_UPDATE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageSinceLastUpdateSeconds,
+        "Time since last update of usage metrics in seconds",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageTotalBytes,
+        "Total cluster usage in bytes",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_OBJECTS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageObjectsCount,
+        "Total cluster objects count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_VERSIONS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageVersionsCount,
+        "Total cluster object versions (including delete markers) count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_DELETE_MARKERS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageDeleteMarkersCount,
+        "Total cluster delete markers count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_BUCKETS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketsCount,
+        "Total cluster buckets count",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_OBJECTS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageSizeDistribution,
+        "Cluster object size distribution",
+        &[RANGE_LABEL],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_VERSIONS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageVersionCountDistribution,
+        "Cluster object version count distribution",
+        &[RANGE_LABEL],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
+pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketTotalBytes,
+        "Total bucket size in bytes",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectsCount,
+        "Total objects count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_VERSIONS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketVersionsCount,
+        "Total object versions (including delete markers) count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_DELETE_MARKERS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketDeleteMarkersCount,
+        "Total delete markers count in bucket",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketQuotaTotalBytes,
+        "Total bucket quota in bytes",
+        &[BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectSizeDistribution,
+        "Bucket object size distribution",
+        &[RANGE_LABEL, BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});
+
+pub static USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageBucketObjectVersionCountDistribution,
+        "Bucket object version count distribution",
+        &[RANGE_LABEL, BUCKET_LABEL],
+        subsystems::CLUSTER_USAGE_BUCKETS,
+    )
+});

crates/obs/src/metrics/schema/entry/descriptor.rs

@@ -0,0 +1,113 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
+use std::collections::HashSet;
+
+/// MetricDescriptor - Metric descriptors
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct MetricDescriptor {
+    pub name: MetricName,
+    pub metric_type: MetricType,
+    pub help: String,
+    pub variable_labels: Vec<String>,
+    pub namespace: MetricNamespace,
+    pub subsystem: MetricSubsystem,
+
+    // Internal management values
+    label_set: Option<HashSet<String>>,
+}
+
+impl MetricDescriptor {
+    /// Create a new metric descriptor
+    pub fn new(
+        name: MetricName,
+        metric_type: MetricType,
+        help: String,
+        variable_labels: Vec<String>,
+        namespace: MetricNamespace,
+        subsystem: impl Into<MetricSubsystem>, // Modify the parameter type
+    ) -> Self {
+        Self {
+            name,
+            metric_type,
+            help,
+            variable_labels,
+            namespace,
+            subsystem: subsystem.into(),
+            label_set: None,
+        }
+    }
+
+    /// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name>
+    #[allow(dead_code)]
+    pub fn get_full_metric_name(&self) -> String {
+        let namespace = self.namespace.as_str();
+        let formatted_subsystem = self.subsystem.as_str();
+
+        format!("{}_{}_{}", namespace, formatted_subsystem, self.name.as_str())
+    }
+
+    /// check whether the label is in the label set
+    #[allow(dead_code)]
+    pub fn has_label(&mut self, label: &str) -> bool {
+        self.get_label_set().contains(label)
+    }
+
+    /// Gets a collection of tags and creates them if they don't exist
+    pub fn get_label_set(&mut self) -> &HashSet<String> {
+        if self.label_set.is_none() {
+            let mut set = HashSet::with_capacity(self.variable_labels.len());
+            for label in &self.variable_labels {
+                set.insert(label.clone());
+            }
+            self.label_set = Some(set);
+        }
+        self.label_set.as_ref().unwrap()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn full_metric_name_uses_prometheus_convention_without_type_prefix() {
+        let descriptor = MetricDescriptor::new(
+            MetricName::ApiRequestsTotal,
+            MetricType::Counter,
+            "test help".to_string(),
+            vec![],
+            MetricNamespace::RustFS,
+            MetricSubsystem::ApiRequests,
+        );
+
+        assert_eq!(descriptor.get_full_metric_name(), "rustfs_api_requests_total");
+    }
+
+    #[test]
+    fn full_metric_name_formats_custom_subsystems_without_type_prefix() {
+        let descriptor = MetricDescriptor::new(
+            MetricName::Custom("latency_seconds".to_string()),
+            MetricType::Histogram,
+            "test help".to_string(),
+            vec![],
+            MetricNamespace::RustFS,
+            MetricSubsystem::new("/custom/path-metrics"),
+        );
+
+        assert_eq!(descriptor.get_full_metric_name(), "rustfs_custom_path_metrics_latency_seconds");
+    }
+}

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -0,0 +1,711 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// The metric name is the individual name of the metric
+#[allow(dead_code)]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MetricName {
+    // The generic metric name
+    AuthTotal,
+    CanceledTotal,
+    ErrorsTotal,
+    HeaderTotal,
+    HealTotal,
+    HitsTotal,
+    InflightTotal,
+    InvalidTotal,
+    LimitTotal,
+    MissedTotal,
+    WaitingTotal,
+    IncomingTotal,
+    ObjectTotal,
+    VersionTotal,
+    DeleteMarkerTotal,
+    OfflineTotal,
+    OnlineTotal,
+    OpenTotal,
+    ReadTotal,
+    TimestampTotal,
+    WriteTotal,
+    Total,
+    FreeInodes,
+
+    // Failure statistical metrics
+    LastMinFailedCount,
+    LastMinFailedBytes,
+    LastHourFailedCount,
+    LastHourFailedBytes,
+    TotalFailedCount,
+    TotalFailedBytes,
+
+    // Worker metrics
+    CurrActiveWorkers,
+    AvgActiveWorkers,
+    MaxActiveWorkers,
+    RecentBacklogCount,
+    CurrInQueueCount,
+    CurrInQueueBytes,
+    ReceivedCount,
+    SentCount,
+    CurrTransferRate,
+    AvgTransferRate,
+    MaxTransferRate,
+    CredentialErrors,
+
+    // Link latency metrics
+    CurrLinkLatency,
+    AvgLinkLatency,
+    MaxLinkLatency,
+
+    // Link status metrics
+    LinkOnline,
+    LinkOfflineDuration,
+    LinkDowntimeTotalDuration,
+
+    // Queue metrics
+    AvgInQueueCount,
+    AvgInQueueBytes,
+    MaxInQueueCount,
+    MaxInQueueBytes,
+
+    // Proxy request metrics
+    ProxiedGetRequestsTotal,
+    ProxiedHeadRequestsTotal,
+    ProxiedPutTaggingRequestsTotal,
+    ProxiedGetTaggingRequestsTotal,
+    ProxiedDeleteTaggingRequestsTotal,
+    ProxiedGetRequestsFailures,
+    ProxiedHeadRequestsFailures,
+    ProxiedPutTaggingRequestFailures,
+    ProxiedGetTaggingRequestFailures,
+    ProxiedDeleteTaggingRequestFailures,
+
+    // Byte-related metrics
+    FreeBytes,
+    ReadBytes,
+    RcharBytes,
+    ReceivedBytes,
+    LatencyMilliSec,
+    SentBytes,
+    TotalBytes,
+    UsedBytes,
+    WriteBytes,
+    WcharBytes,
+
+    // Latency metrics
+    LatencyMicroSec,
+    LatencyNanoSec,
+
+    // Information metrics
+    CommitInfo,
+    UsageInfo,
+    VersionInfo,
+
+    // Distribution metrics
+    SizeDistribution,
+    VersionDistribution,
+    TtfbDistribution,
+    TtlbDistribution,
+
+    // Time metrics
+    LastActivityTime,
+    StartTime,
+    UpTime,
+    Memory,
+    Vmemory,
+    Cpu,
+
+    // Expiration and conversion metrics
+    ExpiryMissedTasks,
+    ExpiryMissedFreeVersions,
+    ExpiryMissedTierJournalTasks,
+    ExpiryNumWorkers,
+    TransitionMissedTasks,
+    TransitionedBytes,
+    TransitionedObjects,
+    TransitionedVersions,
+
+    //Tier request metrics
+    TierRequestsSuccess,
+    TierRequestsFailure,
+
+    // KMS metrics
+    KmsOnline,
+    KmsRequestsSuccess,
+    KmsRequestsError,
+    KmsRequestsFail,
+    KmsUptime,
+
+    // Webhook metrics
+    WebhookOnline,
+
+    // API rejection metrics
+    ApiRejectedAuthTotal,
+    ApiRejectedHeaderTotal,
+    ApiRejectedTimestampTotal,
+    ApiRejectedInvalidTotal,
+
+    //API request metrics
+    ApiRequestsWaitingTotal,
+    ApiRequestsIncomingTotal,
+    ApiRequestsInFlightTotal,
+    ApiRequestsTotal,
+    ApiRequestsErrorsTotal,
+    ApiRequests5xxErrorsTotal,
+    ApiRequests4xxErrorsTotal,
+    ApiRequestsCanceledTotal,
+
+    // API distribution metrics
+    ApiRequestsTTFBSecondsDistribution,
+
+    // API traffic metrics
+    ApiTrafficSentBytes,
+    ApiTrafficRecvBytes,
+
+    // Audit metrics
+    AuditFailedMessages,
+    AuditTargetQueueLength,
+    AuditTotalMessages,
+
+    // Metrics related to cluster configurations
+    ConfigRRSParity,
+    ConfigStandardParity,
+
+    // Erasure coding set related metrics
+    ErasureSetSize,
+    ErasureSetParity,
+    ErasureSetDataShards,
+    ErasureSetOverallWriteQuorum,
+    ErasureSetOverallHealth,
+    ErasureSetReadQuorum,
+    ErasureSetWriteQuorum,
+    ErasureSetOnlineDrivesCount,
+    ErasureSetHealingDrivesCount,
+    ErasureSetHealth,
+    ErasureSetReadTolerance,
+    ErasureSetWriteTolerance,
+    ErasureSetReadHealth,
+    ErasureSetWriteHealth,
+
+    // Cluster health-related metrics
+    HealthDrivesOfflineCount,
+    HealthDrivesOnlineCount,
+    HealthDrivesCount,
+
+    // IAM-related metrics
+    LastSyncDurationMillis,
+    PluginAuthnServiceFailedRequestsMinute,
+    PluginAuthnServiceLastFailSeconds,
+    PluginAuthnServiceLastSuccSeconds,
+    PluginAuthnServiceSuccAvgRttMsMinute,
+    PluginAuthnServiceSuccMaxRttMsMinute,
+    PluginAuthnServiceTotalRequestsMinute,
+    SinceLastSyncMillis,
+    SyncFailures,
+    SyncSuccesses,
+
+    // Notify relevant metrics
+    NotificationCurrentSendInProgress,
+    NotificationEventsErrorsTotal,
+    NotificationEventsSentTotal,
+    NotificationEventsSkippedTotal,
+    NotificationTargetFailedMessages,
+    NotificationTargetQueueLength,
+    NotificationTargetTotalMessages,
+
+    // Metrics related to the usage of cluster objects
+    UsageSinceLastUpdateSeconds,
+    UsageTotalBytes,
+    UsageObjectsCount,
+    UsageVersionsCount,
+    UsageDeleteMarkersCount,
+    UsageBucketsCount,
+    UsageSizeDistribution,
+    UsageVersionCountDistribution,
+
+    // Metrics related to bucket usage
+    UsageBucketQuotaTotalBytes,
+    UsageBucketTotalBytes,
+    UsageBucketObjectsCount,
+    UsageBucketVersionsCount,
+    UsageBucketDeleteMarkersCount,
+    UsageBucketObjectSizeDistribution,
+    UsageBucketObjectVersionCountDistribution,
+
+    // ILM-related metrics
+    IlmExpiryPendingTasks,
+    IlmTransitionActiveTasks,
+    IlmTransitionPendingTasks,
+    IlmTransitionMissedImmediateTasks,
+    IlmVersionsScanned,
+
+    // Copy the relevant metrics
+    ReplicationAverageActiveWorkers,
+    ReplicationAverageQueuedBytes,
+    ReplicationAverageQueuedCount,
+    ReplicationAverageDataTransferRate,
+    ReplicationCurrentActiveWorkers,
+    ReplicationCurrentDataTransferRate,
+    ReplicationLastMinuteQueuedBytes,
+    ReplicationLastMinuteQueuedCount,
+    ReplicationMaxActiveWorkers,
+    ReplicationMaxQueuedBytes,
+    ReplicationMaxQueuedCount,
+    ReplicationMaxDataTransferRate,
+    ReplicationRecentBacklogCount,
+    BandwidthLimitBytesPerSecond,
+    BandwidthCurrentBytesPerSecond,
+
+    // Scanner-related metrics
+    ScannerBucketScansFinished,
+    ScannerBucketScansStarted,
+    ScannerDirectoriesScanned,
+    ScannerObjectsScanned,
+    ScannerVersionsScanned,
+    ScannerLastActivitySeconds,
+
+    // CPU system-related metrics
+    SysCPUAvgIdle,
+    SysCPUAvgIOWait,
+    SysCPULoad,
+    SysCPULoadPerc,
+    SysCPUNice,
+    SysCPUSteal,
+    SysCPUSystem,
+    SysCPUUser,
+
+    // Drive-related metrics
+    DriveUsedBytes,
+    DriveFreeBytes,
+    DriveTotalBytes,
+    DriveUsedInodes,
+    DriveFreeInodes,
+    DriveTotalInodes,
+    DriveTimeoutErrorsTotal,
+    DriveIOErrorsTotal,
+    DriveAvailabilityErrorsTotal,
+    DriveWaitingIO,
+    DriveAPILatencyMicros,
+    DriveHealth,
+
+    DriveOfflineCount,
+    DriveOnlineCount,
+    DriveCount,
+
+    // iostat related metrics
+    DriveReadsPerSec,
+    DriveReadsKBPerSec,
+    DriveReadsAwait,
+    DriveWritesPerSec,
+    DriveWritesKBPerSec,
+    DriveWritesAwait,
+    DrivePercUtil,
+
+    // Memory-related metrics
+    MemTotal,
+    MemUsed,
+    MemUsedPerc,
+    MemFree,
+    MemBuffers,
+    MemCache,
+    MemShared,
+    MemAvailable,
+
+    // Network-related metrics
+    InternodeErrorsTotal,
+    InternodeDialErrorsTotal,
+    InternodeDialAvgTimeNanos,
+    InternodeSentBytesTotal,
+    InternodeRecvBytesTotal,
+
+    // Process-related metrics
+    ProcessLocksReadTotal,
+    ProcessLocksWriteTotal,
+    ProcessCPUTotalSeconds,
+    ProcessGoRoutineTotal,
+    ProcessIORCharBytes,
+    ProcessIOReadBytes,
+    ProcessIOWCharBytes,
+    ProcessIOWriteBytes,
+    ProcessStartTimeSeconds,
+    ProcessUptimeSeconds,
+    ProcessFileDescriptorLimitTotal,
+    ProcessFileDescriptorOpenTotal,
+    ProcessSyscallReadTotal,
+    ProcessSyscallWriteTotal,
+    ProcessResidentMemoryBytes,
+    ProcessVirtualMemoryBytes,
+    ProcessVirtualMemoryMaxBytes,
+
+    // Process-level system monitoring metrics (migrated from rustfs-obs::system)
+    /// Process CPU usage percentage (0-100)
+    ProcessCPUUsage,
+    /// Process CPU utilization percentage (considering multiple cores)
+    ProcessCPUUtilization,
+    /// Process disk I/O bytes
+    ProcessDiskIO,
+    /// Process network I/O bytes
+    ProcessNetworkIO,
+    /// Process network I/O bytes per interface
+    ProcessNetworkIOPerInterface,
+    /// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)
+    ProcessStatus,
+    /// Process GPU memory usage in bytes
+    ProcessGpuMemoryUsage,
+
+    // Custom metrics
+    Custom(String),
+}
+
+impl MetricName {
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> String {
+        match self {
+            Self::AuthTotal => "auth_total".to_string(),
+            Self::CanceledTotal => "canceled_total".to_string(),
+            Self::ErrorsTotal => "errors_total".to_string(),
+            Self::HeaderTotal => "header_total".to_string(),
+            Self::HealTotal => "heal_total".to_string(),
+            Self::HitsTotal => "hits_total".to_string(),
+            Self::InflightTotal => "inflight_total".to_string(),
+            Self::InvalidTotal => "invalid_total".to_string(),
+            Self::LimitTotal => "limit_total".to_string(),
+            Self::MissedTotal => "missed_total".to_string(),
+            Self::WaitingTotal => "waiting_total".to_string(),
+            Self::IncomingTotal => "incoming_total".to_string(),
+            Self::ObjectTotal => "object_total".to_string(),
+            Self::VersionTotal => "version_total".to_string(),
+            Self::DeleteMarkerTotal => "deletemarker_total".to_string(),
+            Self::OfflineTotal => "offline_total".to_string(),
+            Self::OnlineTotal => "online_total".to_string(),
+            Self::OpenTotal => "open_total".to_string(),
+            Self::ReadTotal => "read_total".to_string(),
+            Self::TimestampTotal => "timestamp_total".to_string(),
+            Self::WriteTotal => "write_total".to_string(),
+            Self::Total => "total".to_string(),
+            Self::FreeInodes => "free_inodes".to_string(),
+
+            Self::LastMinFailedCount => "last_minute_failed_count".to_string(),
+            Self::LastMinFailedBytes => "last_minute_failed_bytes".to_string(),
+            Self::LastHourFailedCount => "last_hour_failed_count".to_string(),
+            Self::LastHourFailedBytes => "last_hour_failed_bytes".to_string(),
+            Self::TotalFailedCount => "total_failed_count".to_string(),
+            Self::TotalFailedBytes => "total_failed_bytes".to_string(),
+
+            Self::CurrActiveWorkers => "current_active_workers".to_string(),
+            Self::AvgActiveWorkers => "average_active_workers".to_string(),
+            Self::MaxActiveWorkers => "max_active_workers".to_string(),
+            Self::RecentBacklogCount => "recent_backlog_count".to_string(),
+            Self::CurrInQueueCount => "last_minute_queued_count".to_string(),
+            Self::CurrInQueueBytes => "last_minute_queued_bytes".to_string(),
+            Self::ReceivedCount => "received_count".to_string(),
+            Self::SentCount => "sent_count".to_string(),
+            Self::CurrTransferRate => "current_transfer_rate".to_string(),
+            Self::AvgTransferRate => "average_transfer_rate".to_string(),
+            Self::MaxTransferRate => "max_transfer_rate".to_string(),
+            Self::CredentialErrors => "credential_errors".to_string(),
+
+            Self::CurrLinkLatency => "current_link_latency_ms".to_string(),
+            Self::AvgLinkLatency => "average_link_latency_ms".to_string(),
+            Self::MaxLinkLatency => "max_link_latency_ms".to_string(),
+
+            Self::LinkOnline => "link_online".to_string(),
+            Self::LinkOfflineDuration => "link_offline_duration_seconds".to_string(),
+            Self::LinkDowntimeTotalDuration => "link_downtime_duration_seconds".to_string(),
+
+            Self::AvgInQueueCount => "average_queued_count".to_string(),
+            Self::AvgInQueueBytes => "average_queued_bytes".to_string(),
+            Self::MaxInQueueCount => "max_queued_count".to_string(),
+            Self::MaxInQueueBytes => "max_queued_bytes".to_string(),
+
+            Self::ProxiedGetRequestsTotal => "proxied_get_requests_total".to_string(),
+            Self::ProxiedHeadRequestsTotal => "proxied_head_requests_total".to_string(),
+            Self::ProxiedPutTaggingRequestsTotal => "proxied_put_tagging_requests_total".to_string(),
+            Self::ProxiedGetTaggingRequestsTotal => "proxied_get_tagging_requests_total".to_string(),
+            Self::ProxiedDeleteTaggingRequestsTotal => "proxied_delete_tagging_requests_total".to_string(),
+            Self::ProxiedGetRequestsFailures => "proxied_get_requests_failures".to_string(),
+            Self::ProxiedHeadRequestsFailures => "proxied_head_requests_failures".to_string(),
+            Self::ProxiedPutTaggingRequestFailures => "proxied_put_tagging_requests_failures".to_string(),
+            Self::ProxiedGetTaggingRequestFailures => "proxied_get_tagging_requests_failures".to_string(),
+            Self::ProxiedDeleteTaggingRequestFailures => "proxied_delete_tagging_requests_failures".to_string(),
+
+            Self::FreeBytes => "free_bytes".to_string(),
+            Self::ReadBytes => "read_bytes".to_string(),
+            Self::RcharBytes => "rchar_bytes".to_string(),
+            Self::ReceivedBytes => "received_bytes".to_string(),
+            Self::LatencyMilliSec => "latency_ms".to_string(),
+            Self::SentBytes => "sent_bytes".to_string(),
+            Self::TotalBytes => "total_bytes".to_string(),
+            Self::UsedBytes => "used_bytes".to_string(),
+            Self::WriteBytes => "write_bytes".to_string(),
+            Self::WcharBytes => "wchar_bytes".to_string(),
+
+            Self::LatencyMicroSec => "latency_us".to_string(),
+            Self::LatencyNanoSec => "latency_ns".to_string(),
+
+            Self::CommitInfo => "commit_info".to_string(),
+            Self::UsageInfo => "usage_info".to_string(),
+            Self::VersionInfo => "version_info".to_string(),
+
+            Self::SizeDistribution => "size_distribution".to_string(),
+            Self::VersionDistribution => "version_distribution".to_string(),
+            Self::TtfbDistribution => "seconds_distribution".to_string(),
+            Self::TtlbDistribution => "ttlb_seconds_distribution".to_string(),
+
+            Self::LastActivityTime => "last_activity_nano_seconds".to_string(),
+            Self::StartTime => "starttime_seconds".to_string(),
+            Self::UpTime => "uptime_seconds".to_string(),
+            Self::Memory => "resident_memory_bytes".to_string(),
+            Self::Vmemory => "virtual_memory_bytes".to_string(),
+            Self::Cpu => "cpu_total_seconds".to_string(),
+
+            Self::ExpiryMissedTasks => "expiry_missed_tasks".to_string(),
+            Self::ExpiryMissedFreeVersions => "expiry_missed_freeversions".to_string(),
+            Self::ExpiryMissedTierJournalTasks => "expiry_missed_tierjournal_tasks".to_string(),
+            Self::ExpiryNumWorkers => "expiry_num_workers".to_string(),
+            Self::TransitionMissedTasks => "transition_missed_immediate_tasks".to_string(),
+
+            Self::TransitionedBytes => "transitioned_bytes".to_string(),
+            Self::TransitionedObjects => "transitioned_objects".to_string(),
+            Self::TransitionedVersions => "transitioned_versions".to_string(),
+
+            Self::TierRequestsSuccess => "requests_success".to_string(),
+            Self::TierRequestsFailure => "requests_failure".to_string(),
+
+            Self::KmsOnline => "online".to_string(),
+            Self::KmsRequestsSuccess => "request_success".to_string(),
+            Self::KmsRequestsError => "request_error".to_string(),
+            Self::KmsRequestsFail => "request_failure".to_string(),
+            Self::KmsUptime => "uptime".to_string(),
+
+            Self::WebhookOnline => "online".to_string(),
+
+            Self::ApiRejectedAuthTotal => "rejected_auth_total".to_string(),
+            Self::ApiRejectedHeaderTotal => "rejected_header_total".to_string(),
+            Self::ApiRejectedTimestampTotal => "rejected_timestamp_total".to_string(),
+            Self::ApiRejectedInvalidTotal => "rejected_invalid_total".to_string(),
+
+            Self::ApiRequestsWaitingTotal => "waiting_total".to_string(),
+            Self::ApiRequestsIncomingTotal => "incoming_total".to_string(),
+            Self::ApiRequestsInFlightTotal => "inflight_total".to_string(),
+            Self::ApiRequestsTotal => "total".to_string(),
+            Self::ApiRequestsErrorsTotal => "errors_total".to_string(),
+            Self::ApiRequests5xxErrorsTotal => "5xx_errors_total".to_string(),
+            Self::ApiRequests4xxErrorsTotal => "4xx_errors_total".to_string(),
+            Self::ApiRequestsCanceledTotal => "canceled_total".to_string(),
+
+            Self::ApiRequestsTTFBSecondsDistribution => "ttfb_seconds_distribution".to_string(),
+
+            Self::ApiTrafficSentBytes => "traffic_sent_bytes".to_string(),
+            Self::ApiTrafficRecvBytes => "traffic_received_bytes".to_string(),
+
+            Self::AuditFailedMessages => "failed_messages".to_string(),
+            Self::AuditTargetQueueLength => "target_queue_length".to_string(),
+            Self::AuditTotalMessages => "total_messages".to_string(),
+
+            // metrics related to cluster configurations
+            Self::ConfigRRSParity => "rrs_parity".to_string(),
+            Self::ConfigStandardParity => "standard_parity".to_string(),
+
+            // Erasure coding set related metrics
+            Self::ErasureSetSize => "size".to_string(),
+            Self::ErasureSetParity => "parity".to_string(),
+            Self::ErasureSetDataShards => "data_shards".to_string(),
+            Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(),
+            Self::ErasureSetOverallHealth => "overall_health".to_string(),
+            Self::ErasureSetReadQuorum => "read_quorum".to_string(),
+            Self::ErasureSetWriteQuorum => "write_quorum".to_string(),
+            Self::ErasureSetOnlineDrivesCount => "online_drives_count".to_string(),
+            Self::ErasureSetHealingDrivesCount => "healing_drives_count".to_string(),
+            Self::ErasureSetHealth => "health".to_string(),
+            Self::ErasureSetReadTolerance => "read_tolerance".to_string(),
+            Self::ErasureSetWriteTolerance => "write_tolerance".to_string(),
+            Self::ErasureSetReadHealth => "read_health".to_string(),
+            Self::ErasureSetWriteHealth => "write_health".to_string(),
+
+            // Cluster health-related metrics
+            Self::HealthDrivesOfflineCount => "drives_offline_count".to_string(),
+            Self::HealthDrivesOnlineCount => "drives_online_count".to_string(),
+            Self::HealthDrivesCount => "drives_count".to_string(),
+
+            // IAM-related metrics
+            Self::LastSyncDurationMillis => "last_sync_duration_millis".to_string(),
+            Self::PluginAuthnServiceFailedRequestsMinute => "plugin_authn_service_failed_requests_minute".to_string(),
+            Self::PluginAuthnServiceLastFailSeconds => "plugin_authn_service_last_fail_seconds".to_string(),
+            Self::PluginAuthnServiceLastSuccSeconds => "plugin_authn_service_last_succ_seconds".to_string(),
+            Self::PluginAuthnServiceSuccAvgRttMsMinute => "plugin_authn_service_succ_avg_rtt_ms_minute".to_string(),
+            Self::PluginAuthnServiceSuccMaxRttMsMinute => "plugin_authn_service_succ_max_rtt_ms_minute".to_string(),
+            Self::PluginAuthnServiceTotalRequestsMinute => "plugin_authn_service_total_requests_minute".to_string(),
+            Self::SinceLastSyncMillis => "since_last_sync_millis".to_string(),
+            Self::SyncFailures => "sync_failures".to_string(),
+            Self::SyncSuccesses => "sync_successes".to_string(),
+
+            // Notify relevant metrics
+            Self::NotificationCurrentSendInProgress => "current_send_in_progress".to_string(),
+            Self::NotificationEventsErrorsTotal => "events_errors_total".to_string(),
+            Self::NotificationEventsSentTotal => "events_sent_total".to_string(),
+            Self::NotificationEventsSkippedTotal => "events_skipped_total".to_string(),
+            Self::NotificationTargetFailedMessages => "failed_messages".to_string(),
+            Self::NotificationTargetQueueLength => "target_queue_length".to_string(),
+            Self::NotificationTargetTotalMessages => "total_messages".to_string(),
+
+            // Metrics related to the usage of cluster objects
+            Self::UsageSinceLastUpdateSeconds => "since_last_update_seconds".to_string(),
+            Self::UsageTotalBytes => "total_bytes".to_string(),
+            Self::UsageObjectsCount => "count".to_string(),
+            Self::UsageVersionsCount => "versions_count".to_string(),
+            Self::UsageDeleteMarkersCount => "delete_markers_count".to_string(),
+            Self::UsageBucketsCount => "buckets_count".to_string(),
+            Self::UsageSizeDistribution => "size_distribution".to_string(),
+            Self::UsageVersionCountDistribution => "version_count_distribution".to_string(),
+
+            // Metrics related to bucket usage
+            Self::UsageBucketQuotaTotalBytes => "quota_total_bytes".to_string(),
+            Self::UsageBucketTotalBytes => "total_bytes".to_string(),
+            Self::UsageBucketObjectsCount => "objects_count".to_string(),
+            Self::UsageBucketVersionsCount => "versions_count".to_string(),
+            Self::UsageBucketDeleteMarkersCount => "delete_markers_count".to_string(),
+            Self::UsageBucketObjectSizeDistribution => "object_size_distribution".to_string(),
+            Self::UsageBucketObjectVersionCountDistribution => "object_version_count_distribution".to_string(),
+
+            // ILM-related metrics
+            Self::IlmExpiryPendingTasks => "expiry_pending_tasks".to_string(),
+            Self::IlmTransitionActiveTasks => "transition_active_tasks".to_string(),
+            Self::IlmTransitionPendingTasks => "transition_pending_tasks".to_string(),
+            Self::IlmTransitionMissedImmediateTasks => "transition_missed_immediate_tasks".to_string(),
+            Self::IlmVersionsScanned => "versions_scanned".to_string(),
+
+            // Copy the relevant metrics
+            Self::ReplicationAverageActiveWorkers => "average_active_workers".to_string(),
+            Self::ReplicationAverageQueuedBytes => "average_queued_bytes".to_string(),
+            Self::ReplicationAverageQueuedCount => "average_queued_count".to_string(),
+            Self::ReplicationAverageDataTransferRate => "average_data_transfer_rate".to_string(),
+            Self::ReplicationCurrentActiveWorkers => "current_active_workers".to_string(),
+            Self::ReplicationCurrentDataTransferRate => "current_data_transfer_rate".to_string(),
+            Self::ReplicationLastMinuteQueuedBytes => "last_minute_queued_bytes".to_string(),
+            Self::ReplicationLastMinuteQueuedCount => "last_minute_queued_count".to_string(),
+            Self::ReplicationMaxActiveWorkers => "max_active_workers".to_string(),
+            Self::ReplicationMaxQueuedBytes => "max_queued_bytes".to_string(),
+            Self::ReplicationMaxQueuedCount => "max_queued_count".to_string(),
+            Self::ReplicationMaxDataTransferRate => "max_data_transfer_rate".to_string(),
+            Self::ReplicationRecentBacklogCount => "recent_backlog_count".to_string(),
+            Self::BandwidthLimitBytesPerSecond => "bandwidth_limit_bytes_per_second".to_string(),
+            Self::BandwidthCurrentBytesPerSecond => "bandwidth_current_bytes_per_second".to_string(),
+
+            // Scanner-related metrics
+            Self::ScannerBucketScansFinished => "bucket_scans_finished".to_string(),
+            Self::ScannerBucketScansStarted => "bucket_scans_started".to_string(),
+            Self::ScannerDirectoriesScanned => "directories_scanned".to_string(),
+            Self::ScannerObjectsScanned => "objects_scanned".to_string(),
+            Self::ScannerVersionsScanned => "versions_scanned".to_string(),
+            Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
+
+            // CPU system-related metrics
+            Self::SysCPUAvgIdle => "avg_idle".to_string(),
+            Self::SysCPUAvgIOWait => "avg_iowait".to_string(),
+            Self::SysCPULoad => "load".to_string(),
+            Self::SysCPULoadPerc => "load_perc".to_string(),
+            Self::SysCPUNice => "nice".to_string(),
+            Self::SysCPUSteal => "steal".to_string(),
+            Self::SysCPUSystem => "system".to_string(),
+            Self::SysCPUUser => "user".to_string(),
+
+            // Drive-related metrics
+            Self::DriveUsedBytes => "used_bytes".to_string(),
+            Self::DriveFreeBytes => "free_bytes".to_string(),
+            Self::DriveTotalBytes => "total_bytes".to_string(),
+            Self::DriveUsedInodes => "used_inodes".to_string(),
+            Self::DriveFreeInodes => "free_inodes".to_string(),
+            Self::DriveTotalInodes => "total_inodes".to_string(),
+            Self::DriveTimeoutErrorsTotal => "timeout_errors_total".to_string(),
+            Self::DriveIOErrorsTotal => "io_errors_total".to_string(),
+            Self::DriveAvailabilityErrorsTotal => "availability_errors_total".to_string(),
+            Self::DriveWaitingIO => "waiting_io".to_string(),
+            Self::DriveAPILatencyMicros => "api_latency_micros".to_string(),
+            Self::DriveHealth => "health".to_string(),
+
+            Self::DriveOfflineCount => "offline_count".to_string(),
+            Self::DriveOnlineCount => "online_count".to_string(),
+            Self::DriveCount => "count".to_string(),
+
+            // iostat related metrics
+            Self::DriveReadsPerSec => "reads_per_sec".to_string(),
+            Self::DriveReadsKBPerSec => "reads_kb_per_sec".to_string(),
+            Self::DriveReadsAwait => "reads_await".to_string(),
+            Self::DriveWritesPerSec => "writes_per_sec".to_string(),
+            Self::DriveWritesKBPerSec => "writes_kb_per_sec".to_string(),
+            Self::DriveWritesAwait => "writes_await".to_string(),
+            Self::DrivePercUtil => "perc_util".to_string(),
+
+            // Memory-related metrics
+            Self::MemTotal => "total".to_string(),
+            Self::MemUsed => "used".to_string(),
+            Self::MemUsedPerc => "used_perc".to_string(),
+            Self::MemFree => "free".to_string(),
+            Self::MemBuffers => "buffers".to_string(),
+            Self::MemCache => "cache".to_string(),
+            Self::MemShared => "shared".to_string(),
+            Self::MemAvailable => "available".to_string(),
+
+            // Network-related metrics
+            Self::InternodeErrorsTotal => "errors_total".to_string(),
+            Self::InternodeDialErrorsTotal => "dial_errors_total".to_string(),
+            Self::InternodeDialAvgTimeNanos => "dial_avg_time_nanos".to_string(),
+            Self::InternodeSentBytesTotal => "sent_bytes_total".to_string(),
+            Self::InternodeRecvBytesTotal => "recv_bytes_total".to_string(),
+
+            // Process-related metrics
+            Self::ProcessLocksReadTotal => "locks_read_total".to_string(),
+            Self::ProcessLocksWriteTotal => "locks_write_total".to_string(),
+            Self::ProcessCPUTotalSeconds => "cpu_total_seconds".to_string(),
+            Self::ProcessGoRoutineTotal => "go_routine_total".to_string(),
+            Self::ProcessIORCharBytes => "io_rchar_bytes".to_string(),
+            Self::ProcessIOReadBytes => "io_read_bytes".to_string(),
+            Self::ProcessIOWCharBytes => "io_wchar_bytes".to_string(),
+            Self::ProcessIOWriteBytes => "io_write_bytes".to_string(),
+            Self::ProcessStartTimeSeconds => "start_time_seconds".to_string(),
+            Self::ProcessUptimeSeconds => "uptime_seconds".to_string(),
+            Self::ProcessFileDescriptorLimitTotal => "file_descriptor_limit_total".to_string(),
+            Self::ProcessFileDescriptorOpenTotal => "file_descriptor_open_total".to_string(),
+            Self::ProcessSyscallReadTotal => "syscall_read_total".to_string(),
+            Self::ProcessSyscallWriteTotal => "syscall_write_total".to_string(),
+            Self::ProcessResidentMemoryBytes => "resident_memory_bytes".to_string(),
+            Self::ProcessVirtualMemoryBytes => "virtual_memory_bytes".to_string(),
+            Self::ProcessVirtualMemoryMaxBytes => "virtual_memory_max_bytes".to_string(),
+
+            // Process-level system monitoring metrics (migrated from rustfs-obs::system)
+            Self::ProcessCPUUsage => "cpu_usage".to_string(),
+            Self::ProcessCPUUtilization => "cpu_utilization".to_string(),
+            Self::ProcessDiskIO => "disk_io".to_string(),
+            Self::ProcessNetworkIO => "network_io".to_string(),
+            Self::ProcessNetworkIOPerInterface => "network_io_per_interface".to_string(),
+            Self::ProcessGpuMemoryUsage => "gpu_memory_usage".to_string(),
+            Self::ProcessStatus => "status".to_string(),
+
+            Self::Custom(name) => name.clone(),
+        }
+    }
+}
+
+impl From<String> for MetricName {
+    fn from(s: String) -> Self {
+        Self::Custom(s)
+    }
+}
+
+impl From<&str> for MetricName {
+    fn from(s: &str) -> Self {
+        Self::Custom(s.to_string())
+    }
+}

crates/obs/src/metrics/schema/entry/metric_type.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// MetricType - Indicates the type of indicator
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MetricType {
+    Counter,
+    Gauge,
+    Histogram,
+}
+
+impl MetricType {
+    /// convert the metric type to a string representation
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Counter => "counter",
+            Self::Gauge => "gauge",
+            Self::Histogram => "histogram",
+        }
+    }
+
+    /// Convert the metric type to the Prometheus value type
+    /// In a Rust implementation, this might return the corresponding Prometheus Rust client type
+    #[allow(dead_code)]
+    pub fn as_prom(&self) -> &'static str {
+        match self {
+            Self::Counter => "counter.",
+            Self::Gauge => "gauge.",
+            Self::Histogram => "histogram.", // Histograms still use the counter value in Prometheus
+        }
+    }
+}

crates/obs/src/metrics/schema/entry/mod.rs

@@ -0,0 +1,129 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{MetricDescriptor, MetricName, MetricNamespace, MetricSubsystem, MetricType};
+
+pub mod descriptor;
+pub mod metric_name;
+pub mod metric_type;
+pub mod namespace;
+mod path_utils;
+pub mod subsystem;
+
+/// Create a new counter metric descriptor
+pub fn new_counter_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Counter,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+/// create a new dashboard metric descriptor
+pub fn new_gauge_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Gauge,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+/// create a new histogram indicator descriptor
+#[allow(dead_code)]
+pub fn new_histogram_md(
+    name: impl Into<MetricName>,
+    help: impl Into<String>,
+    labels: &[&str],
+    subsystem: impl Into<MetricSubsystem>,
+) -> MetricDescriptor {
+    MetricDescriptor::new(
+        name.into(),
+        MetricType::Histogram,
+        help.into(),
+        labels.iter().map(|&s| s.to_string()).collect(),
+        MetricNamespace::RustFS,
+        subsystem,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{MetricName, MetricNamespace, MetricSubsystem, subsystems};
+
+    #[test]
+    fn test_new_histogram_md() {
+        // create a histogram indicator descriptor
+        let histogram_md = new_histogram_md(
+            MetricName::TtfbDistribution,
+            "test the response time distribution",
+            &["api", "method", "le"],
+            subsystems::API_REQUESTS,
+        );
+
+        // verify that the metric type is correct
+        assert_eq!(histogram_md.metric_type, MetricType::Histogram);
+
+        // verify that the metric name is correct
+        assert_eq!(histogram_md.name.as_str(), "seconds_distribution");
+
+        // verify that the help information is correct
+        assert_eq!(histogram_md.help, "test the response time distribution");
+
+        // Verify that the label is correct
+        assert_eq!(histogram_md.variable_labels.len(), 3);
+        assert!(histogram_md.variable_labels.contains(&"api".to_string()));
+        assert!(histogram_md.variable_labels.contains(&"method".to_string()));
+        assert!(histogram_md.variable_labels.contains(&"le".to_string()));
+
+        // Verify that the namespace is correct
+        assert_eq!(histogram_md.namespace, MetricNamespace::RustFS);
+
+        // Verify that the subsystem is correct
+        assert_eq!(histogram_md.subsystem, MetricSubsystem::ApiRequests);
+
+        // Verify that the full metric name generated is formatted correctly
+        assert_eq!(histogram_md.get_full_metric_name(), "rustfs_api_requests_seconds_distribution");
+
+        // Tests use custom subsystems
+        let custom_histogram_md = new_histogram_md(
+            "custom_latency_distribution",
+            "custom latency distribution",
+            &["endpoint", "le"],
+            MetricSubsystem::new("/custom/path-metrics"),
+        );
+
+        // Verify the custom name and subsystem
+        assert_eq!(
+            custom_histogram_md.get_full_metric_name(),
+            "rustfs_custom_path_metrics_custom_latency_distribution"
+        );
+    }
+}

crates/obs/src/metrics/schema/entry/namespace.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// The metric namespace, which represents the top-level grouping of the metric
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MetricNamespace {
+    RustFS,
+}
+
+impl MetricNamespace {
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::RustFS => "rustfs",
+        }
+    }
+}

crates/obs/src/metrics/schema/entry/path_utils.rs

@@ -0,0 +1,33 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Format the path to the metric name format
+/// Replace '/' and '-' with '_'
+#[allow(dead_code)]
+pub fn format_path_to_metric_name(path: &str) -> String {
+    path.trim_start_matches('/').replace(['/', '-'], "_")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_path_to_metric_name() {
+        assert_eq!(format_path_to_metric_name("/api/requests"), "api_requests");
+        assert_eq!(format_path_to_metric_name("/system/network/internode"), "system_network_internode");
+        assert_eq!(format_path_to_metric_name("/bucket-api"), "bucket_api");
+        assert_eq!(format_path_to_metric_name("cluster/health"), "cluster_health");
+    }
+}

crates/obs/src/metrics/schema/entry/subsystem.rs

@@ -0,0 +1,245 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::entry::path_utils::format_path_to_metric_name;
+
+/// The metrics subsystem is a subgroup of metrics within a namespace
+/// The metrics subsystem, which represents a subgroup of metrics within a namespace
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum MetricSubsystem {
+    // API related subsystems
+    ApiRequests,
+
+    // bucket related subsystems
+    BucketApi,
+    BucketReplication,
+
+    // system related subsystems
+    SystemNetworkInternode,
+    SystemDrive,
+    SystemMemory,
+    SystemCpu,
+    SystemProcess,
+
+    SystemGpu,
+    // debug related subsystems
+    DebugGo,
+
+    // cluster related subsystems
+    ClusterHealth,
+    ClusterUsageObjects,
+    ClusterUsageBuckets,
+    ClusterErasureSet,
+    ClusterIam,
+    ClusterConfig,
+
+    // other service related subsystems
+    Ilm,
+    Audit,
+    Replication,
+    Notification,
+    Scanner,
+
+    // Custom paths
+    Custom(String),
+}
+
+impl MetricSubsystem {
+    /// Gets the original path string
+    pub fn path(&self) -> &str {
+        match self {
+            // api related subsystems
+            Self::ApiRequests => "/api/requests",
+
+            // bucket related subsystems
+            Self::BucketApi => "/bucket/api",
+            Self::BucketReplication => "/bucket/replication",
+
+            // system related subsystems
+            Self::SystemNetworkInternode => "/system/network/internode",
+            Self::SystemDrive => "/system/drive",
+            Self::SystemMemory => "/system/memory",
+            Self::SystemCpu => "/system/cpu",
+            Self::SystemProcess => "/system/process",
+
+            // debug related subsystems
+            Self::SystemGpu => "/system/gpu",
+            Self::DebugGo => "/debug/go",
+
+            // cluster related subsystems
+            Self::ClusterHealth => "/cluster/health",
+            Self::ClusterUsageObjects => "/cluster/usage/objects",
+            Self::ClusterUsageBuckets => "/cluster/usage/buckets",
+            Self::ClusterErasureSet => "/cluster/erasure-set",
+            Self::ClusterIam => "/cluster/iam",
+            Self::ClusterConfig => "/cluster/config",
+
+            // other service related subsystems
+            Self::Ilm => "/ilm",
+            Self::Audit => "/audit",
+            Self::Replication => "/replication",
+            Self::Notification => "/notification",
+            Self::Scanner => "/scanner",
+
+            // Custom paths
+            Self::Custom(path) => path,
+        }
+    }
+
+    /// Get the formatted metric name format string
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> String {
+        format_path_to_metric_name(self.path())
+    }
+
+    /// Create a subsystem enumeration from a path string
+    pub fn from_path(path: &str) -> Self {
+        match path {
+            // API-related subsystems
+            "/api/requests" => Self::ApiRequests,
+
+            // Bucket-related subsystems
+            "/bucket/api" => Self::BucketApi,
+            "/bucket/replication" => Self::BucketReplication,
+
+            // System-related subsystems
+            "/system/network/internode" => Self::SystemNetworkInternode,
+            "/system/drive" => Self::SystemDrive,
+            "/system/memory" => Self::SystemMemory,
+            "/system/cpu" => Self::SystemCpu,
+            "/system/process" => Self::SystemProcess,
+
+            "/system/gpu" => Self::SystemGpu,
+            // Debug related subsystems
+            "/debug/go" => Self::DebugGo,
+
+            // Cluster-related subsystems
+            "/cluster/health" => Self::ClusterHealth,
+            "/cluster/usage/objects" => Self::ClusterUsageObjects,
+            "/cluster/usage/buckets" => Self::ClusterUsageBuckets,
+            "/cluster/erasure-set" => Self::ClusterErasureSet,
+            "/cluster/iam" => Self::ClusterIam,
+            "/cluster/config" => Self::ClusterConfig,
+
+            // Other service-related subsystems
+            "/ilm" => Self::Ilm,
+            "/audit" => Self::Audit,
+            "/replication" => Self::Replication,
+            "/notification" => Self::Notification,
+            "/scanner" => Self::Scanner,
+
+            // Treat other paths as custom subsystems
+            _ => Self::Custom(path.to_string()),
+        }
+    }
+
+    /// A convenient way to create custom subsystems directly
+    #[allow(dead_code)]
+    pub fn new(path: impl Into<String>) -> Self {
+        Self::Custom(path.into())
+    }
+}
+
+/// Implementations that facilitate conversion to and from strings
+impl From<&str> for MetricSubsystem {
+    fn from(s: &str) -> Self {
+        Self::from_path(s)
+    }
+}
+
+impl From<String> for MetricSubsystem {
+    fn from(s: String) -> Self {
+        Self::from_path(&s)
+    }
+}
+
+impl std::fmt::Display for MetricSubsystem {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.path())
+    }
+}
+
+#[allow(dead_code)]
+pub mod subsystems {
+    use super::MetricSubsystem;
+
+    // cluster base path constant
+    pub const CLUSTER_BASE_PATH: &str = "/cluster";
+
+    // Quick access to constants for each subsystem
+    pub const API_REQUESTS: MetricSubsystem = MetricSubsystem::ApiRequests;
+    pub const BUCKET_API: MetricSubsystem = MetricSubsystem::BucketApi;
+    pub const BUCKET_REPLICATION: MetricSubsystem = MetricSubsystem::BucketReplication;
+    pub const SYSTEM_GPU: MetricSubsystem = MetricSubsystem::SystemGpu;
+    pub const SYSTEM_NETWORK_INTERNODE: MetricSubsystem = MetricSubsystem::SystemNetworkInternode;
+    pub const SYSTEM_DRIVE: MetricSubsystem = MetricSubsystem::SystemDrive;
+    pub const SYSTEM_MEMORY: MetricSubsystem = MetricSubsystem::SystemMemory;
+    pub const SYSTEM_CPU: MetricSubsystem = MetricSubsystem::SystemCpu;
+    pub const SYSTEM_PROCESS: MetricSubsystem = MetricSubsystem::SystemProcess;
+    pub const DEBUG_GO: MetricSubsystem = MetricSubsystem::DebugGo;
+    pub const CLUSTER_HEALTH: MetricSubsystem = MetricSubsystem::ClusterHealth;
+    pub const CLUSTER_USAGE_OBJECTS: MetricSubsystem = MetricSubsystem::ClusterUsageObjects;
+    pub const CLUSTER_USAGE_BUCKETS: MetricSubsystem = MetricSubsystem::ClusterUsageBuckets;
+    pub const CLUSTER_ERASURE_SET: MetricSubsystem = MetricSubsystem::ClusterErasureSet;
+    pub const CLUSTER_IAM: MetricSubsystem = MetricSubsystem::ClusterIam;
+    pub const CLUSTER_CONFIG: MetricSubsystem = MetricSubsystem::ClusterConfig;
+    pub const ILM: MetricSubsystem = MetricSubsystem::Ilm;
+    pub const AUDIT: MetricSubsystem = MetricSubsystem::Audit;
+    pub const REPLICATION: MetricSubsystem = MetricSubsystem::Replication;
+    pub const NOTIFICATION: MetricSubsystem = MetricSubsystem::Notification;
+    pub const SCANNER: MetricSubsystem = MetricSubsystem::Scanner;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{MetricDescriptor, MetricName, MetricNamespace, MetricType};
+
+    #[test]
+    fn test_metric_subsystem_formatting() {
+        assert_eq!(MetricSubsystem::ApiRequests.as_str(), "api_requests");
+        assert_eq!(MetricSubsystem::SystemNetworkInternode.as_str(), "system_network_internode");
+        assert_eq!(MetricSubsystem::BucketApi.as_str(), "bucket_api");
+        assert_eq!(MetricSubsystem::ClusterHealth.as_str(), "cluster_health");
+
+        // Test custom paths
+        let custom = MetricSubsystem::new("/custom/path-test");
+        assert_eq!(custom.as_str(), "custom_path_test");
+    }
+
+    #[test]
+    fn test_metric_descriptor_name_generation() {
+        let md = MetricDescriptor::new(
+            MetricName::ApiRequestsTotal,
+            MetricType::Counter,
+            "Test help".to_string(),
+            vec!["label1".to_string(), "label2".to_string()],
+            MetricNamespace::RustFS,
+            MetricSubsystem::ApiRequests,
+        );
+
+        assert_eq!(md.get_full_metric_name(), "rustfs_api_requests_total");
+
+        let custom_md = MetricDescriptor::new(
+            MetricName::Custom("test_metric".to_string()),
+            MetricType::Gauge,
+            "Test help".to_string(),
+            vec!["label1".to_string()],
+            MetricNamespace::RustFS,
+            MetricSubsystem::new("/custom/path-with-dash"),
+        );
+
+        assert_eq!(custom_md.get_full_metric_name(), "rustfs_custom_path_with_dash_test_metric");
+    }
+}

crates/obs/src/metrics/schema/ilm.rs

@@ -0,0 +1,63 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmExpiryPendingTasks,
+        "Number of pending ILM expiry tasks in the queue",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_ACTIVE_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmTransitionActiveTasks,
+        "Number of active ILM transition tasks",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmTransitionPendingTasks,
+        "Number of pending ILM transition tasks in the queue",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmTransitionMissedImmediateTasks,
+        "Number of missed immediate ILM transition tasks",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmVersionsScanned,
+        "Total number of object versions checked for ILM actions since server start",
+        &[],
+        subsystems::ILM,
+    )
+});

crates/obs/src/metrics/schema/mod.rs

@@ -0,0 +1,47 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod audit;
+pub mod bucket;
+pub mod bucket_replication;
+pub mod cluster;
+pub mod cluster_config;
+pub mod cluster_erasure_set;
+pub mod cluster_health;
+pub mod cluster_iam;
+pub mod cluster_notification;
+pub mod cluster_usage;
+pub mod entry;
+pub mod ilm;
+pub mod node_bucket;
+pub mod node_disk;
+pub mod notification_target;
+pub mod process_resource;
+pub mod replication;
+pub mod request;
+pub mod scanner;
+pub mod system_cpu;
+pub mod system_drive;
+pub mod system_gpu;
+pub mod system_memory;
+pub mod system_network;
+pub mod system_process;
+
+pub use entry::descriptor::MetricDescriptor;
+pub use entry::metric_name::MetricName;
+pub use entry::metric_type::MetricType;
+pub use entry::namespace::MetricNamespace;
+pub use entry::subsystem::MetricSubsystem;
+pub use entry::subsystem::subsystems;
+pub use entry::{new_counter_md, new_gauge_md, new_histogram_md};

crates/obs/src/metrics/schema/node_bucket.rs

@@ -0,0 +1,50 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+const BUCKET_LABEL: &str = "bucket";
+
+/// Total bytes used by the bucket
+pub static BUCKET_USAGE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("usage_bytes".to_string()),
+        "Total bytes used by the bucket",
+        &[BUCKET_LABEL],
+        subsystems::BUCKET_API,
+    )
+});
+
+/// Total number of objects in the bucket
+pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("objects_total".to_string()),
+        "Total number of objects in the bucket",
+        &[BUCKET_LABEL],
+        subsystems::BUCKET_API,
+    )
+});
+
+/// Quota limit in bytes for the bucket
+pub static BUCKET_QUOTA_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("quota_bytes".to_string()),
+        "Quota limit in bytes for the bucket",
+        &[BUCKET_LABEL],
+        subsystems::BUCKET_API,
+    )
+});

crates/obs/src/metrics/schema/node_disk.rs

@@ -0,0 +1,51 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
+use std::sync::LazyLock;
+
+const SERVER_LABEL: &str = "server";
+const DRIVE_LABEL: &str = "drive";
+
+/// Total disk capacity in bytes
+pub static NODE_DISK_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("disk_total_bytes".to_string()),
+        "Total disk capacity in bytes",
+        &[SERVER_LABEL, DRIVE_LABEL],
+        MetricSubsystem::new("/node"),
+    )
+});
+
+/// Used disk space in bytes
+pub static NODE_DISK_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("disk_used_bytes".to_string()),
+        "Used disk space in bytes",
+        &[SERVER_LABEL, DRIVE_LABEL],
+        MetricSubsystem::new("/node"),
+    )
+});
+
+/// Free disk space in bytes
+pub static NODE_DISK_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("disk_free_bytes".to_string()),
+        "Free disk space in bytes",
+        &[SERVER_LABEL, DRIVE_LABEL],
+        MetricSubsystem::new("/node"),
+    )
+});

crates/obs/src/metrics/schema/notification_target.rs

@@ -0,0 +1,50 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub const TARGET_ID: &str = "target_id";
+pub const TARGET_TYPE: &str = "target_type";
+
+const NOTIFICATION_TARGET_LABELS: [&str; 2] = [TARGET_ID, TARGET_TYPE];
+
+pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationTargetFailedMessages,
+        "Total number of notification messages that permanently failed to send",
+        &NOTIFICATION_TARGET_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::NotificationTargetQueueLength,
+        "Number of queued notification messages pending delivery",
+        &NOTIFICATION_TARGET_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
+pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::NotificationTargetTotalMessages,
+        "Total number of notification messages successfully delivered",
+        &NOTIFICATION_TARGET_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});

crates/obs/src/metrics/schema/process_resource.rs

@@ -0,0 +1,48 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
+use std::sync::LazyLock;
+
+/// CPU usage of the RustFS process as a percentage
+pub static PROCESS_CPU_PERCENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("cpu_percent".to_string()),
+        "CPU usage of the RustFS process as a percentage",
+        &[],
+        MetricSubsystem::new("/process"),
+    )
+});
+
+/// Resident memory usage of the RustFS process in bytes
+pub static PROCESS_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("memory_bytes".to_string()),
+        "Resident memory usage of the RustFS process in bytes",
+        &[],
+        MetricSubsystem::new("/process"),
+    )
+});
+
+/// Uptime of the RustFS process in seconds
+pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("uptime_seconds".to_string()),
+        "Uptime of the RustFS process in seconds",
+        &[],
+        MetricSubsystem::new("/process"),
+    )
+});

crates/obs/src/metrics/schema/replication.rs

@@ -0,0 +1,135 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageActiveWorkers,
+        "Average number of active replication workers",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageQueuedBytes,
+        "Average number of bytes queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageQueuedCount,
+        "Average number of objects queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationAverageDataTransferRate,
+        "Average replication data transfer rate in bytes/sec",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationCurrentActiveWorkers,
+        "Total number of active replication workers",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationCurrentDataTransferRate,
+        "Current replication data transfer rate in bytes/sec",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationLastMinuteQueuedBytes,
+        "Number of bytes queued for replication in the last full minute",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationLastMinuteQueuedCount,
+        "Number of objects queued for replication in the last full minute",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxActiveWorkers,
+        "Maximum number of active replication workers seen since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxQueuedBytes,
+        "Maximum number of bytes queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxQueuedCount,
+        "Maximum number of objects queued for replication since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationMaxDataTransferRate,
+        "Maximum replication data transfer rate in bytes/sec seen since server start",
+        &[],
+        subsystems::REPLICATION,
+    )
+});
+
+pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ReplicationRecentBacklogCount,
+        "Total number of objects seen in replication backlog in the last 5 minutes",
+        &[],
+        subsystems::REPLICATION,
+    )
+});

crates/obs/src/metrics/schema/request.rs

@@ -0,0 +1,159 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+/// name label
+pub const NAME_LABEL: &str = "name";
+/// type label
+pub const TYPE_LABEL: &str = "type";
+/// le label (for histogram buckets)
+pub const LE_LABEL: &str = "le";
+
+pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedAuthTotal,
+        "Total number of requests rejected for auth failure",
+        &["type"],
+        subsystems::API_REQUESTS,
+    )
+});
+
+pub static API_REJECTED_HEADER_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedHeaderTotal,
+        "Total number of requests rejected for invalid header",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REJECTED_TIMESTAMP_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedTimestampTotal,
+        "Total number of requests rejected for invalid timestamp",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REJECTED_INVALID_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRejectedInvalidTotal,
+        "Total number of invalid requests",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_WAITING_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsWaitingTotal,
+        "Total number of requests in the waiting queue",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_INCOMING_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsIncomingTotal,
+        "Total number of incoming requests",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ApiRequestsInFlightTotal,
+        "Total number of requests currently in flight",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTotal,
+        "Total number of requests",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsErrorsTotal,
+        "Total number of requests with (4xx and 5xx) errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests5xxErrorsTotal,
+        "Total number of requests with 5xx errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequests4xxErrorsTotal,
+        "Total number of requests with 4xx errors",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsCanceledTotal,
+        "Total number of requests canceled by the client",
+        &["name", "type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiRequestsTTFBSecondsDistribution,
+        "Distribution of time to first byte across API calls",
+        &["name", "type", "le"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficSentBytes,
+        "Total number of bytes sent",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});
+
+pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ApiTrafficRecvBytes,
+        "Total number of bytes received",
+        &["type"],
+        MetricSubsystem::ApiRequests,
+    )
+});

crates/obs/src/metrics/schema/scanner.rs

@@ -0,0 +1,72 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerBucketScansFinished,
+        "Total number of bucket scans finished since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerBucketScansStarted,
+        "Total number of bucket scans started since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_DIRECTORIES_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerDirectoriesScanned,
+        "Total number of directories scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_OBJECTS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerObjectsScanned,
+        "Total number of unique objects scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerVersionsScanned,
+        "Total number of object versions scanned since server start",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_ACTIVITY_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastActivitySeconds,
+        "Time elapsed (in seconds) since last scan activity.",
+        &[],
+        subsystems::SCANNER,
+    )
+});

crates/obs/src/metrics/schema/system_cpu.rs

@@ -0,0 +1,49 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+/// CPU system-related metric descriptors
+use std::sync::LazyLock;
+
+pub static SYS_CPU_AVG_IDLE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIdle, "Average CPU idle time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIOWait, "Average CPU IOWait time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_LOAD_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_LOAD_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::SysCPULoadPerc,
+        "CPU load average 1min (percentage)",
+        &[],
+        subsystems::SYSTEM_CPU,
+    )
+});
+
+pub static SYS_CPU_NICE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_STEAL_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_SYSTEM_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[], subsystems::SYSTEM_CPU));
+
+pub static SYS_CPU_USER_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[], subsystems::SYSTEM_CPU));

crates/obs/src/metrics/schema/system_drive.rs

@@ -0,0 +1,214 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// drive related labels
+pub const DRIVE_LABEL: &str = "drive";
+/// server label
+pub const SERVER_LABEL: &str = "server";
+/// pool index label
+pub const POOL_INDEX_LABEL: &str = "pool_index";
+/// set index label
+pub const SET_INDEX_LABEL: &str = "set_index";
+/// drive index label
+pub const DRIVE_INDEX_LABEL: &str = "drive_index";
+/// API label
+pub const API_LABEL: &str = "api";
+
+/// All drive-related labels
+pub const ALL_DRIVE_LABELS: [&str; 4] = [DRIVE_LABEL, POOL_INDEX_LABEL, SET_INDEX_LABEL, DRIVE_INDEX_LABEL];
+
+pub static DRIVE_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveUsedBytes,
+        "Total storage used on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveFreeBytes,
+        "Total storage free on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveTotalBytes,
+        "Total storage available on a drive in bytes",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_USED_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveUsedInodes,
+        "Total used inodes on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_FREE_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveFreeInodes,
+        "Total free inodes on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TOTAL_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveTotalInodes,
+        "Total inodes available on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_TIMEOUT_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveTimeoutErrorsTotal,
+        "Total timeout errors on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_IO_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveIOErrorsTotal,
+        "Total I/O errors on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_AVAILABILITY_ERRORS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveAvailabilityErrorsTotal,
+        "Total availability errors (I/O errors, timeouts) on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WAITING_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWaitingIO,
+        "Total waiting I/O operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_API_LATENCY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveAPILatencyMicros,
+        "Average last minute latency in µs for drive API storage operations",
+        &[&ALL_DRIVE_LABELS[..], &[API_LABEL]].concat(),
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveHealth,
+        "Drive health (0 = offline, 1 = healthy, 2 = healing)",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveOfflineCount, "Count of offline drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_ONLINE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveOnlineCount, "Count of online drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_COUNT_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::DriveCount, "Count of all drives", &[], subsystems::SYSTEM_DRIVE));
+
+pub static DRIVE_READS_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsPerSec,
+        "Reads per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_READS_KB_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsKBPerSec,
+        "Kilobytes read per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_READS_AWAIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveReadsAwait,
+        "Average time for read requests served on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesPerSec,
+        "Writes per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_KB_PER_SEC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesKBPerSec,
+        "Kilobytes written per second on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_WRITES_AWAIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DriveWritesAwait,
+        "Average time for write requests served on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_PERC_UTIL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::DrivePercUtil,
+        "Percentage of time the disk was busy",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});

crates/obs/src/metrics/schema/system_gpu.rs

@@ -0,0 +1,35 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! GPU-related metric descriptors.
+//!
+//! This module defines metric descriptors for GPU monitoring,
+//! including GPU memory usage metrics.
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Process GPU memory usage metric descriptor.
+///
+/// Records the amount of physical GPU memory in use by the process.
+pub static PROCESS_GPU_MEMORY_USAGE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessGpuMemoryUsage,
+        "The amount of physical GPU memory in use",
+        &[],
+        subsystems::SYSTEM_GPU,
+    )
+});

crates/obs/src/metrics/schema/system_memory.rs

@@ -0,0 +1,56 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Total memory available on the node
+pub static MEM_TOTAL_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemTotal, "Total memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory currently in use on the node
+pub static MEM_USED_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Percentage of total memory currently in use
+pub static MEM_USED_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::MemUsedPerc,
+        "Used memory percentage on the node",
+        &[],
+        subsystems::SYSTEM_MEMORY,
+    )
+});
+
+/// Memory not currently in use and available for allocation
+pub static MEM_FREE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory used for file buffers by the kernel
+pub static MEM_BUFFERS_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemBuffers, "Buffers memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory used for caching file data by the kernel
+pub static MEM_CACHE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemCache, "Cache memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Memory shared between multiple processes
+pub static MEM_SHARED_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemShared, "Shared memory on the node", &[], subsystems::SYSTEM_MEMORY));
+
+/// Estimate of memory available for new applications without swapping
+pub static MEM_AVAILABLE_MD: LazyLock<MetricDescriptor> =
+    LazyLock::new(|| new_gauge_md(MetricName::MemAvailable, "Available memory on the node", &[], subsystems::SYSTEM_MEMORY));

crates/obs/src/metrics/schema/system_network.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Total number of failed internode calls counter
+pub static INTERNODE_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeErrorsTotal,
+        "Total number of failed internode calls",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// TCP dial timeouts and errors counter
+pub static INTERNODE_DIAL_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeDialErrorsTotal,
+        "Total number of internode TCP dial timeouts and errors",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Average dial time gauge in nanoseconds
+pub static INTERNODE_DIAL_AVG_TIME_NANOS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::InternodeDialAvgTimeNanos,
+        "Average dial time of internode TCP calls in nanoseconds",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Outbound network traffic counter in bytes
+pub static INTERNODE_SENT_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeSentBytesTotal,
+        "Total number of bytes sent to other peer nodes",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});
+
+/// Inbound network traffic counter in bytes
+pub static INTERNODE_RECV_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::InternodeRecvBytesTotal,
+        "Total number of bytes received from other peer nodes",
+        &[],
+        subsystems::SYSTEM_NETWORK_INTERNODE,
+    )
+});

crates/obs/src/metrics/schema/system_process.rs

@@ -0,0 +1,252 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
+use std::sync::LazyLock;
+
+/// Number of current READ locks on this peer
+pub static PROCESS_LOCKS_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessLocksReadTotal,
+        "Number of current READ locks on this peer",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Number of current WRITE locks on this peer
+pub static PROCESS_LOCKS_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessLocksWriteTotal,
+        "Number of current WRITE locks on this peer",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total user and system CPU time spent in seconds
+pub static PROCESS_CPU_TOTAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessCPUTotalSeconds,
+        "Total user and system CPU time spent in seconds",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total number of go routines running
+pub static PROCESS_GO_ROUTINE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessGoRoutineTotal,
+        "Total number of go routines running",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes read by the process from the underlying storage system including cache
+pub static PROCESS_IO_RCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIORCharBytes,
+        "Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes read by the process from the underlying storage system
+pub static PROCESS_IO_READ_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOReadBytes,
+        "Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes written by the process to the underlying storage system including page cache
+pub static PROCESS_IO_WCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOWCharBytes,
+        "Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total bytes written by the process to the underlying storage system
+pub static PROCESS_IO_WRITE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessIOWriteBytes,
+        "Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Start time for RustFS process in seconds since Unix epoch
+pub static PROCESS_START_TIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessStartTimeSeconds,
+        "Start time for RustFS process in seconds since Unix epoch",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Uptime for RustFS process in seconds
+pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessUptimeSeconds,
+        "Uptime for RustFS process in seconds",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Limit on total number of open file descriptors for the RustFS Server process
+pub static PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessFileDescriptorLimitTotal,
+        "Limit on total number of open file descriptors for the RustFS Server process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total number of open file descriptors by the RustFS Server process
+pub static PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessFileDescriptorOpenTotal,
+        "Total number of open file descriptors by the RustFS Server process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total read SysCalls to the kernel
+pub static PROCESS_SYSCALL_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessSyscallReadTotal,
+        "Total read SysCalls to the kernel. /proc/[pid]/io syscr",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Total write SysCalls to the kernel
+pub static PROCESS_SYSCALL_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ProcessSyscallWriteTotal,
+        "Total write SysCalls to the kernel. /proc/[pid]/io syscw",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Resident memory size in bytes
+pub static PROCESS_RESIDENT_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessResidentMemoryBytes,
+        "Resident memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Virtual memory size in bytes
+pub static PROCESS_VIRTUAL_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessVirtualMemoryBytes,
+        "Virtual memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Maximum virtual memory size in bytes
+pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessVirtualMemoryMaxBytes,
+        "Maximum virtual memory size in bytes",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+// ============================================================================
+// Process-level system monitoring metrics (migrated from rustfs-obs::system)
+// ============================================================================
+
+/// Process CPU usage percentage (0-100)
+pub static PROCESS_CPU_USAGE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessCPUUsage,
+        "The percentage of CPU in use by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process CPU utilization percentage (considering multiple cores)
+pub static PROCESS_CPU_UTILIZATION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessCPUUtilization,
+        "The amount of CPU in use by the process (considering multiple cores)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process disk I/O bytes
+pub static PROCESS_DISK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessDiskIO,
+        "Disk bytes transferred by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process network I/O bytes
+pub static PROCESS_NETWORK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessNetworkIO,
+        "Network bytes transferred by the process",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process network I/O bytes per interface
+pub static PROCESS_NETWORK_IO_PER_INTERFACE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessNetworkIOPerInterface,
+        "Network bytes transferred by the process (per interface)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});
+
+/// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)
+pub static PROCESS_STATUS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ProcessStatus,
+        "Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)",
+        &[],
+        subsystems::SYSTEM_PROCESS,
+    )
+});

crates/obs/src/metrics/stats_collector.rs

@@ -0,0 +1,242 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code)]
+
+//! Statistics collection functions for metrics.
+//!
+//! This module contains functions that collect statistics from various
+//! RustFS internal sources (storage layer, bucket monitor, system info)
+//! and convert them to the Stats structs used by collectors.
+
+use crate::metrics::collectors::{
+    BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ProcessStats, ProcessStatusType, ResourceStats,
+};
+use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
+use rustfs_ecstore::data_usage::load_data_usage_from_backend;
+use rustfs_ecstore::global::get_global_bucket_monitor;
+use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
+use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
+use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
+use tracing::{instrument, warn};
+
+/// Collect cluster statistics from the storage layer.
+#[instrument]
+pub async fn collect_cluster_stats() -> ClusterStats {
+    let Some(store) = new_object_layer_fn() else {
+        return ClusterStats::default();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
+    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
+    let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
+    let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
+
+    // Get bucket and object counts from data usage info
+    let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
+        Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
+        Err(e) => {
+            warn!("Failed to load data usage from backend: {}", e);
+            // Fall back to bucket list for buckets_count, objects_count stays 0
+            let buckets = store
+                .list_bucket(&BucketOptions {
+                    cached: true,
+                    ..Default::default()
+                })
+                .await
+                .unwrap_or_else(|e| {
+                    warn!("Failed to list buckets for cluster metrics: {}", e);
+                    Vec::new()
+                });
+            (buckets.len() as u64, 0)
+        }
+    };
+
+    ClusterStats {
+        raw_capacity_bytes: raw_capacity,
+        usable_capacity_bytes: usable_capacity,
+        used_bytes: used,
+        free_bytes: free,
+        objects_count,
+        buckets_count,
+    }
+}
+
+/// Collect bucket statistics from the storage layer.
+pub async fn collect_bucket_stats() -> Vec<BucketStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    // Load data usage info from backend to get bucket sizes and object counts
+    let data_usage = match load_data_usage_from_backend(store.clone()).await {
+        Ok(info) => Some(info),
+        Err(e) => {
+            warn!("Failed to load data usage for bucket metrics: {}", e);
+            None
+        }
+    };
+
+    // List all buckets
+    let buckets = match store
+        .list_bucket(&BucketOptions {
+            cached: true,
+            ..Default::default()
+        })
+        .await
+    {
+        Ok(buckets) => buckets,
+        Err(e) => {
+            warn!("Failed to list buckets for bucket metrics: {}", e);
+            return Vec::new();
+        }
+    };
+
+    let mut stats = Vec::with_capacity(buckets.len());
+
+    for bucket in buckets {
+        if bucket.name.starts_with('.') {
+            continue;
+        }
+
+        // Get size and objects_count from data usage info
+        let (size_bytes, objects_count) = data_usage
+            .as_ref()
+            .and_then(|du| du.buckets_usage.get(&bucket.name))
+            .map(|bui| (bui.size, bui.objects_count))
+            .unwrap_or((0, 0));
+
+        // Get quota from bucket metadata
+        let quota_bytes = match get_quota_config(&bucket.name).await {
+            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
+            Err(_) => 0, // No quota configured or error
+        };
+
+        stats.push(BucketStats {
+            name: bucket.name,
+            size_bytes,
+            objects_count,
+            quota_bytes,
+        });
+    }
+
+    stats
+}
+
+/// Collect bucket replication bandwidth stats from the global monitor.
+pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
+    let Some(monitor) = get_global_bucket_monitor() else {
+        return Vec::new();
+    };
+
+    monitor
+        .get_report(|_| true)
+        .bucket_stats
+        .into_iter()
+        .map(|(opts, details)| {
+            let target_arn = opts.replication_arn;
+            let limit_bytes_per_sec = u64::try_from(details.limit_bytes_per_sec).unwrap_or_else(|_| {
+                warn!(
+                    "Invalid bandwidth limit value for target {:?}: {}",
+                    target_arn, details.limit_bytes_per_sec
+                );
+                0
+            });
+
+            BucketReplicationBandwidthStats {
+                bucket: opts.name,
+                target_arn,
+                limit_bytes_per_sec,
+                current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
+            }
+        })
+        .collect()
+}
+
+/// Collect disk statistics from the storage layer.
+pub async fn collect_disk_stats() -> Vec<DiskStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    storage_info
+        .disks
+        .iter()
+        .map(|disk| DiskStats {
+            server: disk.endpoint.clone(),
+            drive: disk.drive_path.clone(),
+            total_bytes: disk.total_space,
+            used_bytes: disk.used_space,
+            free_bytes: disk.available_space,
+        })
+        .collect()
+}
+
+/// Collect resource and process statistics for the current process in one sysinfo refresh.
+#[inline]
+pub fn collect_process_resource_and_system_stats() -> (ResourceStats, ProcessStats) {
+    let (resource_snapshot, process_snapshot) = snapshot_process_resource_and_system();
+    let status = match process_snapshot.status {
+        ProcessStatusSnapshot::Running => ProcessStatusType::Running,
+        ProcessStatusSnapshot::Sleeping => ProcessStatusType::Sleeping,
+        ProcessStatusSnapshot::Zombie => ProcessStatusType::Zombie,
+        ProcessStatusSnapshot::Other => ProcessStatusType::Other,
+    };
+
+    let resource_stats = ResourceStats {
+        cpu_percent: resource_snapshot.cpu_percent,
+        memory_bytes: resource_snapshot.memory_bytes,
+        uptime_seconds: resource_snapshot.uptime_seconds,
+    };
+    let process_stats = ProcessStats {
+        locks_read_total: process_snapshot.locks_read_total,
+        locks_write_total: process_snapshot.locks_write_total,
+        cpu_total_seconds: process_snapshot.cpu_total_seconds,
+        file_descriptor_limit_total: process_snapshot.file_descriptor_limit_total,
+        file_descriptor_open_total: process_snapshot.file_descriptor_open_total,
+        go_routine_total: process_snapshot.go_routine_total,
+        io_rchar_bytes: process_snapshot.io_rchar_bytes,
+        io_read_bytes: process_snapshot.io_read_bytes,
+        io_wchar_bytes: process_snapshot.io_wchar_bytes,
+        io_write_bytes: process_snapshot.io_write_bytes,
+        resident_memory_bytes: process_snapshot.resident_memory_bytes,
+        start_time_seconds: process_snapshot.start_time_seconds,
+        status,
+        status_value: process_snapshot.status_value,
+        syscall_read_total: process_snapshot.syscall_read_total,
+        syscall_write_total: process_snapshot.syscall_write_total,
+        uptime_seconds: process_snapshot.uptime_seconds,
+        virtual_memory_bytes: process_snapshot.virtual_memory_bytes,
+        virtual_memory_max_bytes: process_snapshot.virtual_memory_max_bytes,
+    };
+
+    (resource_stats, process_stats)
+}
+
+/// Collect resource statistics for the current process.
+#[inline]
+pub fn collect_process_stats() -> ResourceStats {
+    collect_process_resource_and_system_stats().0
+}
+
+/// Collect process statistics for the current process.
+#[inline]
+pub fn collect_process_system_stats() -> ProcessStats {
+    collect_process_resource_and_system_stats().1
+}