From a4411a24d6dffa6df13292c78e239d2141b84bcb Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 18 Mar 2026 23:52:42 +0800 Subject: [PATCH] refactor(metrics): modularize collectors and add comprehensive metrics support (#2208) Co-authored-by: overtrue Co-authored-by: heihutu --- crates/metrics/src/collectors/audit.rs | 109 +++++++ crates/metrics/src/collectors/bucket.rs | 101 +++--- .../src/collectors/bucket_replication.rs | 67 ++-- crates/metrics/src/collectors/cluster.rs | 112 ++----- .../metrics/src/collectors/cluster_config.rs | 81 +++++ .../src/collectors/cluster_erasure_set.rs | 179 ++++++++++ .../metrics/src/collectors/cluster_health.rs | 85 +++++ crates/metrics/src/collectors/cluster_iam.rs | 117 +++++++ .../metrics/src/collectors/cluster_usage.rs | 199 ++++++++++++ crates/metrics/src/collectors/global.rs | 305 +++++------------- crates/metrics/src/collectors/ilm.rs | 96 ++++++ .../metrics/src/collectors/logger_webhook.rs | 122 +++++++ crates/metrics/src/collectors/mod.rs | 35 ++ crates/metrics/src/collectors/node.rs | 78 ++--- crates/metrics/src/collectors/notification.rs | 91 ++++++ crates/metrics/src/collectors/replication.rs | 107 ++++++ crates/metrics/src/collectors/request.rs | 174 ++++++++++ crates/metrics/src/collectors/resource.rs | 97 ++---- crates/metrics/src/collectors/scanner.rs | 99 ++++++ .../metrics/src/collectors/stats_collector.rs | 233 +++++++++++++ crates/metrics/src/collectors/system_cpu.rs | 104 ++++++ crates/metrics/src/collectors/system_drive.rs | 228 +++++++++++++ .../metrics/src/collectors/system_memory.rs | 102 ++++++ .../metrics/src/collectors/system_network.rs | 90 ++++++ .../metrics/src/collectors/system_process.rs | 145 +++++++++ crates/metrics/src/format.rs | 83 ++++- crates/metrics/src/metrics_type/cluster.rs | 78 +++++ .../src/metrics_type/cluster_erasure_set.rs | 27 ++ .../src/metrics_type/entry/metric_name.rs | 6 + crates/metrics/src/metrics_type/mod.rs | 4 + .../metrics/src/metrics_type/node_bucket.rs | 50 +++ crates/metrics/src/metrics_type/node_disk.rs | 51 +++ .../src/metrics_type/process_resource.rs | 48 +++ crates/metrics/src/metrics_type/request.rs | 6 + .../metrics/src/metrics_type/system_drive.rs | 2 + 35 files changed, 2959 insertions(+), 552 deletions(-) create mode 100644 crates/metrics/src/collectors/audit.rs create mode 100644 crates/metrics/src/collectors/cluster_config.rs create mode 100644 crates/metrics/src/collectors/cluster_erasure_set.rs create mode 100644 crates/metrics/src/collectors/cluster_health.rs create mode 100644 crates/metrics/src/collectors/cluster_iam.rs create mode 100644 crates/metrics/src/collectors/cluster_usage.rs create mode 100644 crates/metrics/src/collectors/ilm.rs create mode 100644 crates/metrics/src/collectors/logger_webhook.rs create mode 100644 crates/metrics/src/collectors/notification.rs create mode 100644 crates/metrics/src/collectors/replication.rs create mode 100644 crates/metrics/src/collectors/request.rs create mode 100644 crates/metrics/src/collectors/scanner.rs create mode 100644 crates/metrics/src/collectors/stats_collector.rs create mode 100644 crates/metrics/src/collectors/system_cpu.rs create mode 100644 crates/metrics/src/collectors/system_drive.rs create mode 100644 crates/metrics/src/collectors/system_memory.rs create mode 100644 crates/metrics/src/collectors/system_network.rs create mode 100644 crates/metrics/src/collectors/system_process.rs create mode 100644 crates/metrics/src/metrics_type/cluster.rs create mode 100644 crates/metrics/src/metrics_type/node_bucket.rs create mode 100644 crates/metrics/src/metrics_type/node_disk.rs create mode 100644 crates/metrics/src/metrics_type/process_resource.rs diff --git a/crates/metrics/src/collectors/audit.rs b/crates/metrics/src/collectors/audit.rs new file mode 100644 index 000000000..9eda890f1 --- /dev/null +++ b/crates/metrics/src/collectors/audit.rs @@ -0,0 +1,109 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Audit metrics collector. +//! +//! Collects audit log metrics including failed messages, queue length, +//! and total messages per target. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::audit` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::audit::*; +use std::borrow::Cow; + +/// Audit target statistics for metrics collection. +#[derive(Debug, Clone, Default)] +pub struct AuditTargetStats { + /// Target identifier + pub target_id: String, + /// Number of messages that failed to send + pub failed_messages: u64, + /// Number of unsent messages in queue + pub queue_length: u64, + /// Total number of messages sent + pub total_messages: u64, +} + +/// Collects audit metrics from the provided audit target statistics. +/// +/// Uses the metric descriptors from `metrics_type::audit` module. +/// Returns a vector of Prometheus metrics for audit statistics. +pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec { + if stats.is_empty() { + return Vec::new(); + } + + let mut metrics = Vec::with_capacity(stats.len() * 3); + for stat in stats { + let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone()); + + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64) + .with_label("target_id", target_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64) + .with_label("target_id", target_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64) + .with_label("target_id", target_id_label), + ); + } + + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collect_audit_metrics() { + let stats = vec![ + AuditTargetStats { + target_id: "target-1".to_string(), + failed_messages: 5, + queue_length: 10, + total_messages: 1000, + }, + AuditTargetStats { + target_id: "target-2".to_string(), + failed_messages: 2, + queue_length: 5, + total_messages: 500, + }, + ]; + + let metrics = collect_audit_metrics(&stats); + + assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each + + let failed = metrics + .iter() + .find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1")); + assert!(failed.is_some()); + } + + #[test] + fn test_collect_audit_metrics_empty() { + let stats: Vec = vec![]; + let metrics = collect_audit_metrics(&stats); + assert!(metrics.is_empty()); + } +} diff --git a/crates/metrics/src/collectors/bucket.rs b/crates/metrics/src/collectors/bucket.rs index d3d3a944e..ca421dc92 100644 --- a/crates/metrics/src/collectors/bucket.rs +++ b/crates/metrics/src/collectors/bucket.rs @@ -16,90 +16,56 @@ //! //! Collects usage metrics for each bucket in the cluster, including //! size, object counts, and quota information. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::node_bucket` +//! to avoid duplication of metric names, types, and help text. -use crate::MetricType; use crate::format::PrometheusMetric; +use crate::metrics_type::node_bucket::*; use std::borrow::Cow; -/// Usage statistics for a single bucket. +/// Bucket statistics for metrics collection. +/// +/// This struct provides a decoupled interface for collecting bucket metrics +/// without depending on specific internal types. HTTP handlers should populate +/// this struct from their available data sources. #[derive(Debug, Clone, Default)] pub struct BucketStats { - /// Bucket name + /// Name of the bucket pub name: String, - /// Total bytes used by the bucket + /// Total size of all objects in the bucket (bytes) pub size_bytes: u64, - /// Total number of objects in the bucket + /// Number of objects in the bucket pub objects_count: u64, - /// Quota limit in bytes (0 means no quota) + /// Quota limit for the bucket (bytes), 0 if no quota pub quota_bytes: u64, } -// Static metric definitions -const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes"; -const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total"; -const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes"; - -const HELP_SIZE: &str = "Total bytes used by the bucket"; -const HELP_OBJECTS: &str = "Total number of objects in the bucket"; -const HELP_QUOTA: &str = "Quota limit in bytes for the bucket"; - -/// Collects per-bucket usage metrics from the provided bucket statistics. +/// Collects per-bucket metrics from the provided bucket statistics. /// -/// # Metrics Produced -/// -/// For each bucket, the following metrics are produced with a `bucket` label: -/// -/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket -/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket -/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured) -/// -/// # Arguments -/// -/// * `buckets` - Slice of bucket statistics -/// -/// # Example -/// -/// ``` -/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats}; -/// -/// let buckets = vec![ -/// BucketStats { -/// name: "my-bucket".to_string(), -/// size_bytes: 1_000_000, -/// objects_count: 100, -/// quota_bytes: 10_000_000, -/// }, -/// ]; -/// let metrics = collect_bucket_metrics(&buckets); -/// assert_eq!(metrics.len(), 3); // size, objects, quota -/// ``` -#[must_use] -#[inline] +/// Uses the metric descriptors from `metrics_type::node_bucket` module. +/// Returns a vector of Prometheus metrics for all buckets. pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec { if buckets.is_empty() { return Vec::new(); } let mut metrics = Vec::with_capacity(buckets.len() * 3); - for bucket in buckets { let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone()); - // Bucket size in bytes metrics.push( - PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64) + PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64) .with_label("bucket", bucket_label.clone()), ); - // Object count metrics.push( - PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64) + PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64) .with_label("bucket", bucket_label.clone()), ); - // Quota (always emit, 0 when no quota configured for consistent PromQL queries) metrics.push( - PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64) + PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64) .with_label("bucket", bucket_label), ); } @@ -118,27 +84,28 @@ mod tests { BucketStats { name: "test-bucket".to_string(), size_bytes: 1000, - objects_count: 50, + objects_count: 100, quota_bytes: 0, }, BucketStats { - name: "other-bucket".to_string(), + name: "another-bucket".to_string(), size_bytes: 2000, - objects_count: 100, + objects_count: 200, quota_bytes: 0, }, ]; let metrics = collect_bucket_metrics(&buckets); - report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here. + report_metrics(&metrics); // 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics assert_eq!(metrics.len(), 6); - // Verify test-bucket metrics + // Verify test-bucket metrics have correct labels + let test_bucket_size_name = BUCKET_USAGE_BYTES_MD.get_full_metric_name(); let test_bucket_size = metrics .iter() - .find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket")); + .find(|m| m.name == test_bucket_size_name && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket")); assert!(test_bucket_size.is_some()); assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0)); } @@ -159,9 +126,13 @@ mod tests { assert_eq!(metrics.len(), 3); // Verify quota metric exists - let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA); + let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name(); + let quota_metric = metrics.iter().find(|m| { + m.name == quota_metric_name + && m.value == 10000.0 + && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "quota-bucket") + }); assert!(quota_metric.is_some()); - assert_eq!(quota_metric.map(|m| m.value), Some(10000.0)); } #[test] @@ -185,9 +156,13 @@ mod tests { // Zero quota should still produce a quota metric with value 0 for consistent PromQL queries assert_eq!(metrics.len(), 3); - let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA); + let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name(); + let quota_metric = metrics.iter().find(|m| { + m.name == quota_metric_name + && m.value == 0.0 + && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "no-quota-bucket") + }); assert!(quota_metric.is_some()); - assert_eq!(quota_metric.map(|m| m.value), Some(0.0)); } #[test] diff --git a/crates/metrics/src/collectors/bucket_replication.rs b/crates/metrics/src/collectors/bucket_replication.rs index 85c114d57..b0ce8e8a9 100644 --- a/crates/metrics/src/collectors/bucket_replication.rs +++ b/crates/metrics/src/collectors/bucket_replication.rs @@ -13,32 +13,33 @@ // limitations under the License. //! Bucket replication bandwidth metrics collector. +//! +//! Collects bandwidth metrics for bucket replication targets. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication` +//! to avoid duplication of metric names, types, and help text. -use crate::MetricType; use crate::format::PrometheusMetric; +use crate::metrics_type::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD}; use std::borrow::Cow; -/// Bucket replication bandwidth stats for one replication target. +/// Bucket replication bandwidth statistics for metrics collection. #[derive(Debug, Clone, Default)] pub struct BucketReplicationBandwidthStats { + /// Name of the bucket pub bucket: String, + /// Target ARN for replication pub target_arn: String, - pub limit_bytes_per_sec: i64, + /// Configured bandwidth limit in bytes per second + pub limit_bytes_per_sec: u64, + /// Current bandwidth in bytes per second (EWMA) pub current_bandwidth_bytes_per_sec: f64, } -const BUCKET_LABEL: &str = "bucket"; -const TARGET_ARN_LABEL: &str = "targetArn"; - -const METRIC_BANDWIDTH_LIMIT: &str = "rustfs_bucket_replication_bandwidth_limit_bytes_per_second"; -const METRIC_BANDWIDTH_CURRENT: &str = "rustfs_bucket_replication_bandwidth_current_bytes_per_second"; - -const HELP_BANDWIDTH_LIMIT: &str = "Configured bandwidth limit for replication in bytes per second"; -const HELP_BANDWIDTH_CURRENT: &str = "Current replication bandwidth in bytes per second (EWMA)"; - -/// Collect bucket replication bandwidth metrics for Prometheus/OpenTelemetry export. -#[must_use] -#[inline] +/// Collects bucket replication bandwidth metrics from the provided statistics. +/// +/// Uses the metric descriptors from `metrics_type::bucket_replication` module. +/// Returns a vector of Prometheus metrics for replication bandwidth. pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec { if stats.is_empty() { return Vec::new(); @@ -50,25 +51,15 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone()); metrics.push( - PrometheusMetric::new( - METRIC_BANDWIDTH_LIMIT, - MetricType::Gauge, - HELP_BANDWIDTH_LIMIT, - stat.limit_bytes_per_sec as f64, - ) - .with_label(BUCKET_LABEL, bucket_label.clone()) - .with_label(TARGET_ARN_LABEL, target_arn_label.clone()), + PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64) + .with_label("bucket", bucket_label.clone()) + .with_label("targetArn", target_arn_label.clone()), ); metrics.push( - PrometheusMetric::new( - METRIC_BANDWIDTH_CURRENT, - MetricType::Gauge, - HELP_BANDWIDTH_CURRENT, - stat.current_bandwidth_bytes_per_sec, - ) - .with_label(BUCKET_LABEL, bucket_label) - .with_label(TARGET_ARN_LABEL, target_arn_label), + PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec) + .with_label("bucket", bucket_label) + .with_label("targetArn", target_arn_label), ); } @@ -91,23 +82,27 @@ mod tests { let metrics = collect_bucket_replication_bandwidth_metrics(&stats); assert_eq!(metrics.len(), 2); - let limit_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_LIMIT); + let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name(); + let limit_metric = metrics.iter().find(|m| { + m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1") + }); assert!(limit_metric.is_some()); - assert_eq!(limit_metric.map(|m| m.value), Some(1_048_576.0)); assert!( limit_metric .and_then(|m| { m.labels .iter() - .find(|(k, _)| *k == TARGET_ARN_LABEL) + .find(|(k, _)| *k == "targetArn") .map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2") }) .unwrap_or(false) ); - let current_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_CURRENT); + let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name(); + let current_metric = metrics.iter().find(|m| { + m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1") + }); assert!(current_metric.is_some()); - assert_eq!(current_metric.map(|m| m.value), Some(204_800.0)); } #[test] diff --git a/crates/metrics/src/collectors/cluster.rs b/crates/metrics/src/collectors/cluster.rs index 09f2ead73..6f512b28d 100644 --- a/crates/metrics/src/collectors/cluster.rs +++ b/crates/metrics/src/collectors/cluster.rs @@ -16,9 +16,12 @@ //! //! Collects aggregate metrics across the entire RustFS cluster including //! total capacity, usage, and object counts. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::cluster` +//! to avoid duplication of metric names, types, and help text. -use crate::MetricType; use crate::format::PrometheusMetric; +use crate::metrics_type::cluster::*; /// Cluster capacity and usage statistics for metrics collection. /// @@ -41,88 +44,19 @@ pub struct ClusterStats { pub buckets_count: u64, } -// Static metric definitions to avoid allocations -const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes"; -const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes"; -const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes"; -const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes"; -const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total"; -const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total"; - -const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks"; -const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)"; -const HELP_USED: &str = "Total used storage capacity in bytes"; -const HELP_FREE: &str = "Total free storage capacity in bytes"; -const HELP_OBJECTS: &str = "Total number of objects in the cluster"; -const HELP_BUCKETS: &str = "Total number of buckets in the cluster"; - -/// Number of metrics produced by this collector. -const METRIC_COUNT: usize = 6; - -/// Collects cluster-wide metrics from the provided statistics. +/// Collects cluster-wide metrics from the provided cluster statistics. /// -/// # Metrics Produced -/// -/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks -/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead -/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity -/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity -/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster -/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster -/// -/// # Arguments -/// -/// * `stats` - Cluster statistics containing capacity and usage data -/// -/// # Example -/// -/// ``` -/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats}; -/// -/// let stats = ClusterStats { -/// raw_capacity_bytes: 10_000_000_000, -/// usable_capacity_bytes: 8_000_000_000, -/// used_bytes: 2_000_000_000, -/// free_bytes: 6_000_000_000, -/// objects_count: 1000, -/// buckets_count: 10, -/// }; -/// let metrics = collect_cluster_metrics(&stats); -/// assert_eq!(metrics.len(), 6); -/// ``` -#[must_use] -#[inline] +/// Uses the metric descriptors from `metrics_type::cluster` module. +/// Returns a vector of Prometheus metrics for cluster statistics. pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec { - let mut metrics = Vec::with_capacity(METRIC_COUNT); - - metrics.push(PrometheusMetric::new( - METRIC_RAW_CAPACITY, - MetricType::Gauge, - HELP_RAW_CAPACITY, - stats.raw_capacity_bytes as f64, - )); - metrics.push(PrometheusMetric::new( - METRIC_USABLE_CAPACITY, - MetricType::Gauge, - HELP_USABLE_CAPACITY, - stats.usable_capacity_bytes as f64, - )); - metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64)); - metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64)); - metrics.push(PrometheusMetric::new( - METRIC_OBJECTS, - MetricType::Gauge, - HELP_OBJECTS, - stats.objects_count as f64, - )); - metrics.push(PrometheusMetric::new( - METRIC_BUCKETS, - MetricType::Gauge, - HELP_BUCKETS, - stats.buckets_count as f64, - )); - - metrics + vec![ + PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64), + PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64), + PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64), + PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64), + PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64), + PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64), + ] } #[cfg(test)] @@ -147,24 +81,24 @@ mod tests { assert_eq!(metrics.len(), 6); // Verify raw capacity - let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY); + let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name(); + let raw_capacity = metrics.iter().find(|m| m.name == raw_capacity_name && m.value == 3000.0); assert!(raw_capacity.is_some()); - assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0)); // Verify used capacity - let used = metrics.iter().find(|m| m.name == METRIC_USED); + let used_name = CLUSTER_CAPACITY_USED_BYTES_MD.get_full_metric_name(); + let used = metrics.iter().find(|m| m.name == used_name && m.value == 1200.0); assert!(used.is_some()); - assert_eq!(used.map(|m| m.value), Some(1200.0)); // Verify object count - let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS); + let objects_name = CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(); + let objects = metrics.iter().find(|m| m.name == objects_name && m.value == 100.0); assert!(objects.is_some()); - assert_eq!(objects.map(|m| m.value), Some(100.0)); // Verify bucket count - let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS); + let buckets_name = CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(); + let buckets = metrics.iter().find(|m| m.name == buckets_name && m.value == 5.0); assert!(buckets.is_some()); - assert_eq!(buckets.map(|m| m.value), Some(5.0)); } #[test] diff --git a/crates/metrics/src/collectors/cluster_config.rs b/crates/metrics/src/collectors/cluster_config.rs new file mode 100644 index 000000000..cac2723fd --- /dev/null +++ b/crates/metrics/src/collectors/cluster_config.rs @@ -0,0 +1,81 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Cluster config metrics collector. +//! +//! Collects cluster configuration metrics including storage class +//! parity settings. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::cluster_config` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_config::*; + +/// Cluster configuration statistics. +#[derive(Debug, Clone, Default)] +pub struct ClusterConfigStats { + /// Parity for reduced redundancy storage (RRS) class + pub rrs_parity: u32, + /// Parity for standard storage class + pub standard_parity: u32, +} + +/// Collects cluster config metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::cluster_config` module. +/// Returns a vector of Prometheus metrics for cluster configuration. +pub fn collect_cluster_config_metrics(stats: &ClusterConfigStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&CONFIG_RRS_PARITY_MD, stats.rrs_parity as f64), + PrometheusMetric::from_descriptor(&CONFIG_STANDARD_PARITY_MD, stats.standard_parity as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collect_cluster_config_metrics() { + let stats = ClusterConfigStats { + rrs_parity: 2, + standard_parity: 4, + }; + + let metrics = collect_cluster_config_metrics(&stats); + + assert_eq!(metrics.len(), 2); + + let rrs = metrics.iter().find(|m| m.value == 2.0); + assert!(rrs.is_some()); + + let standard = metrics.iter().find(|m| m.value == 4.0); + assert!(standard.is_some()); + } + + #[test] + fn test_collect_cluster_config_metrics_default() { + let stats = ClusterConfigStats::default(); + let metrics = collect_cluster_config_metrics(&stats); + + assert_eq!(metrics.len(), 2); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/cluster_erasure_set.rs b/crates/metrics/src/collectors/cluster_erasure_set.rs new file mode 100644 index 000000000..4b12a461b --- /dev/null +++ b/crates/metrics/src/collectors/cluster_erasure_set.rs @@ -0,0 +1,179 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Cluster erasure set metrics collector. +//! +//! Collects erasure coding set metrics including parity, quorum, +//! drive counts, and health status. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_erasure_set::*; + +/// Erasure set statistics. +#[derive(Debug, Clone, Default)] +pub struct ErasureSetStats { + /// Pool ID + pub pool_id: u32, + /// Set ID within the pool + pub set_id: u32, + /// Total number of drives in the set + pub size: u32, + /// Number of parity drives + pub parity: u32, + /// Number of data shards + pub data_shards: u32, + /// Read quorum + pub read_quorum: u32, + /// Write quorum + pub write_quorum: u32, + /// Number of online drives + pub online_drives_count: u32, + /// Number of healing drives + pub healing_drives_count: u32, + /// Health status (1=healthy, 0=unhealthy) + pub health: u8, + /// Read tolerance (number of drive failures tolerated for reads) + pub read_tolerance: u32, + /// Write tolerance (number of drive failures tolerated for writes) + pub write_tolerance: u32, + /// Read health status (1=healthy, 0=unhealthy) + pub read_health: u8, + /// Write health status (1=healthy, 0=unhealthy) + pub write_health: u8, +} + +/// Collects erasure set metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for erasure sets. +pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec { + let mut metrics = Vec::with_capacity(stats.len() * 12); + + for stat in stats { + let pool_id_label = stat.pool_id.to_string(); + let set_id_label = stat.set_id.to_string(); + + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_SIZE_MD, stat.size as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_PARITY_MD, stat.parity as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_DATA_SHARDS_MD, stat.data_shards as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_READ_QUORUM_MD, stat.read_quorum as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_QUORUM_MD, stat.write_quorum as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_ONLINE_DRIVES_COUNT_MD, stat.online_drives_count as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_HEALING_DRIVES_COUNT_MD, stat.healing_drives_count as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_HEALTH_MD, stat.health as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_READ_TOLERANCE_MD, stat.read_tolerance as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_TOLERANCE_MD, stat.write_tolerance as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_READ_HEALTH_MD, stat.read_health as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64) + .with_label_owned(POOL_ID_L, pool_id_label.clone()) + .with_label_owned(SET_ID_L, set_id_label.clone()), + ); + } + + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_erasure_set_metrics() { + let stats = vec![ErasureSetStats { + pool_id: 1, + set_id: 0, + size: 16, + parity: 4, + data_shards: 12, + read_quorum: 13, + write_quorum: 13, + online_drives_count: 16, + healing_drives_count: 0, + health: 1, + read_tolerance: 4, + write_tolerance: 4, + read_health: 1, + write_health: 1, + }]; + + let metrics = collect_erasure_set_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 12); + + let size_name = ERASURE_SET_SIZE_MD.get_full_metric_name(); + let size = metrics.iter().find(|m| m.name == size_name); + assert!(size.is_some()); + assert_eq!(size.map(|m| m.value), Some(16.0)); + + let health_name = ERASURE_SET_HEALTH_MD.get_full_metric_name(); + let health = metrics.iter().find(|m| m.name == health_name); + assert!(health.is_some()); + assert_eq!(health.map(|m| m.value), Some(1.0)); + } + + #[test] + fn test_collect_erasure_set_metrics_empty() { + let stats: Vec = vec![]; + let metrics = collect_erasure_set_metrics(&stats); + assert!(metrics.is_empty()); + } +} diff --git a/crates/metrics/src/collectors/cluster_health.rs b/crates/metrics/src/collectors/cluster_health.rs new file mode 100644 index 000000000..268fadf76 --- /dev/null +++ b/crates/metrics/src/collectors/cluster_health.rs @@ -0,0 +1,85 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Cluster health metrics collector. +//! +//! Collects cluster-wide health metrics including drive counts +//! (offline, online, total). +//! +//! This collector reuses the metric descriptors defined in `metrics_type::cluster_health` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_health::*; + +/// Cluster health statistics. +#[derive(Debug, Clone, Default)] +pub struct ClusterHealthStats { + /// Number of offline drives in the cluster + pub drives_offline_count: u64, + /// Number of online drives in the cluster + pub drives_online_count: u64, + /// Total number of drives in the cluster + pub drives_count: u64, +} + +/// Collects cluster health metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::cluster_health` module. +/// Returns a vector of Prometheus metrics for cluster health. +pub fn collect_cluster_health_metrics(stats: &ClusterHealthStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&HEALTH_DRIVES_OFFLINE_COUNT_MD, stats.drives_offline_count as f64), + PrometheusMetric::from_descriptor(&HEALTH_DRIVES_ONLINE_COUNT_MD, stats.drives_online_count as f64), + PrometheusMetric::from_descriptor(&HEALTH_DRIVES_COUNT_MD, stats.drives_count as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collect_cluster_health_metrics() { + let stats = ClusterHealthStats { + drives_offline_count: 2, + drives_online_count: 18, + drives_count: 20, + }; + + let metrics = collect_cluster_health_metrics(&stats); + + assert_eq!(metrics.len(), 3); + + let offline = metrics.iter().find(|m| m.value == 2.0); + assert!(offline.is_some()); + + let online = metrics.iter().find(|m| m.value == 18.0); + assert!(online.is_some()); + } + + #[test] + fn test_collect_cluster_health_metrics_default() { + let stats = ClusterHealthStats::default(); + let metrics = collect_cluster_health_metrics(&stats); + + assert_eq!(metrics.len(), 3); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/cluster_iam.rs b/crates/metrics/src/collectors/cluster_iam.rs new file mode 100644 index 000000000..07b141249 --- /dev/null +++ b/crates/metrics/src/collectors/cluster_iam.rs @@ -0,0 +1,117 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Cluster IAM metrics collector. +//! +//! Collects IAM (Identity and Access Management) metrics including +//! plugin authentication service stats and sync statistics. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_iam::*; + +/// IAM statistics. +#[derive(Debug, Clone, Default)] +pub struct IamStats { + /// Time in seconds since last failed authn service request + pub plugin_authn_service_last_fail_seconds: u64, + /// Time in seconds since last successful authn service request + pub plugin_authn_service_last_succ_seconds: u64, + /// Average RTT of successful requests in the last minute (ms) + pub plugin_authn_service_succ_avg_rtt_ms_minute: u64, + /// Maximum RTT of successful requests in the last minute (ms) + pub plugin_authn_service_succ_max_rtt_ms_minute: u64, + /// Total requests count in the last full minute + pub plugin_authn_service_total_requests_minute: u64, + /// Time in milliseconds since last successful IAM data sync + pub since_last_sync_millis: u64, + /// Number of failed IAM data syncs since server start + pub sync_failures: u64, + /// Number of successful IAM data syncs since server start + pub sync_successes: u64, +} + +/// Collects IAM metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for IAM statistics. +pub fn collect_iam_metrics(stats: &IamStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor( + &PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD, + stats.plugin_authn_service_last_fail_seconds as f64, + ), + PrometheusMetric::from_descriptor( + &PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD, + stats.plugin_authn_service_last_succ_seconds as f64, + ), + PrometheusMetric::from_descriptor( + &PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD, + stats.plugin_authn_service_succ_avg_rtt_ms_minute as f64, + ), + PrometheusMetric::from_descriptor( + &PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD, + stats.plugin_authn_service_succ_max_rtt_ms_minute as f64, + ), + PrometheusMetric::from_descriptor( + &PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD, + stats.plugin_authn_service_total_requests_minute as f64, + ), + PrometheusMetric::from_descriptor(&SINCE_LAST_SYNC_MILLIS_MD, stats.since_last_sync_millis as f64), + PrometheusMetric::from_descriptor(&SYNC_FAILURES_MD, stats.sync_failures as f64), + PrometheusMetric::from_descriptor(&SYNC_SUCCESSES_MD, stats.sync_successes as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_iam_metrics() { + let stats = IamStats { + plugin_authn_service_last_fail_seconds: 3600, + plugin_authn_service_last_succ_seconds: 10, + plugin_authn_service_succ_avg_rtt_ms_minute: 50, + plugin_authn_service_succ_max_rtt_ms_minute: 200, + plugin_authn_service_total_requests_minute: 1000, + since_last_sync_millis: 5000, + sync_failures: 5, + sync_successes: 1000, + }; + + let metrics = collect_iam_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 8); + + let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name(); + let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name); + assert!(sync_successes.is_some()); + assert_eq!(sync_successes.map(|m| m.value), Some(1000.0)); + } + + #[test] + fn test_collect_iam_metrics_default() { + let stats = IamStats::default(); + let metrics = collect_iam_metrics(&stats); + + assert_eq!(metrics.len(), 8); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/cluster_usage.rs b/crates/metrics/src/collectors/cluster_usage.rs new file mode 100644 index 000000000..4c38f2a6b --- /dev/null +++ b/crates/metrics/src/collectors/cluster_usage.rs @@ -0,0 +1,199 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Cluster usage metrics collector. +//! +//! Collects cluster-wide and per-bucket usage metrics including +//! object counts, sizes, versions, and distributions. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_usage::*; + +/// Cluster-wide usage statistics. +#[derive(Debug, Clone, Default)] +pub struct ClusterUsageStats { + /// Total bytes used in the cluster + pub total_bytes: u64, + /// Total number of objects + pub objects_count: u64, + /// Total number of object versions (including delete markers) + pub versions_count: u64, + /// Total number of delete markers + pub delete_markers_count: u64, + /// Object size distribution by range + pub object_size_distribution: Vec<(String, u64)>, + /// Version count distribution by range + pub versions_distribution: Vec<(String, u64)>, +} + +/// Per-bucket usage statistics. +#[derive(Debug, Clone, Default)] +pub struct BucketUsageStats { + /// Bucket name + pub bucket: String, + /// Total bytes used in the bucket + pub total_bytes: u64, + /// Total number of objects in the bucket + pub objects_count: u64, + /// Total number of object versions (including delete markers) + pub versions_count: u64, + /// Total number of delete markers + pub delete_markers_count: u64, + /// Bucket quota in bytes (0 if no quota) + pub quota_bytes: u64, + /// Object size distribution by range + pub object_size_distribution: Vec<(String, u64)>, + /// Version count distribution by range + pub version_count_distribution: Vec<(String, u64)>, +} + +/// Collects cluster-wide usage metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for cluster usage. +pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec { + let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len()); + + metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64)); + metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64)); + metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64)); + metrics.push(PrometheusMetric::from_descriptor( + &USAGE_DELETE_MARKERS_COUNT_MD, + stats.delete_markers_count as f64, + )); + + // Object size distribution + for (range, count) in &stats.object_size_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_OBJECTS_DISTRIBUTION_MD, *count as f64) + .with_label_owned(RANGE_LABEL, range.clone()), + ); + } + + // Version distribution + for (range, count) in &stats.versions_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_VERSIONS_DISTRIBUTION_MD, *count as f64) + .with_label_owned(RANGE_LABEL, range.clone()), + ); + } + + metrics +} + +/// Collects per-bucket usage metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for bucket usage. +pub fn collect_bucket_usage_metrics(stats: &[BucketUsageStats]) -> Vec { + let mut metrics = Vec::new(); + + for stat in stats { + let bucket_label = stat.bucket.clone(); + + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, stat.total_bytes as f64) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, stat.objects_count as f64) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, stat.versions_count as f64) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, stat.delete_markers_count as f64) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, stat.quota_bytes as f64) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + + // Object size distribution per bucket + for (range, count) in &stat.object_size_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, *count as f64) + .with_label_owned(RANGE_LABEL, range.clone()) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + } + + // Version count distribution per bucket + for (range, count) in &stat.version_count_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, *count as f64) + .with_label_owned(RANGE_LABEL, range.clone()) + .with_label_owned(BUCKET_LABEL, bucket_label.clone()), + ); + } + } + + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_cluster_usage_metrics() { + let stats = ClusterUsageStats { + total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB + objects_count: 10000, + versions_count: 15000, + delete_markers_count: 500, + object_size_distribution: vec![ + ("0-1KB".to_string(), 5000), + ("1KB-1MB".to_string(), 3000), + ("1MB-100MB".to_string(), 1500), + ("100MB+".to_string(), 500), + ], + versions_distribution: vec![("1".to_string(), 8000), ("2-5".to_string(), 1500), ("6+".to_string(), 500)], + }; + + let metrics = collect_cluster_usage_metrics(&stats); + report_metrics(&metrics); + + // 4 base metrics + 4 size distribution + 3 version distribution = 11 + assert_eq!(metrics.len(), 11); + + let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name(); + let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name); + assert!(total_bytes.is_some()); + } + + #[test] + fn test_collect_bucket_usage_metrics() { + let stats = vec![BucketUsageStats { + bucket: "test-bucket".to_string(), + total_bytes: 1024 * 1024 * 1024 * 10, // 10 GB + objects_count: 1000, + versions_count: 1200, + delete_markers_count: 50, + quota_bytes: 1024 * 1024 * 1024 * 100, // 100 GB quota + object_size_distribution: vec![("0-1KB".to_string(), 500)], + version_count_distribution: vec![("1".to_string(), 800)], + }]; + + let metrics = collect_bucket_usage_metrics(&stats); + report_metrics(&metrics); + + // 5 base metrics + 1 size distribution + 1 version distribution = 7 + assert_eq!(metrics.len(), 7); + } +} diff --git a/crates/metrics/src/collectors/global.rs b/crates/metrics/src/collectors/global.rs index f856d433a..444b19835 100644 --- a/crates/metrics/src/collectors/global.rs +++ b/crates/metrics/src/collectors/global.rs @@ -12,9 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Global metrics collector initialization. +//! +//! This module provides the entry point for initializing all metrics collectors. +//! The actual statistics collection functions are in `stats_collector.rs`. + +use crate::collectors::stats_collector::{ + collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats, + collect_process_stats, +}; use crate::collectors::{ - BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics, - collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, collect_resource_metrics, + collect_bucket_metrics, collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, + collect_resource_metrics, }; use crate::constants::{ DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL, @@ -23,245 +32,85 @@ use crate::constants::{ ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL, }; use crate::format::report_metrics; -use rustfs_ecstore::bucket::metadata_sys::get_quota_config; -use rustfs_ecstore::data_usage::load_data_usage_from_backend; -use rustfs_ecstore::global::get_global_bucket_monitor; -use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; -use rustfs_ecstore::store_api::{BucketOperations, BucketOptions}; -use rustfs_ecstore::{StorageAPI, new_object_layer_fn}; use rustfs_utils::get_env_opt_u64; -use std::sync::OnceLock; -use std::time::{Duration, Instant}; -use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System}; +use std::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{instrument, warn}; +use tracing::warn; -/// Process start time for calculating uptime. -static PROCESS_START: OnceLock = OnceLock::new(); - -/// Get the process start time, initializing it on first call. -#[inline] -fn get_process_start() -> &'static Instant { - PROCESS_START.get_or_init(Instant::now) -} - -/// Collect cluster statistics from the storage layer. -#[instrument] -async fn collect_cluster_stats() -> ClusterStats { - let Some(store) = new_object_layer_fn() else { - return ClusterStats::default(); - }; - - let storage_info = store.storage_info().await; - - let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum(); - let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum(); - let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64; - let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64; - - // Get bucket and object counts from data usage info - let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await { - Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count), - Err(e) => { - warn!("Failed to load data usage from backend: {}", e); - // Fall back to bucket list for buckets_count, objects_count stays 0 - let buckets = store - .list_bucket(&BucketOptions { - cached: true, - ..Default::default() - }) - .await - .unwrap_or_else(|e| { - warn!("Failed to list buckets for cluster metrics: {}", e); - Vec::new() - }); - (buckets.len() as u64, 0) - } - }; - - ClusterStats { - raw_capacity_bytes: raw_capacity, - usable_capacity_bytes: usable_capacity, - used_bytes: used, - free_bytes: free, - objects_count, - buckets_count, - } -} - -/// Collect bucket statistics from the storage layer. -async fn collect_bucket_stats() -> Vec { - let Some(store) = new_object_layer_fn() else { - return Vec::new(); - }; - - // Load data usage info from backend to get bucket sizes and object counts - let data_usage = match load_data_usage_from_backend(store.clone()).await { - Ok(info) => Some(info), - Err(e) => { - warn!("Failed to load data usage from backend for bucket metrics: {}", e); - None - } - }; - - let buckets = match store - .list_bucket(&BucketOptions { - cached: true, - ..Default::default() - }) - .await - { - Ok(b) => b, - Err(e) => { - warn!("Failed to list buckets for metrics: {}", e); - return Vec::new(); - } - }; - - // Build bucket stats with real data from DataUsageInfo - let mut stats = Vec::with_capacity(buckets.len()); - for bucket in buckets { - if bucket.name.starts_with('.') { - continue; - } - - // Get size and objects_count from data usage info - let (size_bytes, objects_count) = data_usage - .as_ref() - .and_then(|du| du.buckets_usage.get(&bucket.name)) - .map(|bui| (bui.size, bui.objects_count)) - .unwrap_or((0, 0)); - - // Get quota from bucket metadata - let quota_bytes = match get_quota_config(&bucket.name).await { - Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0), - Err(_) => 0, // No quota configured or error - }; - - stats.push(BucketStats { - name: bucket.name, - size_bytes, - objects_count, - quota_bytes, - }); - } - - stats -} - -/// Collect bucket replication bandwidth stats from the global monitor. -fn collect_bucket_replication_bandwidth_stats() -> Vec { - let Some(monitor) = get_global_bucket_monitor() else { - return Vec::new(); - }; - - monitor - .get_report(|_| true) - .bucket_stats - .into_iter() - .map(|(opts, details)| BucketReplicationBandwidthStats { - bucket: opts.name, - target_arn: opts.replication_arn, - limit_bytes_per_sec: details.limit_bytes_per_sec, - current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec, - }) - .collect() -} - -/// Collect disk statistics from the storage layer. -async fn collect_disk_stats() -> Vec { - let Some(store) = new_object_layer_fn() else { - return Vec::new(); - }; - - let storage_info = store.storage_info().await; - - storage_info - .disks - .iter() - .map(|disk| DiskStats { - server: disk.endpoint.clone(), - drive: disk.drive_path.clone(), - total_bytes: disk.total_space, - used_bytes: disk.used_space, - free_bytes: disk.available_space, - }) - .collect() -} - -/// Collect resource statistics for the current process. -/// -/// Collects: -/// - Uptime: Calculated from process start time -/// - Memory: Process resident set size from sysinfo -/// - CPU: Process CPU usage percentage from sysinfo -#[inline] -fn collect_process_stats() -> ResourceStats { - let uptime_seconds = get_process_start().elapsed().as_secs(); - - // Use sysinfo for process metrics - let mut sys = System::new(); - let pid = Pid::from_u32(std::process::id()); - sys.refresh_processes_specifics( - ProcessesToUpdate::Some(&[pid]), - true, - ProcessRefreshKind::nothing().with_cpu().with_memory(), - ); - - if let Some(process) = sys.process(pid) { - ResourceStats { - cpu_percent: process.cpu_usage() as f64, - memory_bytes: process.memory(), - uptime_seconds, - } - } else { - // Fallback if process not found - ResourceStats { - cpu_percent: 0.0, - memory_bytes: 0, - uptime_seconds, - } - } -} - -/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics. +/// Initialize all metrics collectors. /// /// This function spawns background tasks that periodically collect metrics -/// and report them using the `metrics` crate. +/// from various sources and report them to the metrics system. /// /// # Arguments +/// * `token` - A `CancellationToken` that can be used to gracefully shut down +/// all metrics collection tasks. /// -/// * `token` - A cancellation token to gracefully stop the metrics collection tasks. +/// # Environment Variables +/// The collection intervals can be configured via environment variables: +/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60) +/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300) +/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60) +/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30) +/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15) +/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds. +/// +/// Legacy interval names without `_SEC` are still accepted for backward compatibility: +/// - `RUSTFS_METRICS_CLUSTER_INTERVAL` +/// - `RUSTFS_METRICS_BUCKET_INTERVAL` +/// - `RUSTFS_METRICS_NODE_INTERVAL` +/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL` +/// - `RUSTFS_METRICS_RESOURCE_INTERVAL` pub fn init_metrics_collectors(token: CancellationToken) { - // Initialize process start time - get_process_start(); + const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL"; + const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL"; + const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL"; + const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL"; + const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL"; + const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL"; - // Helper closure to determine interval for a specific metric type - let get_interval = |env_key: &str, type_default: Duration| -> Duration { - // 1. Try specific env var - // 2. Fallback to global default env var (if set differently from hardcoded default) - // 3. Fallback to type specific default + fn parse_interval(msc: &str, legacy_msc: &str) -> Option { + get_env_opt_u64(msc) + .or_else(|| get_env_opt_u64(legacy_msc)) + .filter(|&v| v > 0) + } - // Helper to check if value is valid (non-zero) - let is_valid = |v: u64| v > 0; + // Read intervals from environment or use defaults, ensuring zero is ignored. + let cluster_interval = parse_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL) + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_CLUSTER_METRICS_INTERVAL); - if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) { - Duration::from_secs(val) - } else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) { - Duration::from_secs(val) - } else { - type_default - } - }; + let bucket_interval = parse_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL) + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_BUCKET_METRICS_INTERVAL); - let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL); - let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL); - let bucket_replication_bandwidth_interval = get_interval( - ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, - DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, - ); - let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL); - let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL); + let bucket_replication_bandwidth_interval = + parse_interval(ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, LEGACY_REPLICATION_BANDWIDTH_INTERVAL) + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL); + + let node_interval = parse_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL) + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_NODE_METRICS_INTERVAL); + + let resource_interval = parse_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL) + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_RESOURCE_METRICS_INTERVAL); // Spawn task for cluster metrics let token_clone = token.clone(); diff --git a/crates/metrics/src/collectors/ilm.rs b/crates/metrics/src/collectors/ilm.rs new file mode 100644 index 000000000..2a18e7c73 --- /dev/null +++ b/crates/metrics/src/collectors/ilm.rs @@ -0,0 +1,96 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! ILM (Information Lifecycle Management) metrics collector. +//! +//! Collects ILM metrics including pending tasks, active tasks, +//! and scanned versions. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::ilm` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::ilm::*; + +/// ILM statistics for metrics collection. +#[derive(Debug, Clone, Default)] +pub struct IlmStats { + /// Number of pending ILM expiry tasks + pub expiry_pending_tasks: u64, + /// Number of active ILM transition tasks + pub transition_active_tasks: u64, + /// Number of pending ILM transition tasks + pub transition_pending_tasks: u64, + /// Number of missed immediate ILM transition tasks + pub transition_missed_immediate_tasks: u64, + /// Total number of object versions scanned for ILM + pub versions_scanned: u64, +} + +/// Collects ILM metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::ilm` module. +/// Returns a vector of Prometheus metrics for ILM statistics. +pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&ILM_EXPIRY_PENDING_TASKS_MD, stats.expiry_pending_tasks as f64), + PrometheusMetric::from_descriptor(&ILM_TRANSITION_ACTIVE_TASKS_MD, stats.transition_active_tasks as f64), + PrometheusMetric::from_descriptor(&ILM_TRANSITION_PENDING_TASKS_MD, stats.transition_pending_tasks as f64), + PrometheusMetric::from_descriptor( + &ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD, + stats.transition_missed_immediate_tasks as f64, + ), + PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collect_ilm_metrics() { + let stats = IlmStats { + expiry_pending_tasks: 100, + transition_active_tasks: 5, + transition_pending_tasks: 50, + transition_missed_immediate_tasks: 10, + versions_scanned: 1000000, + }; + + let metrics = collect_ilm_metrics(&stats); + + assert_eq!(metrics.len(), 5); + + let pending = metrics.iter().find(|m| m.value == 100.0); + assert!(pending.is_some()); + + let scanned = metrics.iter().find(|m| m.value == 1000000.0); + assert!(scanned.is_some()); + } + + #[test] + fn test_collect_ilm_metrics_default() { + let stats = IlmStats::default(); + let metrics = collect_ilm_metrics(&stats); + + assert_eq!(metrics.len(), 5); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/logger_webhook.rs b/crates/metrics/src/collectors/logger_webhook.rs new file mode 100644 index 000000000..48c6b37a8 --- /dev/null +++ b/crates/metrics/src/collectors/logger_webhook.rs @@ -0,0 +1,122 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Logger webhook metrics collector. +//! +//! Collects webhook metrics including failed messages, queue length, +//! and total messages per webhook target. + +use crate::format::PrometheusMetric; +use crate::metrics_type::logger_webhook::{ + ENDPOINT_LABEL, NAME_LABEL, WEBHOOK_FAILED_MESSAGES_MD, WEBHOOK_QUEUE_LENGTH_MD, WEBHOOK_TOTAL_MESSAGES_MD, +}; +use std::borrow::Cow; + +/// Webhook target statistics. +#[derive(Debug, Clone, Default)] +pub struct WebhookTargetStats { + /// Webhook name + pub name: String, + /// Webhook endpoint URL + pub endpoint: String, + /// Number of failed messages + pub failed_messages: u64, + /// Number of messages in queue + pub queue_length: u64, + /// Total number of messages sent + pub total_messages: u64, +} + +/// Collects webhook metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::logger_webhook` module. +/// Returns a vector of Prometheus metrics for webhook statistics. +pub fn collect_webhook_metrics(stats: &[WebhookTargetStats]) -> Vec { + let mut metrics = Vec::with_capacity(stats.len() * 3); + + for stat in stats { + let name_label: Cow<'static, str> = Cow::Owned(stat.name.clone()); + let endpoint_label: Cow<'static, str> = Cow::Owned(stat.endpoint.clone()); + + metrics.push( + PrometheusMetric::from_descriptor(&WEBHOOK_FAILED_MESSAGES_MD, stat.failed_messages as f64) + .with_label(NAME_LABEL, name_label.clone()) + .with_label(ENDPOINT_LABEL, endpoint_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&WEBHOOK_QUEUE_LENGTH_MD, stat.queue_length as f64) + .with_label(NAME_LABEL, name_label.clone()) + .with_label(ENDPOINT_LABEL, endpoint_label.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&WEBHOOK_TOTAL_MESSAGES_MD, stat.total_messages as f64) + .with_label(NAME_LABEL, name_label.clone()) + .with_label(ENDPOINT_LABEL, endpoint_label.clone()), + ); + } + + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_webhook_metrics() { + let stats = vec![WebhookTargetStats { + name: "alert-webhook".to_string(), + endpoint: "https://hooks.example.com/alert".to_string(), + failed_messages: 3, + queue_length: 15, + total_messages: 5000, + }]; + + let metrics = collect_webhook_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 3); + + // Verify we have metrics with the expected values + let failed = metrics.iter().find(|m| m.value == 3.0); + assert!(failed.is_some()); + + // Verify labels + let total = metrics.iter().find(|m| m.value == 5000.0); + assert!(total.is_some()); + let total_metric = total.unwrap(); + assert!( + total_metric + .labels + .iter() + .any(|(k, v)| *k == NAME_LABEL && v == "alert-webhook") + ); + assert!( + total_metric + .labels + .iter() + .any(|(k, v)| *k == ENDPOINT_LABEL && v == "https://hooks.example.com/alert") + ); + } + + #[test] + fn test_collect_webhook_metrics_empty() { + let stats: Vec = vec![]; + let metrics = collect_webhook_metrics(&stats); + assert!(metrics.is_empty()); + } +} diff --git a/crates/metrics/src/collectors/mod.rs b/crates/metrics/src/collectors/mod.rs index 6131dc4bd..bc64c9728 100644 --- a/crates/metrics/src/collectors/mod.rs +++ b/crates/metrics/src/collectors/mod.rs @@ -61,16 +61,51 @@ //! report_metrics(&metrics); //! ``` +mod audit; mod bucket; mod bucket_replication; mod cluster; +mod cluster_config; +mod cluster_erasure_set; +mod cluster_health; +mod cluster_iam; +mod cluster_usage; pub(crate) mod global; +mod ilm; +mod logger_webhook; mod node; +mod notification; +mod replication; +mod request; mod resource; +mod scanner; +mod stats_collector; +mod system_cpu; +mod system_drive; +mod system_memory; +mod system_network; +mod system_process; +pub use audit::{AuditTargetStats, collect_audit_metrics}; pub use bucket::{BucketStats, collect_bucket_metrics}; pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics}; pub use cluster::{ClusterStats, collect_cluster_metrics}; +pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics}; +pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics}; +pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics}; +pub use cluster_iam::{IamStats, collect_iam_metrics}; +pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics}; pub use global::init_metrics_collectors; +pub use ilm::{IlmStats, collect_ilm_metrics}; +pub use logger_webhook::{WebhookTargetStats, collect_webhook_metrics}; pub use node::{DiskStats, collect_node_metrics}; +pub use notification::{NotificationStats, collect_notification_metrics}; +pub use replication::{ReplicationStats, collect_replication_metrics}; +pub use request::{ApiRequestStats, collect_request_metrics}; pub use resource::{ResourceStats, collect_resource_metrics}; +pub use scanner::{ScannerStats, collect_scanner_metrics}; +pub use system_cpu::{CpuStats, collect_cpu_metrics}; +pub use system_drive::{DriveCountStats, DriveDetailedStats, collect_drive_count_metrics, collect_drive_detailed_metrics}; +pub use system_memory::{MemoryStats, collect_memory_metrics}; +pub use system_network::{NetworkStats, collect_network_metrics}; +pub use system_process::{ProcessStats, collect_process_metrics}; diff --git a/crates/metrics/src/collectors/node.rs b/crates/metrics/src/collectors/node.rs index ba60d19e2..f2b5c294b 100644 --- a/crates/metrics/src/collectors/node.rs +++ b/crates/metrics/src/collectors/node.rs @@ -16,93 +16,61 @@ //! //! Collects storage metrics for each disk/drive in the cluster, //! including capacity, usage, and health status. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::node_disk` +//! to avoid duplication of metric names, types, and help text. -use crate::MetricType; use crate::format::PrometheusMetric; +use crate::metrics_type::node_disk::*; use std::borrow::Cow; -/// Statistics for a single disk/drive. +/// Disk statistics for metrics collection. +/// +/// This struct provides a decoupled interface for collecting disk metrics +/// without depending on specific internal types. HTTP handlers should populate +/// this struct from their available data sources. #[derive(Debug, Clone, Default)] pub struct DiskStats { /// Server endpoint (e.g., "node1:9000") pub server: String, /// Drive path (e.g., "/data/disk1") pub drive: String, - /// Total capacity in bytes + /// Total disk capacity in bytes pub total_bytes: u64, - /// Used space in bytes + /// Used disk space in bytes pub used_bytes: u64, - /// Free space in bytes + /// Free disk space in bytes pub free_bytes: u64, } -// Static metric definitions -const METRIC_TOTAL: &str = "rustfs_node_disk_total_bytes"; -const METRIC_USED: &str = "rustfs_node_disk_used_bytes"; -const METRIC_FREE: &str = "rustfs_node_disk_free_bytes"; - -const HELP_TOTAL: &str = "Total disk capacity in bytes"; -const HELP_USED: &str = "Used disk space in bytes"; -const HELP_FREE: &str = "Free disk space in bytes"; - -/// Collects per-node disk metrics from the provided disk statistics. +/// Collects per-disk metrics from the provided disk statistics. /// -/// # Metrics Produced -/// -/// For each disk, the following metrics are produced with `server` and `drive` labels: -/// -/// - `rustfs_node_disk_total_bytes`: Total capacity of the disk -/// - `rustfs_node_disk_used_bytes`: Used space on the disk -/// - `rustfs_node_disk_free_bytes`: Free space on the disk -/// -/// # Arguments -/// -/// * `disks` - Slice of disk statistics -/// -/// # Example -/// -/// ``` -/// use rustfs_metrics::collectors::{collect_node_metrics, DiskStats}; -/// -/// let disks = vec![ -/// DiskStats { -/// server: "node1:9000".to_string(), -/// drive: "/data/disk1".to_string(), -/// total_bytes: 1_000_000_000, -/// used_bytes: 400_000_000, -/// free_bytes: 600_000_000, -/// }, -/// ]; -/// let metrics = collect_node_metrics(&disks); -/// assert_eq!(metrics.len(), 3); -/// ``` -#[must_use] -#[inline] +/// Uses the metric descriptors from `metrics_type::node_disk` module. +/// Returns a vector of Prometheus metrics for all disks. pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec { if disks.is_empty() { return Vec::new(); } let mut metrics = Vec::with_capacity(disks.len() * 3); - for disk in disks { let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone()); let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone()); metrics.push( - PrometheusMetric::new(METRIC_TOTAL, MetricType::Gauge, HELP_TOTAL, disk.total_bytes as f64) + PrometheusMetric::from_descriptor(&NODE_DISK_TOTAL_BYTES_MD, disk.total_bytes as f64) .with_label("server", server_label.clone()) .with_label("drive", drive_label.clone()), ); metrics.push( - PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, disk.used_bytes as f64) + PrometheusMetric::from_descriptor(&NODE_DISK_USED_BYTES_MD, disk.used_bytes as f64) .with_label("server", server_label.clone()) .with_label("drive", drive_label.clone()), ); metrics.push( - PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, disk.free_bytes as f64) + PrometheusMetric::from_descriptor(&NODE_DISK_FREE_BYTES_MD, disk.free_bytes as f64) .with_label("server", server_label) .with_label("drive", drive_label), ); @@ -140,22 +108,24 @@ mod tests { assert_eq!(metrics.len(), 6); // Verify node1 disk1 total bytes + let node1_total_name = NODE_DISK_TOTAL_BYTES_MD.get_full_metric_name(); let node1_total = metrics.iter().find(|m| { - m.name == METRIC_TOTAL + m.name == node1_total_name + && m.value == 1000000.0 && m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000") && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1") }); assert!(node1_total.is_some()); - assert_eq!(node1_total.map(|m| m.value), Some(1000000.0)); // Verify node2 disk2 used bytes + let node2_used_name = NODE_DISK_USED_BYTES_MD.get_full_metric_name(); let node2_used = metrics.iter().find(|m| { - m.name == METRIC_USED + m.name == node2_used_name + && m.value == 800000.0 && m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000") && m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2") }); assert!(node2_used.is_some()); - assert_eq!(node2_used.map(|m| m.value), Some(800000.0)); } #[test] diff --git a/crates/metrics/src/collectors/notification.rs b/crates/metrics/src/collectors/notification.rs new file mode 100644 index 000000000..7e6a81d7d --- /dev/null +++ b/crates/metrics/src/collectors/notification.rs @@ -0,0 +1,91 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Notification metrics collector. +//! +//! Collects notification system metrics including events sent, +//! errors, and skipped events. + +use crate::format::PrometheusMetric; +use crate::metrics_type::cluster_notification::{ + NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD, + NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, +}; + +/// Notification statistics. +#[derive(Debug, Clone, Default)] +pub struct NotificationStats { + /// Number of concurrent send operations in progress + pub current_send_in_progress: u64, + /// Total number of events that encountered errors + pub events_errors_total: u64, + /// Total number of events successfully sent + pub events_sent_total: u64, + /// Total number of events skipped + pub events_skipped_total: u64, +} + +/// Collects notification metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::cluster_notification` module. +/// Returns a vector of Prometheus metrics for notification statistics. +pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, stats.current_send_in_progress as f64), + PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, stats.events_errors_total as f64), + PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_MD, stats.events_sent_total as f64), + PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, stats.events_skipped_total as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_notification_metrics() { + let stats = NotificationStats { + current_send_in_progress: 5, + events_errors_total: 10, + events_sent_total: 10000, + events_skipped_total: 50, + }; + + let metrics = collect_notification_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 4); + + let sent = metrics.iter().find(|m| m.value == 10000.0); + assert!(sent.is_some()); + + let errors = metrics.iter().find(|m| m.value == 10.0); + assert!(errors.is_some()); + } + + #[test] + fn test_collect_notification_metrics_default() { + let stats = NotificationStats::default(); + let metrics = collect_notification_metrics(&stats); + + assert_eq!(metrics.len(), 4); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/replication.rs b/crates/metrics/src/collectors/replication.rs new file mode 100644 index 000000000..387025c83 --- /dev/null +++ b/crates/metrics/src/collectors/replication.rs @@ -0,0 +1,107 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Replication metrics collector. +//! +//! Collects cluster-wide replication metrics including queue stats, +//! data transfer rates, and worker information. + +use crate::format::PrometheusMetric; +use crate::metrics_type::replication::*; + +/// Replication statistics. +#[derive(Debug, Clone, Default)] +pub struct ReplicationStats { + /// Number of active replication workers + pub active_workers: u64, + /// Current data transfer rate in bytes/sec + pub current_data_transfer_rate: f64, + /// Bytes queued in the last full minute + pub last_minute_queued_bytes: u64, + /// Objects queued in the last full minute + pub last_minute_queued_count: u64, + /// Maximum active workers seen since server start + pub max_active_workers: u64, + /// Maximum bytes queued since server start + pub max_queued_bytes: u64, + /// Maximum objects queued since server start + pub max_queued_count: u64, + /// Maximum data transfer rate seen since server start + pub max_data_transfer_rate: f64, + /// Objects in replication backlog in the last 5 minutes + pub recent_backlog_count: u64, +} + +/// Collects replication metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for replication statistics. +pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64), + PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate), + PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64), + PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD, stats.last_minute_queued_count as f64), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_MD, stats.max_active_workers as f64), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_MD, stats.max_queued_bytes as f64), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_MD, stats.max_queued_count as f64), + PrometheusMetric::from_descriptor(&REPLICATION_MAX_DATA_TRANSFER_RATE_MD, stats.max_data_transfer_rate), + PrometheusMetric::from_descriptor(&REPLICATION_RECENT_BACKLOG_COUNT_MD, stats.recent_backlog_count as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_replication_metrics() { + let stats = ReplicationStats { + active_workers: 10, + current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s + last_minute_queued_bytes: 1024 * 1024 * 100, // 100 MB + last_minute_queued_count: 500, + max_active_workers: 20, + max_queued_bytes: 1024 * 1024 * 500, // 500 MB + max_queued_count: 2000, + max_data_transfer_rate: 1024.0 * 1024.0 * 10.0, // 10 MB/s + recent_backlog_count: 1500, + }; + + let metrics = collect_replication_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 9); + + // Verify active workers + let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name(); + let active = metrics.iter().find(|m| m.name == active_name); + assert!(active.is_some()); + assert_eq!(active.map(|m| m.value), Some(10.0)); + } + + #[test] + fn test_collect_replication_metrics_default() { + let stats = ReplicationStats::default(); + let metrics = collect_replication_metrics(&stats); + + assert_eq!(metrics.len(), 9); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/request.rs b/crates/metrics/src/collectors/request.rs new file mode 100644 index 000000000..cd47aee04 --- /dev/null +++ b/crates/metrics/src/collectors/request.rs @@ -0,0 +1,174 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! API request metrics collector. +//! +//! Collects API request metrics including request counts, errors, +//! latency, and traffic statistics. + +use crate::format::PrometheusMetric; +use crate::metrics_type::request::*; + +/// API request statistics for a specific API endpoint. +#[derive(Debug, Clone, Default)] +pub struct ApiRequestStats { + /// API name (e.g., "GetObject", "PutObject") + pub name: String, + /// Request type (e.g., "s3", "admin") + pub req_type: String, + /// Number of requests currently in flight + pub in_flight: u64, + /// Total number of requests + pub total: u64, + /// Total number of errors (4xx + 5xx) + pub errors_total: u64, + /// Total number of 5xx errors + pub errors_5xx: u64, + /// Total number of 4xx errors + pub errors_4xx: u64, + /// Total number of canceled requests + pub canceled: u64, + /// TTFB distribution by bucket (le label) + pub ttfb_distribution: Vec<(String, f64)>, + /// Bytes sent + pub sent_bytes: u64, + /// Bytes received + pub recv_bytes: u64, +} + +/// Collects API request metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for API request statistics. +pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec { + let mut metrics = Vec::new(); + + for stat in stats { + // In-flight requests + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // Total requests + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // Total errors + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // 5xx errors + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // 4xx errors + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // Canceled requests + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + + // TTFB distribution (histogram buckets) + for (le, value) in &stat.ttfb_distribution { + metrics.push( + PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value) + .with_label_owned(NAME_LABEL, stat.name.clone()) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()) + .with_label_owned(LE_LABEL, le.clone()), + ); + } + + // Traffic metrics + metrics.push( + PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_MD, stat.sent_bytes as f64) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + metrics.push( + PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_MD, stat.recv_bytes as f64) + .with_label_owned(TYPE_LABEL, stat.req_type.clone()), + ); + } + + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_request_metrics() { + let stats = vec![ApiRequestStats { + name: "GetObject".to_string(), + req_type: "s3".to_string(), + in_flight: 10, + total: 10000, + errors_total: 50, + errors_5xx: 10, + errors_4xx: 40, + canceled: 5, + ttfb_distribution: vec![ + ("0.1".to_string(), 5000.0), + ("0.5".to_string(), 8000.0), + ("1.0".to_string(), 9500.0), + ("+Inf".to_string(), 10000.0), + ], + sent_bytes: 1024 * 1024 * 500, // 500 MB + recv_bytes: 1024 * 1024 * 100, // 100 MB + }]; + + let metrics = collect_request_metrics(&stats); + report_metrics(&metrics); + + // 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12 + assert_eq!(metrics.len(), 12); + + let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name(); + let total = metrics.iter().find(|m| m.name == total_name); + assert!(total.is_some()); + assert_eq!(total.map(|m| m.value), Some(10000.0)); + + let in_flight_name = API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name(); + let in_flight = metrics.iter().find(|m| m.name == in_flight_name); + assert!(in_flight.is_some()); + assert_eq!(in_flight.map(|m| m.value), Some(10.0)); + } + + #[test] + fn test_collect_request_metrics_empty() { + let stats: Vec = vec![]; + let metrics = collect_request_metrics(&stats); + assert!(metrics.is_empty()); + } +} diff --git a/crates/metrics/src/collectors/resource.rs b/crates/metrics/src/collectors/resource.rs index 636c690e9..a149c47b7 100644 --- a/crates/metrics/src/collectors/resource.rs +++ b/crates/metrics/src/collectors/resource.rs @@ -16,17 +16,21 @@ //! //! Collects system-level metrics for the RustFS process including //! CPU usage, memory consumption, and process uptime. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::process_resource` +//! to avoid duplication of metric names, types, and help text. -use crate::MetricType; use crate::format::PrometheusMetric; +use crate::metrics_type::process_resource::*; -/// Resource statistics for a RustFS process. +/// Resource statistics for metrics collection. /// -/// This struct encapsulates the resource usage data that can be -/// collected from the operating system for the current process. +/// This struct provides a decoupled interface for collecting resource metrics +/// without depending on specific internal types. HTTP handlers should populate +/// this struct from their available data sources. #[derive(Debug, Clone, Default)] pub struct ResourceStats { - /// CPU usage as a percentage (0.0 to 100.0+) + /// CPU usage as a percentage (can exceed 100% on multi-core systems) pub cpu_percent: f64, /// Resident memory usage in bytes pub memory_bytes: u64, @@ -34,65 +38,16 @@ pub struct ResourceStats { pub uptime_seconds: u64, } -// Static metric definitions -const METRIC_CPU: &str = "rustfs_process_cpu_percent"; -const METRIC_MEMORY: &str = "rustfs_process_memory_bytes"; -const METRIC_UPTIME: &str = "rustfs_process_uptime_seconds"; - -const HELP_CPU: &str = "CPU usage of the RustFS process as a percentage"; -const HELP_MEMORY: &str = "Resident memory usage of the RustFS process in bytes"; -const HELP_UPTIME: &str = "Uptime of the RustFS process in seconds"; - -/// Number of metrics produced by this collector. -const METRIC_COUNT: usize = 3; - -/// Collects system resource metrics from the provided statistics. +/// Collects resource metrics from the provided resource statistics. /// -/// # Metrics Produced -/// -/// - `rustfs_process_cpu_percent`: CPU usage as a percentage -/// - `rustfs_process_memory_bytes`: Resident memory usage in bytes -/// - `rustfs_process_uptime_seconds`: Process uptime in seconds -/// -/// # Arguments -/// -/// * `stats` - Resource statistics for the current process -/// -/// # Example -/// -/// ``` -/// use rustfs_metrics::collectors::{collect_resource_metrics, ResourceStats}; -/// -/// let stats = ResourceStats { -/// cpu_percent: 25.5, -/// memory_bytes: 1024 * 1024 * 512, // 512 MB -/// uptime_seconds: 3600, -/// }; -/// let metrics = collect_resource_metrics(&stats); -/// assert_eq!(metrics.len(), 3); -/// ``` -#[must_use] -#[inline] +/// Uses the metric descriptors from `metrics_type::process_resource` module. +/// Returns a vector of Prometheus metrics for resource statistics. pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec { - let mut metrics = Vec::with_capacity(METRIC_COUNT); - - metrics.push(PrometheusMetric::new(METRIC_CPU, MetricType::Gauge, HELP_CPU, stats.cpu_percent)); - - metrics.push(PrometheusMetric::new( - METRIC_MEMORY, - MetricType::Gauge, - HELP_MEMORY, - stats.memory_bytes as f64, - )); - - metrics.push(PrometheusMetric::new( - METRIC_UPTIME, - MetricType::Gauge, - HELP_UPTIME, - stats.uptime_seconds as f64, - )); - - metrics + vec![ + PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent), + PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64), + ] } #[cfg(test)] @@ -114,19 +69,21 @@ mod tests { assert_eq!(metrics.len(), 3); // Verify CPU metric - let cpu = metrics.iter().find(|m| m.name == METRIC_CPU); + let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name(); + let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 45.5); assert!(cpu.is_some()); - assert_eq!(cpu.map(|m| m.value), Some(45.5)); // Verify memory metric - let memory = metrics.iter().find(|m| m.name == METRIC_MEMORY); + let memory_metric_name = PROCESS_MEMORY_BYTES_MD.get_full_metric_name(); + let memory = metrics + .iter() + .find(|m| m.name == memory_metric_name && m.value == (1024 * 1024 * 256) as f64); assert!(memory.is_some()); - assert_eq!(memory.map(|m| m.value), Some((1024 * 1024 * 256) as f64)); // Verify uptime metric - let uptime = metrics.iter().find(|m| m.name == METRIC_UPTIME); + let uptime_metric_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name(); + let uptime = metrics.iter().find(|m| m.name == uptime_metric_name && m.value == 7200.0); assert!(uptime.is_some()); - assert_eq!(uptime.map(|m| m.value), Some(7200.0)); } #[test] @@ -155,9 +112,9 @@ mod tests { let metrics = collect_resource_metrics(&stats); report_metrics(&metrics); - let cpu = metrics.iter().find(|m| m.name == METRIC_CPU); + let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name(); + let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 150.0); assert!(cpu.is_some()); - assert_eq!(cpu.map(|m| m.value), Some(150.0)); } #[test] diff --git a/crates/metrics/src/collectors/scanner.rs b/crates/metrics/src/collectors/scanner.rs new file mode 100644 index 000000000..4b3bcf3d2 --- /dev/null +++ b/crates/metrics/src/collectors/scanner.rs @@ -0,0 +1,99 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Scanner metrics collector. +//! +//! Collects background scanner metrics including bucket scans, +//! directory scans, and object scans. + +use crate::format::PrometheusMetric; +use crate::metrics_type::scanner::{ + SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD, + SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD, +}; + +/// Scanner statistics. +#[derive(Debug, Clone, Default)] +pub struct ScannerStats { + /// Number of bucket scans finished + pub bucket_scans_finished: u64, + /// Number of bucket scans started + pub bucket_scans_started: u64, + /// Number of directories scanned + pub directories_scanned: u64, + /// Number of objects scanned + pub objects_scanned: u64, + /// Number of object versions scanned + pub versions_scanned: u64, + /// Seconds since last scanner activity + pub last_activity_seconds: u64, +} + +/// Collects scanner metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::scanner` module. +/// Returns a vector of Prometheus metrics for scanner statistics. +pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64), + PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64), + PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64), + PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64), + PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_scanner_metrics() { + let stats = ScannerStats { + bucket_scans_finished: 100, + bucket_scans_started: 100, + directories_scanned: 50000, + objects_scanned: 1000000, + versions_scanned: 1500000, + last_activity_seconds: 30, + }; + + let metrics = collect_scanner_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 6); + + let objects = metrics.iter().find(|m| m.value == 1000000.0); + assert!(objects.is_some()); + + let last_activity = metrics.iter().find(|m| m.value == 30.0); + assert!(last_activity.is_some()); + } + + #[test] + fn test_collect_scanner_metrics_default() { + let stats = ScannerStats::default(); + let metrics = collect_scanner_metrics(&stats); + + assert_eq!(metrics.len(), 6); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/stats_collector.rs b/crates/metrics/src/collectors/stats_collector.rs new file mode 100644 index 000000000..03e6b0951 --- /dev/null +++ b/crates/metrics/src/collectors/stats_collector.rs @@ -0,0 +1,233 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! Statistics collection functions for metrics. +//! +//! This module contains functions that collect statistics from various +//! RustFS internal sources (storage layer, bucket monitor, system info) +//! and convert them to the Stats structs used by collectors. + +use crate::collectors::{BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats}; +use rustfs_ecstore::bucket::metadata_sys::get_quota_config; +use rustfs_ecstore::data_usage::load_data_usage_from_backend; +use rustfs_ecstore::global::get_global_bucket_monitor; +use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions}; +use rustfs_ecstore::{StorageAPI, new_object_layer_fn}; +use std::sync::OnceLock; +use std::time::Instant; +use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System}; +use tracing::{instrument, warn}; + +/// Process start time for calculating uptime. +static PROCESS_START: OnceLock = OnceLock::new(); + +/// Get the process start time, initializing it on first call. +#[inline] +fn get_process_start() -> &'static Instant { + PROCESS_START.get_or_init(Instant::now) +} + +/// Collect cluster statistics from the storage layer. +#[instrument] +pub async fn collect_cluster_stats() -> ClusterStats { + let Some(store) = new_object_layer_fn() else { + return ClusterStats::default(); + }; + + let storage_info = store.storage_info().await; + + let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum(); + let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum(); + let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64; + let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64; + + // Get bucket and object counts from data usage info + let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await { + Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count), + Err(e) => { + warn!("Failed to load data usage from backend: {}", e); + // Fall back to bucket list for buckets_count, objects_count stays 0 + let buckets = store + .list_bucket(&BucketOptions { + cached: true, + ..Default::default() + }) + .await + .unwrap_or_else(|e| { + warn!("Failed to list buckets for cluster metrics: {}", e); + Vec::new() + }); + (buckets.len() as u64, 0) + } + }; + + ClusterStats { + raw_capacity_bytes: raw_capacity, + usable_capacity_bytes: usable_capacity, + used_bytes: used, + free_bytes: free, + objects_count, + buckets_count, + } +} + +/// Collect bucket statistics from the storage layer. +pub async fn collect_bucket_stats() -> Vec { + let Some(store) = new_object_layer_fn() else { + return Vec::new(); + }; + + // Load data usage info from backend to get bucket sizes and object counts + let data_usage = match load_data_usage_from_backend(store.clone()).await { + Ok(info) => Some(info), + Err(e) => { + warn!("Failed to load data usage for bucket metrics: {}", e); + None + } + }; + + // List all buckets + let buckets = match store + .list_bucket(&BucketOptions { + cached: true, + ..Default::default() + }) + .await + { + Ok(buckets) => buckets, + Err(e) => { + warn!("Failed to list buckets for bucket metrics: {}", e); + return Vec::new(); + } + }; + + let mut stats = Vec::with_capacity(buckets.len()); + + for bucket in buckets { + if bucket.name.starts_with('.') { + continue; + } + + // Get size and objects_count from data usage info + let (size_bytes, objects_count) = data_usage + .as_ref() + .and_then(|du| du.buckets_usage.get(&bucket.name)) + .map(|bui| (bui.size, bui.objects_count)) + .unwrap_or((0, 0)); + + // Get quota from bucket metadata + let quota_bytes = match get_quota_config(&bucket.name).await { + Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0), + Err(_) => 0, // No quota configured or error + }; + + stats.push(BucketStats { + name: bucket.name, + size_bytes, + objects_count, + quota_bytes, + }); + } + + stats +} + +/// Collect bucket replication bandwidth stats from the global monitor. +pub fn collect_bucket_replication_bandwidth_stats() -> Vec { + let Some(monitor) = get_global_bucket_monitor() else { + return Vec::new(); + }; + + monitor + .get_report(|_| true) + .bucket_stats + .into_iter() + .map(|(opts, details)| { + let target_arn = opts.replication_arn; + let limit_bytes_per_sec = u64::try_from(details.limit_bytes_per_sec).unwrap_or_else(|_| { + warn!( + "Invalid bandwidth limit value for target {:?}: {}", + target_arn, details.limit_bytes_per_sec + ); + 0 + }); + + BucketReplicationBandwidthStats { + bucket: opts.name, + target_arn, + limit_bytes_per_sec, + current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec, + } + }) + .collect() +} + +/// Collect disk statistics from the storage layer. +pub async fn collect_disk_stats() -> Vec { + let Some(store) = new_object_layer_fn() else { + return Vec::new(); + }; + + let storage_info = store.storage_info().await; + + storage_info + .disks + .iter() + .map(|disk| DiskStats { + server: disk.endpoint.clone(), + drive: disk.drive_path.clone(), + total_bytes: disk.total_space, + used_bytes: disk.used_space, + free_bytes: disk.available_space, + }) + .collect() +} + +/// Collect resource statistics for the current process. +/// +/// Collects: +/// - Uptime: Calculated from process start time +/// - Memory: Process resident set size from sysinfo +/// - CPU: Process CPU usage percentage from sysinfo +#[inline] +pub fn collect_process_stats() -> ResourceStats { + let uptime_seconds = get_process_start().elapsed().as_secs(); + + // Use sysinfo for process metrics + let mut sys = System::new(); + let pid = Pid::from_u32(std::process::id()); + sys.refresh_processes_specifics( + ProcessesToUpdate::Some(&[pid]), + true, + ProcessRefreshKind::nothing().with_cpu().with_memory(), + ); + + if let Some(process) = sys.process(pid) { + ResourceStats { + cpu_percent: process.cpu_usage() as f64, + memory_bytes: process.memory(), + uptime_seconds, + } + } else { + // Fallback if process info unavailable + ResourceStats { + cpu_percent: 0.0, + memory_bytes: 0, + uptime_seconds, + } + } +} diff --git a/crates/metrics/src/collectors/system_cpu.rs b/crates/metrics/src/collectors/system_cpu.rs new file mode 100644 index 000000000..3d3676de4 --- /dev/null +++ b/crates/metrics/src/collectors/system_cpu.rs @@ -0,0 +1,104 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System CPU metrics collector. +//! +//! Collects CPU-related metrics including load average, idle time, +//! I/O wait, and CPU usage percentages. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::system_cpu` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::system_cpu::*; + +/// CPU statistics for a node. +#[derive(Debug, Clone, Default)] +pub struct CpuStats { + /// Average CPU idle time (percentage, 0-100) + pub avg_idle: f64, + /// Average CPU I/O wait time (percentage, 0-100) + pub avg_iowait: f64, + /// CPU load average over 1 minute + pub load_avg: f64, + /// CPU load average as percentage + pub load_avg_perc: f64, + /// CPU nice time (percentage, 0-100) + pub nice: f64, + /// CPU steal time (percentage, 0-100) + pub steal: f64, + /// CPU system time (percentage, 0-100) + pub system: f64, + /// CPU user time (percentage, 0-100) + pub user: f64, +} + +/// Collects CPU metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::system_cpu` module. +/// Returns a vector of Prometheus metrics for CPU statistics. +pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle), + PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IOWAIT_MD, stats.avg_iowait), + PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg), + PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc), + PrometheusMetric::from_descriptor(&SYS_CPU_NICE_MD, stats.nice), + PrometheusMetric::from_descriptor(&SYS_CPU_STEAL_MD, stats.steal), + PrometheusMetric::from_descriptor(&SYS_CPU_SYSTEM_MD, stats.system), + PrometheusMetric::from_descriptor(&SYS_CPU_USER_MD, stats.user), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_cpu_metrics() { + let stats = CpuStats { + avg_idle: 75.5, + avg_iowait: 2.3, + load_avg: 1.5, + load_avg_perc: 37.5, + nice: 0.5, + steal: 0.1, + system: 10.0, + user: 15.0, + }; + + let metrics = collect_cpu_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 8); + + // Verify that metric names are properly generated from descriptors + assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_cpu_"))); + } + + #[test] + fn test_collect_cpu_metrics_default() { + let stats = CpuStats::default(); + let metrics = collect_cpu_metrics(&stats); + + assert_eq!(metrics.len(), 8); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/system_drive.rs b/crates/metrics/src/collectors/system_drive.rs new file mode 100644 index 000000000..4d819cdfc --- /dev/null +++ b/crates/metrics/src/collectors/system_drive.rs @@ -0,0 +1,228 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System drive metrics collector. +//! +//! Collects detailed drive/disk metrics including capacity, I/O statistics, +//! error counts, and health status. + +use crate::format::PrometheusMetric; +use crate::metrics_type::system_drive::*; + +/// Detailed drive statistics for a single drive. +#[derive(Debug, Clone, Default)] +pub struct DriveDetailedStats { + /// Server identifier (e.g., "node1:9000") + pub server: String, + /// Drive path (e.g., "/data/disk1") + pub drive: String, + /// Total capacity in bytes + pub total_bytes: u64, + /// Used capacity in bytes + pub used_bytes: u64, + /// Free capacity in bytes + pub free_bytes: u64, + /// Used inodes + pub used_inodes: u64, + /// Free inodes + pub free_inodes: u64, + /// Total inodes + pub total_inodes: u64, + /// Total timeout errors + pub timeout_errors_total: u64, + /// Total I/O errors + pub io_errors_total: u64, + /// Total availability errors + pub availability_errors_total: u64, + /// Number of I/O operations waiting + pub waiting_io: u64, + /// API latency in microseconds + pub api_latency_micros: u64, + /// Health status (1=healthy, 0=unhealthy) + pub health: u8, + /// Reads per second + pub reads_per_sec: f64, + /// Kilobytes read per second + pub reads_kb_per_sec: f64, + /// Average read await time + pub reads_await: f64, + /// Writes per second + pub writes_per_sec: f64, + /// Kilobytes written per second + pub writes_kb_per_sec: f64, + /// Average write await time + pub writes_await: f64, + /// Percentage utilization + pub perc_util: f64, +} + +/// Aggregate drive count statistics. +#[derive(Debug, Clone, Default)] +pub struct DriveCountStats { + /// Number of offline Drives + pub offline_count: u64, + /// Number of online drives + pub online_count: u64, + /// Total number of drives + pub total_count: u64, +} + +/// Collects detailed drive metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for each drive. +pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec { + fn push_drive_metric( + metrics: &mut Vec, + descriptor: &'static crate::MetricDescriptor, + value: f64, + server_label: &str, + drive_label: &str, + ) { + metrics.push( + PrometheusMetric::from_descriptor(descriptor, value) + .with_label_owned(DRIVE_LABEL, drive_label.to_string()) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + ); + } + + let mut metrics = Vec::with_capacity(stats.len() * 19); + + for stat in stats { + let server_label = stat.server.as_str(); + let drive_label = stat.drive.as_str(); + + push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label); + push_drive_metric( + &mut metrics, + &DRIVE_TIMEOUT_ERRORS_MD, + stat.timeout_errors_total as f64, + server_label, + drive_label, + ); + push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, stat.io_errors_total as f64, server_label, drive_label); + push_drive_metric( + &mut metrics, + &DRIVE_AVAILABILITY_ERRORS_MD, + stat.availability_errors_total as f64, + server_label, + drive_label, + ); + push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, stat.waiting_io as f64, server_label, drive_label); + push_drive_metric( + &mut metrics, + &DRIVE_API_LATENCY_MD, + stat.api_latency_micros as f64, + server_label, + drive_label, + ); + push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, stat.reads_per_sec, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, stat.reads_kb_per_sec, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, stat.reads_await, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, stat.writes_per_sec, server_label, drive_label); + push_drive_metric( + &mut metrics, + &DRIVE_WRITES_KB_PER_SEC_MD, + stat.writes_kb_per_sec, + server_label, + drive_label, + ); + push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, stat.writes_await, server_label, drive_label); + push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, stat.perc_util, server_label, drive_label); + } + + metrics +} + +/// Collects drive count metrics (offline, online, total). +/// +/// Returns a vector of Prometheus metrics for drive counts. +pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&DRIVE_OFFLINE_COUNT_MD, stats.offline_count as f64), + PrometheusMetric::from_descriptor(&DRIVE_ONLINE_COUNT_MD, stats.online_count as f64), + PrometheusMetric::from_descriptor(&DRIVE_COUNT_MD, stats.total_count as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_drive_detailed_metrics() { + let stats = vec![DriveDetailedStats { + server: "node1:9000".to_string(), + drive: "/data/disk1".to_string(), + total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB + used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB + free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB + used_inodes: 100000, + free_inodes: 900000, + total_inodes: 1000000, + timeout_errors_total: 5, + io_errors_total: 10, + availability_errors_total: 2, + waiting_io: 3, + api_latency_micros: 1500, + health: 1, + reads_per_sec: 100.0, + reads_kb_per_sec: 1024.0, + reads_await: 5.5, + writes_per_sec: 50.0, + writes_kb_per_sec: 512.0, + writes_await: 10.2, + perc_util: 75.5, + }]; + + let metrics = collect_drive_detailed_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 19); + + // Verify total bytes metric + let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name(); + let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name); + assert!(total_bytes.is_some()); + assert_eq!(total_bytes.map(|m| m.value), Some(1024.0 * 1024.0 * 1024.0 * 100.0)); + } + + #[test] + fn test_collect_drive_count_metrics() { + let stats = DriveCountStats { + offline_count: 2, + online_count: 8, + total_count: 10, + }; + + let metrics = collect_drive_count_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 3); + + // Verify offline count + let offline_name = DRIVE_OFFLINE_COUNT_MD.get_full_metric_name(); + let offline = metrics.iter().find(|m| m.name == offline_name); + assert!(offline.is_some()); + assert_eq!(offline.map(|m| m.value), Some(2.0)); + } +} diff --git a/crates/metrics/src/collectors/system_memory.rs b/crates/metrics/src/collectors/system_memory.rs new file mode 100644 index 000000000..0622bde90 --- /dev/null +++ b/crates/metrics/src/collectors/system_memory.rs @@ -0,0 +1,102 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System memory metrics collector. +//! +//! Collects memory-related metrics including total, used, free, +//! buffers, cache, shared, and available memory. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::system_memory` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::system_memory::*; + +/// Memory statistics for a node. +#[derive(Debug, Clone, Default)] +pub struct MemoryStats { + /// Total memory in bytes + pub total: u64, + /// Used memory in bytes + pub used: u64, + /// Used memory percentage (0-100) + pub used_perc: f64, + /// Free memory in bytes + pub free: u64, + /// Buffer memory in bytes + pub buffers: u64, + /// Cache memory in bytes + pub cache: u64, + /// Shared memory in bytes + pub shared: u64, + /// Available memory in bytes + pub available: u64, +} + +/// Collects memory metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::system_memory` module. +/// Returns a vector of Prometheus metrics for memory statistics. +pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64), + PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64), + PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc), + PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64), + PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64), + PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64), + PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64), + PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_memory_metrics() { + let stats = MemoryStats { + total: 16 * 1024 * 1024 * 1024, // 16 GB + used: 8 * 1024 * 1024 * 1024, // 8 GB + used_perc: 50.0, + free: 4 * 1024 * 1024 * 1024, // 4 GB + buffers: 1024 * 1024 * 512, // 512 MB + cache: 2 * 1024 * 1024 * 1024, // 2 GB + shared: 1024 * 1024 * 256, // 256 MB + available: 6 * 1024 * 1024 * 1024, // 6 GB + }; + + let metrics = collect_memory_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 8); + assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_memory_"))); + } + + #[test] + fn test_collect_memory_metrics_default() { + let stats = MemoryStats::default(); + let metrics = collect_memory_metrics(&stats); + + assert_eq!(metrics.len(), 8); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/system_network.rs b/crates/metrics/src/collectors/system_network.rs new file mode 100644 index 000000000..07f916d1c --- /dev/null +++ b/crates/metrics/src/collectors/system_network.rs @@ -0,0 +1,90 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System network metrics collector. +//! +//! Collects internode network metrics including errors, dial times, +//! and bytes sent/received. +//! +//! This collector reuses the metric descriptors defined in `metrics_type::system_network` +//! to avoid duplication of metric names, types, and help text. + +use crate::format::PrometheusMetric; +use crate::metrics_type::system_network::*; + +/// Network statistics for internode communication. +#[derive(Debug, Clone, Default)] +pub struct NetworkStats { + /// Total number of failed internode calls + pub internode_errors_total: u64, + /// Total number of TCP dial timeouts and errors + pub internode_dial_errors_total: u64, + /// Average dial time in nanoseconds + pub internode_dial_avg_time_nanos: u64, + /// Total bytes sent to other nodes + pub internode_sent_bytes_total: u64, + /// Total bytes received from other nodes + pub internode_recv_bytes_total: u64, +} + +/// Collects network metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::system_network` module. +/// Returns a vector of Prometheus metrics for network statistics. +pub fn collect_network_metrics(stats: &NetworkStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64), + PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64), + PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64), + PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64), + PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_network_metrics() { + let stats = NetworkStats { + internode_errors_total: 10, + internode_dial_errors_total: 5, + internode_dial_avg_time_nanos: 1_500_000, // 1.5ms + internode_sent_bytes_total: 1024 * 1024 * 100, // 100 MB + internode_recv_bytes_total: 1024 * 1024 * 200, // 200 MB + }; + + let metrics = collect_network_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 5); + assert!(metrics.iter().all(|m| m.name.contains("internode"))); + } + + #[test] + fn test_collect_network_metrics_default() { + let stats = NetworkStats::default(); + let metrics = collect_network_metrics(&stats); + + assert_eq!(metrics.len(), 5); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/collectors/system_process.rs b/crates/metrics/src/collectors/system_process.rs new file mode 100644 index 000000000..cc11706b1 --- /dev/null +++ b/crates/metrics/src/collectors/system_process.rs @@ -0,0 +1,145 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System process metrics collector. +//! +//! Collects process-level metrics including file descriptors, memory, +//! syscalls, and runtime statistics. + +use crate::format::PrometheusMetric; +use crate::metrics_type::system_process::*; + +/// Process statistics for the RustFS server process. +#[derive(Debug, Clone, Default)] +pub struct ProcessStats { + /// Total read locks held + pub locks_read_total: u64, + /// Total write locks held + pub locks_write_total: u64, + /// Total CPU time in seconds + pub cpu_total_seconds: f64, + /// Total number of async tasks (goroutines equivalent) + pub go_routine_total: u64, + /// Total bytes read via read syscalls (rchar) + pub io_rchar_bytes: u64, + /// Total bytes actually read from storage + pub io_read_bytes: u64, + /// Total bytes written via write syscalls (wchar) + pub io_wchar_bytes: u64, + /// Total bytes actually written to storage + pub io_write_bytes: u64, + /// Process start time in seconds since Unix epoch + pub start_time_seconds: u64, + /// Process uptime in seconds + pub uptime_seconds: u64, + /// File descriptor limit + pub file_descriptor_limit_total: u64, + /// Open file descriptors count + pub file_descriptor_open_total: u64, + /// Total read syscalls + pub syscall_read_total: u64, + /// Total write syscalls + pub syscall_write_total: u64, + /// Resident memory size in bytes + pub resident_memory_bytes: u64, + /// Virtual memory size in bytes + pub virtual_memory_bytes: u64, + /// Maximum virtual memory size in bytes + pub virtual_memory_max_bytes: u64, +} + +/// Collects process metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for process statistics. +pub fn collect_process_metrics(stats: &ProcessStats) -> Vec { + vec![ + PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds), + PrometheusMetric::from_descriptor(&PROCESS_GO_ROUTINE_TOTAL_MD, stats.go_routine_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_IO_RCHAR_BYTES_MD, stats.io_rchar_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_IO_READ_BYTES_MD, stats.io_read_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_IO_WCHAR_BYTES_MD, stats.io_wchar_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_IO_WRITE_BYTES_MD, stats.io_write_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_START_TIME_SECONDS_MD, stats.start_time_seconds as f64), + PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64), + PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD, stats.file_descriptor_limit_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD, stats.file_descriptor_open_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_READ_TOTAL_MD, stats.syscall_read_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_WRITE_TOTAL_MD, stats.syscall_write_total as f64), + PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64), + PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::report_metrics; + + #[test] + fn test_collect_process_metrics() { + let stats = ProcessStats { + locks_read_total: 100, + locks_write_total: 50, + cpu_total_seconds: 1234.56, + go_routine_total: 200, + io_rchar_bytes: 1024 * 1024 * 500, + io_read_bytes: 1024 * 1024 * 400, + io_wchar_bytes: 1024 * 1024 * 300, + io_write_bytes: 1024 * 1024 * 250, + start_time_seconds: 1700000000, + uptime_seconds: 86400, + file_descriptor_limit_total: 65536, + file_descriptor_open_total: 1500, + syscall_read_total: 100000, + syscall_write_total: 50000, + resident_memory_bytes: 1024 * 1024 * 512, + virtual_memory_bytes: 1024 * 1024 * 1024, + virtual_memory_max_bytes: 1024 * 1024 * 2048, + }; + + let metrics = collect_process_metrics(&stats); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 17); + + // Verify uptime + let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name(); + let uptime = metrics.iter().find(|m| m.name == uptime_name); + assert!(uptime.is_some()); + assert_eq!(uptime.map(|m| m.value), Some(86400.0)); + + // Verify file descriptors + let fd_open_name = PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD.get_full_metric_name(); + let fd_open = metrics.iter().find(|m| m.name == fd_open_name); + assert!(fd_open.is_some()); + assert_eq!(fd_open.map(|m| m.value), Some(1500.0)); + } + + #[test] + fn test_collect_process_metrics_default() { + let stats = ProcessStats::default(); + let metrics = collect_process_metrics(&stats); + + assert_eq!(metrics.len(), 17); + for metric in &metrics { + assert_eq!(metric.value, 0.0); + assert!(metric.labels.is_empty()); + } + } +} diff --git a/crates/metrics/src/format.rs b/crates/metrics/src/format.rs index 87c9eec03..e6c9c2fe9 100644 --- a/crates/metrics/src/format.rs +++ b/crates/metrics/src/format.rs @@ -20,21 +20,50 @@ use crate::MetricType; use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge}; use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +static NAME_CACHE: OnceLock>> = OnceLock::new(); +static HELP_CACHE: OnceLock>> = OnceLock::new(); + +fn intern_string(cache: &OnceLock>>, value: &str) -> &'static str { + let cache = cache.get_or_init(Default::default); + let mut cache = match cache.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if let Some(existing) = cache.get(value) { + existing + } else { + let value = Box::leak(value.to_string().into_boxed_str()); + cache.insert(value.to_string(), value); + value + } +} + +fn into_static_str(cache: &OnceLock>>, value: &str) -> &'static str { + intern_string(cache, value) +} /// Report metrics using the `metrics` crate. /// /// This function iterates over the provided metrics and reports them using /// the `metrics` crate's API. This allows integration with various metrics /// exporters (e.g., Prometheus) that are configured globally. +/// pub fn report_metrics(metrics: &[PrometheusMetric]) { for metric in metrics { + let name = into_static_str(&NAME_CACHE, &metric.name); + let help = into_static_str(&HELP_CACHE, &metric.help); + // Register metric description (help text) // Note: In a real-world scenario, descriptions should ideally be registered once at startup. // However, the `metrics` crate handles duplicate registrations gracefully. match metric.metric_type { - MetricType::Counter => describe_counter!(metric.name, metric.help), - MetricType::Gauge => describe_gauge!(metric.name, metric.help), - MetricType::Histogram => describe_histogram!(metric.name, metric.help), + MetricType::Counter => describe_counter!(name, help), + MetricType::Gauge => describe_gauge!(name, help), + MetricType::Histogram => describe_histogram!(name, help), } // Convert labels to the format expected by `metrics` crate @@ -50,15 +79,15 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) { // // Since `metrics` 0.21+, `Counter` has an `absolute` method which sets the counter to a specific value. // This is useful for mirroring an external counter. - let counter = counter!(metric.name, &labels); + let counter = counter!(name, &labels); counter.absolute(metric.value as u64); } MetricType::Gauge => { - let gauge = gauge!(metric.name, &labels); + let gauge = gauge!(name, &labels); gauge.set(metric.value); } MetricType::Histogram => { - let histogram = metrics::histogram!(metric.name, &labels); + let histogram = metrics::histogram!(name, &labels); histogram.record(metric.value); } } @@ -67,17 +96,17 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) { /// A single Prometheus metric with labels and value. /// -/// This struct is optimized for performance by using `&'static str` for -/// the name and help text, which are typically compile-time constants. +/// This struct is optimized for performance by using `Cow<'static, str>` for +/// the name and help text, which allows both static strings and owned strings. /// Labels use `Cow<'static, str>` to avoid allocations when possible. #[derive(Debug, Clone)] pub struct PrometheusMetric { /// The metric name (e.g., "http_requests_total"). - pub name: &'static str, + pub name: Cow<'static, str>, /// The type of this metric (counter, gauge, or histogram). pub metric_type: MetricType, /// Human-readable description shown in Prometheus UI. - pub help: &'static str, + pub help: Cow<'static, str>, /// Key-value label pairs for this metric instance. /// Uses Cow to avoid allocations for static label keys. pub labels: Vec<(&'static str, Cow<'static, str>)>, @@ -92,9 +121,39 @@ impl PrometheusMetric { #[inline] pub const fn new(name: &'static str, metric_type: MetricType, help: &'static str, value: f64) -> Self { Self { - name, + name: Cow::Borrowed(name), metric_type, - help, + help: Cow::Borrowed(help), + labels: Vec::new(), + value, + } + } + + /// Creates a new metric with owned strings for name and help. + /// + /// Use this when the metric name or help text is dynamically generated. + #[inline] + pub fn new_owned(name: String, metric_type: MetricType, help: String, value: f64) -> Self { + Self { + name: Cow::Owned(name), + metric_type, + help: Cow::Owned(help), + labels: Vec::new(), + value, + } + } + + /// Creates a new metric from a MetricDescriptor. + /// + /// This is the recommended way to create metrics when using MetricDescriptor + /// from the metrics_type module. + #[inline] + pub fn from_descriptor(descriptor: &crate::MetricDescriptor, value: f64) -> Self { + let help = intern_string(&HELP_CACHE, &descriptor.help); + Self { + name: Cow::Owned(descriptor.get_full_metric_name()), + metric_type: descriptor.metric_type, + help: Cow::Borrowed(help), labels: Vec::new(), value, } diff --git a/crates/metrics/src/metrics_type/cluster.rs b/crates/metrics/src/metrics_type/cluster.rs new file mode 100644 index 000000000..a786fb7bf --- /dev/null +++ b/crates/metrics/src/metrics_type/cluster.rs @@ -0,0 +1,78 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; +use std::sync::LazyLock; + +/// Total raw storage capacity across all disks in bytes +pub static CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("capacity_raw_total_bytes".to_string()), + "Total raw storage capacity in bytes across all disks", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); + +/// Total usable storage capacity in bytes (accounting for erasure coding) +pub static CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("capacity_usable_total_bytes".to_string()), + "Total usable storage capacity in bytes (accounting for erasure coding)", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); + +/// Total used storage capacity in bytes +pub static CLUSTER_CAPACITY_USED_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("capacity_used_bytes".to_string()), + "Total used storage capacity in bytes", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); + +/// Total free storage capacity in bytes +pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("capacity_free_bytes".to_string()), + "Total free storage capacity in bytes", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); + +/// Total number of objects in the cluster +pub static CLUSTER_OBJECTS_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("objects_total".to_string()), + "Total number of objects in the cluster", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); + +/// Total number of buckets in the cluster +pub static CLUSTER_BUCKETS_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("buckets_total".to_string()), + "Total number of buckets in the cluster", + &[], + subsystems::CLUSTER_BASE_PATH, + ) +}); diff --git a/crates/metrics/src/metrics_type/cluster_erasure_set.rs b/crates/metrics/src/metrics_type/cluster_erasure_set.rs index 44b1ff812..3c496f92f 100644 --- a/crates/metrics/src/metrics_type/cluster_erasure_set.rs +++ b/crates/metrics/src/metrics_type/cluster_erasure_set.rs @@ -22,6 +22,33 @@ pub const POOL_ID_L: &str = "pool_id"; /// The label for the pool ID pub const SET_ID_L: &str = "set_id"; +pub static ERASURE_SET_SIZE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ErasureSetSize, + "Total number of drives in the erasure set in a pool", + &[POOL_ID_L, SET_ID_L], + subsystems::CLUSTER_ERASURE_SET, + ) +}); + +pub static ERASURE_SET_PARITY_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ErasureSetParity, + "Parity drives in the erasure set in a pool", + &[POOL_ID_L, SET_ID_L], + subsystems::CLUSTER_ERASURE_SET, + ) +}); + +pub static ERASURE_SET_DATA_SHARDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ErasureSetDataShards, + "Data shards in the erasure set in a pool", + &[POOL_ID_L, SET_ID_L], + subsystems::CLUSTER_ERASURE_SET, + ) +}); + pub static ERASURE_SET_OVERALL_WRITE_QUORUM_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ErasureSetOverallWriteQuorum, diff --git a/crates/metrics/src/metrics_type/entry/metric_name.rs b/crates/metrics/src/metrics_type/entry/metric_name.rs index 9a53b8c38..af98dc774 100644 --- a/crates/metrics/src/metrics_type/entry/metric_name.rs +++ b/crates/metrics/src/metrics_type/entry/metric_name.rs @@ -183,6 +183,9 @@ pub enum MetricName { ConfigStandardParity, // Erasure coding set related metrics + ErasureSetSize, + ErasureSetParity, + ErasureSetDataShards, ErasureSetOverallWriteQuorum, ErasureSetOverallHealth, ErasureSetReadQuorum, @@ -502,6 +505,9 @@ impl MetricName { Self::ConfigStandardParity => "standard_parity".to_string(), // Erasure coding set related metrics + Self::ErasureSetSize => "size".to_string(), + Self::ErasureSetParity => "parity".to_string(), + Self::ErasureSetDataShards => "data_shards".to_string(), Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(), Self::ErasureSetOverallHealth => "overall_health".to_string(), Self::ErasureSetReadQuorum => "read_quorum".to_string(), diff --git a/crates/metrics/src/metrics_type/mod.rs b/crates/metrics/src/metrics_type/mod.rs index 4bc6d4c55..d3eac431d 100644 --- a/crates/metrics/src/metrics_type/mod.rs +++ b/crates/metrics/src/metrics_type/mod.rs @@ -15,6 +15,7 @@ pub mod audit; pub mod bucket; pub mod bucket_replication; +pub mod cluster; pub mod cluster_config; pub mod cluster_erasure_set; pub mod cluster_health; @@ -24,6 +25,9 @@ pub mod cluster_usage; pub mod entry; pub mod ilm; pub mod logger_webhook; +pub mod node_bucket; +pub mod node_disk; +pub mod process_resource; pub mod replication; pub mod request; pub mod scanner; diff --git a/crates/metrics/src/metrics_type/node_bucket.rs b/crates/metrics/src/metrics_type/node_bucket.rs new file mode 100644 index 000000000..fa006eb43 --- /dev/null +++ b/crates/metrics/src/metrics_type/node_bucket.rs @@ -0,0 +1,50 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; +use std::sync::LazyLock; + +const BUCKET_LABEL: &str = "bucket"; + +/// Total bytes used by the bucket +pub static BUCKET_USAGE_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("usage_bytes".to_string()), + "Total bytes used by the bucket", + &[BUCKET_LABEL], + subsystems::BUCKET_API, + ) +}); + +/// Total number of objects in the bucket +pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("objects_total".to_string()), + "Total number of objects in the bucket", + &[BUCKET_LABEL], + subsystems::BUCKET_API, + ) +}); + +/// Quota limit in bytes for the bucket +pub static BUCKET_QUOTA_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("quota_bytes".to_string()), + "Quota limit in bytes for the bucket", + &[BUCKET_LABEL], + subsystems::BUCKET_API, + ) +}); diff --git a/crates/metrics/src/metrics_type/node_disk.rs b/crates/metrics/src/metrics_type/node_disk.rs new file mode 100644 index 000000000..fad52b6c7 --- /dev/null +++ b/crates/metrics/src/metrics_type/node_disk.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; +use std::sync::LazyLock; + +const SERVER_LABEL: &str = "server"; +const DRIVE_LABEL: &str = "drive"; + +/// Total disk capacity in bytes +pub static NODE_DISK_TOTAL_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("disk_total_bytes".to_string()), + "Total disk capacity in bytes", + &[SERVER_LABEL, DRIVE_LABEL], + MetricSubsystem::new("/node"), + ) +}); + +/// Used disk space in bytes +pub static NODE_DISK_USED_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("disk_used_bytes".to_string()), + "Used disk space in bytes", + &[SERVER_LABEL, DRIVE_LABEL], + MetricSubsystem::new("/node"), + ) +}); + +/// Free disk space in bytes +pub static NODE_DISK_FREE_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("disk_free_bytes".to_string()), + "Free disk space in bytes", + &[SERVER_LABEL, DRIVE_LABEL], + MetricSubsystem::new("/node"), + ) +}); diff --git a/crates/metrics/src/metrics_type/process_resource.rs b/crates/metrics/src/metrics_type/process_resource.rs new file mode 100644 index 000000000..9c0c03e34 --- /dev/null +++ b/crates/metrics/src/metrics_type/process_resource.rs @@ -0,0 +1,48 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; +use std::sync::LazyLock; + +/// CPU usage of the RustFS process as a percentage +pub static PROCESS_CPU_PERCENT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("cpu_percent".to_string()), + "CPU usage of the RustFS process as a percentage", + &[], + MetricSubsystem::new("/process"), + ) +}); + +/// Resident memory usage of the RustFS process in bytes +pub static PROCESS_MEMORY_BYTES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("memory_bytes".to_string()), + "Resident memory usage of the RustFS process in bytes", + &[], + MetricSubsystem::new("/process"), + ) +}); + +/// Uptime of the RustFS process in seconds +pub static PROCESS_UPTIME_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("uptime_seconds".to_string()), + "Uptime of the RustFS process in seconds", + &[], + MetricSubsystem::new("/process"), + ) +}); diff --git a/crates/metrics/src/metrics_type/request.rs b/crates/metrics/src/metrics_type/request.rs index aaeb1f1ba..be037d653 100644 --- a/crates/metrics/src/metrics_type/request.rs +++ b/crates/metrics/src/metrics_type/request.rs @@ -16,6 +16,12 @@ use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; +/// name label +pub const NAME_LABEL: &str = "name"; +/// type label +pub const TYPE_LABEL: &str = "type"; +/// le label (for histogram buckets) +pub const LE_LABEL: &str = "le"; pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( diff --git a/crates/metrics/src/metrics_type/system_drive.rs b/crates/metrics/src/metrics_type/system_drive.rs index a9752946b..ab8e07b2b 100644 --- a/crates/metrics/src/metrics_type/system_drive.rs +++ b/crates/metrics/src/metrics_type/system_drive.rs @@ -19,6 +19,8 @@ use std::sync::LazyLock; /// drive related labels pub const DRIVE_LABEL: &str = "drive"; +/// server label +pub const SERVER_LABEL: &str = "server"; /// pool index label pub const POOL_INDEX_LABEL: &str = "pool_index"; /// set index label