gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(metrics): modularize collectors and add comprehensive metrics support (#2208)

Browse Source 
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-03-18 23:52:42 +08:00
committed by GitHub
parent 4a807d80e3
commit a4411a24d6
35 changed files with 2959 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/metrics/src/collectors/audit.rs
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Audit metrics collector.
//!
//! Collects audit log metrics including failed messages, queue length,
//! and total messages per target.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::audit`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::audit::*;
use std::borrow::Cow;
/// Audit target statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct AuditTargetStats {
/// Target identifier
pub target_id: String,
/// Number of messages that failed to send
pub failed_messages: u64,
/// Number of unsent messages in queue
pub queue_length: u64,
/// Total number of messages sent
pub total_messages: u64,
}
/// Collects audit metrics from the provided audit target statistics.
///
/// Uses the metric descriptors from `metrics_type::audit` module.
/// Returns a vector of Prometheus metrics for audit statistics.
pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * 3);
for stat in stats {
let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label("target_id", target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label("target_id", target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label("target_id", target_id_label),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_audit_metrics() {
let stats = vec![
AuditTargetStats {
target_id: "target-1".to_string(),
failed_messages: 5,
queue_length: 10,
total_messages: 1000,
},
AuditTargetStats {
target_id: "target-2".to_string(),
failed_messages: 2,
queue_length: 5,
total_messages: 500,
},
];
let metrics = collect_audit_metrics(&stats);
assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each
let failed = metrics
.iter()
.find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1"));
assert!(failed.is_some());
}
#[test]
fn test_collect_audit_metrics_empty() {
let stats: Vec<AuditTargetStats> = vec![];
let metrics = collect_audit_metrics(&stats);
assert!(metrics.is_empty());
}
}
crates/metrics/src/collectors/bucket.rs
+38 -63
View File
@@ -16,90 +16,56 @@
//!
//! Collects usage metrics for each bucket in the cluster, including
//! size, object counts, and quota information.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::node_bucket`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::node_bucket::*;
use std::borrow::Cow;
/// Usage statistics for a single bucket.
/// Bucket statistics for metrics collection.
///
/// This struct provides a decoupled interface for collecting bucket metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct BucketStats {
/// Bucket name
/// Name of the bucket
pub name: String,
/// Total bytes used by the bucket
/// Total size of all objects in the bucket (bytes)
pub size_bytes: u64,
/// Total number of objects in the bucket
/// Number of objects in the bucket
pub objects_count: u64,
/// Quota limit in bytes (0 means no quota)
/// Quota limit for the bucket (bytes), 0 if no quota
pub quota_bytes: u64,
}
// Static metric definitions
const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes";
const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total";
const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes";
const HELP_SIZE: &str = "Total bytes used by the bucket";
const HELP_OBJECTS: &str = "Total number of objects in the bucket";
const HELP_QUOTA: &str = "Quota limit in bytes for the bucket";
/// Collects per-bucket usage metrics from the provided bucket statistics.
/// Collects per-bucket metrics from the provided bucket statistics.
///
/// # Metrics Produced
///
/// For each bucket, the following metrics are produced with a `bucket` label:
///
/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket
/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket
/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured)
///
/// # Arguments
///
/// * `buckets` - Slice of bucket statistics
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats};
///
/// let buckets = vec![
/// BucketStats {
/// name: "my-bucket".to_string(),
/// size_bytes: 1_000_000,
/// objects_count: 100,
/// quota_bytes: 10_000_000,
/// },
/// ];
/// let metrics = collect_bucket_metrics(&buckets);
/// assert_eq!(metrics.len(), 3); // size, objects, quota
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::node_bucket` module.
/// Returns a vector of Prometheus metrics for all buckets.
pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
if buckets.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(buckets.len() * 3);
for bucket in buckets {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
// Bucket size in bytes
metrics.push(
PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64)
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
// Object count
metrics.push(
PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64)
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
// Quota (always emit, 0 when no quota configured for consistent PromQL queries)
metrics.push(
PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64)
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
.with_label("bucket", bucket_label),
);
}
@@ -118,27 +84,28 @@ mod tests {
BucketStats {
name: "test-bucket".to_string(),
size_bytes: 1000,
objects_count: 50,
objects_count: 100,
quota_bytes: 0,
},
BucketStats {
name: "other-bucket".to_string(),
name: "another-bucket".to_string(),
size_bytes: 2000,
objects_count: 100,
objects_count: 200,
quota_bytes: 0,
},
];
let metrics = collect_bucket_metrics(&buckets);
report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here.
report_metrics(&metrics);
// 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
assert_eq!(metrics.len(), 6);
// Verify test-bucket metrics
// Verify test-bucket metrics have correct labels
let test_bucket_size_name = BUCKET_USAGE_BYTES_MD.get_full_metric_name();
let test_bucket_size = metrics
.iter()
.find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
.find(|m| m.name == test_bucket_size_name && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
assert!(test_bucket_size.is_some());
assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
}
@@ -159,9 +126,13 @@ mod tests {
assert_eq!(metrics.len(), 3);
// Verify quota metric exists
let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
let quota_metric = metrics.iter().find(|m| {
m.name == quota_metric_name
&& m.value == 10000.0
&& m.labels.iter().any(|(k, v)| *k == "bucket" && v == "quota-bucket")
});
assert!(quota_metric.is_some());
assert_eq!(quota_metric.map(|m| m.value), Some(10000.0));
}
#[test]
@@ -185,9 +156,13 @@ mod tests {
// Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
assert_eq!(metrics.len(), 3);
let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
let quota_metric = metrics.iter().find(|m| {
m.name == quota_metric_name
&& m.value == 0.0
&& m.labels.iter().any(|(k, v)| *k == "bucket" && v == "no-quota-bucket")
});
assert!(quota_metric.is_some());
assert_eq!(quota_metric.map(|m| m.value), Some(0.0));
}
#[test]
crates/metrics/src/collectors/bucket_replication.rs
+31 -36
View File
@@ -13,32 +13,33 @@
// limitations under the License.
//! Bucket replication bandwidth metrics collector.
//!
//! Collects bandwidth metrics for bucket replication targets.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
use std::borrow::Cow;
/// Bucket replication bandwidth stats for one replication target.
/// Bucket replication bandwidth statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
/// Name of the bucket
pub bucket: String,
/// Target ARN for replication
pub target_arn: String,
pub limit_bytes_per_sec: i64,
/// Configured bandwidth limit in bytes per second
pub limit_bytes_per_sec: u64,
/// Current bandwidth in bytes per second (EWMA)
pub current_bandwidth_bytes_per_sec: f64,
}
const BUCKET_LABEL: &str = "bucket";
const TARGET_ARN_LABEL: &str = "targetArn";
const METRIC_BANDWIDTH_LIMIT: &str = "rustfs_bucket_replication_bandwidth_limit_bytes_per_second";
const METRIC_BANDWIDTH_CURRENT: &str = "rustfs_bucket_replication_bandwidth_current_bytes_per_second";
const HELP_BANDWIDTH_LIMIT: &str = "Configured bandwidth limit for replication in bytes per second";
const HELP_BANDWIDTH_CURRENT: &str = "Current replication bandwidth in bytes per second (EWMA)";
/// Collect bucket replication bandwidth metrics for Prometheus/OpenTelemetry export.
#[must_use]
#[inline]
/// Collects bucket replication bandwidth metrics from the provided statistics.
///
/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
/// Returns a vector of Prometheus metrics for replication bandwidth.
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -50,25 +51,15 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone());
metrics.push(
PrometheusMetric::new(
METRIC_BANDWIDTH_LIMIT,
MetricType::Gauge,
HELP_BANDWIDTH_LIMIT,
stat.limit_bytes_per_sec as f64,
)
.with_label(BUCKET_LABEL, bucket_label.clone())
.with_label(TARGET_ARN_LABEL, target_arn_label.clone()),
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
.with_label("bucket", bucket_label.clone())
.with_label("targetArn", target_arn_label.clone()),
);
metrics.push(
PrometheusMetric::new(
METRIC_BANDWIDTH_CURRENT,
MetricType::Gauge,
HELP_BANDWIDTH_CURRENT,
stat.current_bandwidth_bytes_per_sec,
)
.with_label(BUCKET_LABEL, bucket_label)
.with_label(TARGET_ARN_LABEL, target_arn_label),
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
.with_label("bucket", bucket_label)
.with_label("targetArn", target_arn_label),
);
}
@@ -91,23 +82,27 @@ mod tests {
let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
assert_eq!(metrics.len(), 2);
let limit_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_LIMIT);
let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
let limit_metric = metrics.iter().find(|m| {
m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
});
assert!(limit_metric.is_some());
assert_eq!(limit_metric.map(|m| m.value), Some(1_048_576.0));
assert!(
limit_metric
.and_then(|m| {
m.labels
.iter()
.find(|(k, _)| *k == TARGET_ARN_LABEL)
.find(|(k, _)| *k == "targetArn")
.map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
})
.unwrap_or(false)
);
let current_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_CURRENT);
let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
let current_metric = metrics.iter().find(|m| {
m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
});
assert!(current_metric.is_some());
assert_eq!(current_metric.map(|m| m.value), Some(204_800.0));
}
#[test]
crates/metrics/src/collectors/cluster.rs
+23 -89
View File
@@ -16,9 +16,12 @@
//!
//! Collects aggregate metrics across the entire RustFS cluster including
//! total capacity, usage, and object counts.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster::*;
/// Cluster capacity and usage statistics for metrics collection.
///
@@ -41,88 +44,19 @@ pub struct ClusterStats {
pub buckets_count: u64,
}
// Static metric definitions to avoid allocations
const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes";
const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes";
const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes";
const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes";
const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total";
const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total";
const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks";
const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)";
const HELP_USED: &str = "Total used storage capacity in bytes";
const HELP_FREE: &str = "Total free storage capacity in bytes";
const HELP_OBJECTS: &str = "Total number of objects in the cluster";
const HELP_BUCKETS: &str = "Total number of buckets in the cluster";
/// Number of metrics produced by this collector.
const METRIC_COUNT: usize = 6;
/// Collects cluster-wide metrics from the provided statistics.
/// Collects cluster-wide metrics from the provided cluster statistics.
///
/// # Metrics Produced
///
/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks
/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead
/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity
/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity
/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster
/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster
///
/// # Arguments
///
/// * `stats` - Cluster statistics containing capacity and usage data
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats};
///
/// let stats = ClusterStats {
/// raw_capacity_bytes: 10_000_000_000,
/// usable_capacity_bytes: 8_000_000_000,
/// used_bytes: 2_000_000_000,
/// free_bytes: 6_000_000_000,
/// objects_count: 1000,
/// buckets_count: 10,
/// };
/// let metrics = collect_cluster_metrics(&stats);
/// assert_eq!(metrics.len(), 6);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::cluster` module.
/// Returns a vector of Prometheus metrics for cluster statistics.
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(METRIC_COUNT);
metrics.push(PrometheusMetric::new(
METRIC_RAW_CAPACITY,
MetricType::Gauge,
HELP_RAW_CAPACITY,
stats.raw_capacity_bytes as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_USABLE_CAPACITY,
MetricType::Gauge,
HELP_USABLE_CAPACITY,
stats.usable_capacity_bytes as f64,
));
metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64));
metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64));
metrics.push(PrometheusMetric::new(
METRIC_OBJECTS,
MetricType::Gauge,
HELP_OBJECTS,
stats.objects_count as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_BUCKETS,
MetricType::Gauge,
HELP_BUCKETS,
stats.buckets_count as f64,
));
metrics
vec![
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
}
#[cfg(test)]
@@ -147,24 +81,24 @@ mod tests {
assert_eq!(metrics.len(), 6);
// Verify raw capacity
let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY);
let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name();
let raw_capacity = metrics.iter().find(|m| m.name == raw_capacity_name && m.value == 3000.0);
assert!(raw_capacity.is_some());
assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0));
// Verify used capacity
let used = metrics.iter().find(|m| m.name == METRIC_USED);
let used_name = CLUSTER_CAPACITY_USED_BYTES_MD.get_full_metric_name();
let used = metrics.iter().find(|m| m.name == used_name && m.value == 1200.0);
assert!(used.is_some());
assert_eq!(used.map(|m| m.value), Some(1200.0));
// Verify object count
let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS);
let objects_name = CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name();
let objects = metrics.iter().find(|m| m.name == objects_name && m.value == 100.0);
assert!(objects.is_some());
assert_eq!(objects.map(|m| m.value), Some(100.0));
// Verify bucket count
let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS);
let buckets_name = CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name();
let buckets = metrics.iter().find(|m| m.name == buckets_name && m.value == 5.0);
assert!(buckets.is_some());
assert_eq!(buckets.map(|m| m.value), Some(5.0));
}
#[test]
crates/metrics/src/collectors/cluster_config.rs
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster config metrics collector.
//!
//! Collects cluster configuration metrics including storage class
//! parity settings.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster_config`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_config::*;
/// Cluster configuration statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterConfigStats {
/// Parity for reduced redundancy storage (RRS) class
pub rrs_parity: u32,
/// Parity for standard storage class
pub standard_parity: u32,
}
/// Collects cluster config metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_config` module.
/// Returns a vector of Prometheus metrics for cluster configuration.
pub fn collect_cluster_config_metrics(stats: &ClusterConfigStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&CONFIG_RRS_PARITY_MD, stats.rrs_parity as f64),
PrometheusMetric::from_descriptor(&CONFIG_STANDARD_PARITY_MD, stats.standard_parity as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_cluster_config_metrics() {
let stats = ClusterConfigStats {
rrs_parity: 2,
standard_parity: 4,
};
let metrics = collect_cluster_config_metrics(&stats);
assert_eq!(metrics.len(), 2);
let rrs = metrics.iter().find(|m| m.value == 2.0);
assert!(rrs.is_some());
let standard = metrics.iter().find(|m| m.value == 4.0);
assert!(standard.is_some());
}
#[test]
fn test_collect_cluster_config_metrics_default() {
let stats = ClusterConfigStats::default();
let metrics = collect_cluster_config_metrics(&stats);
assert_eq!(metrics.len(), 2);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/cluster_erasure_set.rs
+179
View File
@@ -0,0 +1,179 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster erasure set metrics collector.
//!
//! Collects erasure coding set metrics including parity, quorum,
//! drive counts, and health status.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_erasure_set::*;
/// Erasure set statistics.
#[derive(Debug, Clone, Default)]
pub struct ErasureSetStats {
/// Pool ID
pub pool_id: u32,
/// Set ID within the pool
pub set_id: u32,
/// Total number of drives in the set
pub size: u32,
/// Number of parity drives
pub parity: u32,
/// Number of data shards
pub data_shards: u32,
/// Read quorum
pub read_quorum: u32,
/// Write quorum
pub write_quorum: u32,
/// Number of online drives
pub online_drives_count: u32,
/// Number of healing drives
pub healing_drives_count: u32,
/// Health status (1=healthy, 0=unhealthy)
pub health: u8,
/// Read tolerance (number of drive failures tolerated for reads)
pub read_tolerance: u32,
/// Write tolerance (number of drive failures tolerated for writes)
pub write_tolerance: u32,
/// Read health status (1=healthy, 0=unhealthy)
pub read_health: u8,
/// Write health status (1=healthy, 0=unhealthy)
pub write_health: u8,
}
/// Collects erasure set metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for erasure sets.
pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(stats.len() * 12);
for stat in stats {
let pool_id_label = stat.pool_id.to_string();
let set_id_label = stat.set_id.to_string();
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_SIZE_MD, stat.size as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_PARITY_MD, stat.parity as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_DATA_SHARDS_MD, stat.data_shards as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_QUORUM_MD, stat.read_quorum as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_QUORUM_MD, stat.write_quorum as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_ONLINE_DRIVES_COUNT_MD, stat.online_drives_count as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_HEALING_DRIVES_COUNT_MD, stat.healing_drives_count as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_HEALTH_MD, stat.health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_TOLERANCE_MD, stat.read_tolerance as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_TOLERANCE_MD, stat.write_tolerance as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_HEALTH_MD, stat.read_health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_erasure_set_metrics() {
let stats = vec![ErasureSetStats {
pool_id: 1,
set_id: 0,
size: 16,
parity: 4,
data_shards: 12,
read_quorum: 13,
write_quorum: 13,
online_drives_count: 16,
healing_drives_count: 0,
health: 1,
read_tolerance: 4,
write_tolerance: 4,
read_health: 1,
write_health: 1,
}];
let metrics = collect_erasure_set_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 12);
let size_name = ERASURE_SET_SIZE_MD.get_full_metric_name();
let size = metrics.iter().find(|m| m.name == size_name);
assert!(size.is_some());
assert_eq!(size.map(|m| m.value), Some(16.0));
let health_name = ERASURE_SET_HEALTH_MD.get_full_metric_name();
let health = metrics.iter().find(|m| m.name == health_name);
assert!(health.is_some());
assert_eq!(health.map(|m| m.value), Some(1.0));
}
#[test]
fn test_collect_erasure_set_metrics_empty() {
let stats: Vec<ErasureSetStats> = vec![];
let metrics = collect_erasure_set_metrics(&stats);
assert!(metrics.is_empty());
}
}
crates/metrics/src/collectors/cluster_health.rs
+85
View File
@@ -0,0 +1,85 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster health metrics collector.
//!
//! Collects cluster-wide health metrics including drive counts
//! (offline, online, total).
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster_health`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_health::*;
/// Cluster health statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterHealthStats {
/// Number of offline drives in the cluster
pub drives_offline_count: u64,
/// Number of online drives in the cluster
pub drives_online_count: u64,
/// Total number of drives in the cluster
pub drives_count: u64,
}
/// Collects cluster health metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_health` module.
/// Returns a vector of Prometheus metrics for cluster health.
pub fn collect_cluster_health_metrics(stats: &ClusterHealthStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_OFFLINE_COUNT_MD, stats.drives_offline_count as f64),
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_ONLINE_COUNT_MD, stats.drives_online_count as f64),
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_COUNT_MD, stats.drives_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_cluster_health_metrics() {
let stats = ClusterHealthStats {
drives_offline_count: 2,
drives_online_count: 18,
drives_count: 20,
};
let metrics = collect_cluster_health_metrics(&stats);
assert_eq!(metrics.len(), 3);
let offline = metrics.iter().find(|m| m.value == 2.0);
assert!(offline.is_some());
let online = metrics.iter().find(|m| m.value == 18.0);
assert!(online.is_some());
}
#[test]
fn test_collect_cluster_health_metrics_default() {
let stats = ClusterHealthStats::default();
let metrics = collect_cluster_health_metrics(&stats);
assert_eq!(metrics.len(), 3);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/cluster_iam.rs
+117
View File
@@ -0,0 +1,117 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster IAM metrics collector.
//!
//! Collects IAM (Identity and Access Management) metrics including
//! plugin authentication service stats and sync statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_iam::*;
/// IAM statistics.
#[derive(Debug, Clone, Default)]
pub struct IamStats {
/// Time in seconds since last failed authn service request
pub plugin_authn_service_last_fail_seconds: u64,
/// Time in seconds since last successful authn service request
pub plugin_authn_service_last_succ_seconds: u64,
/// Average RTT of successful requests in the last minute (ms)
pub plugin_authn_service_succ_avg_rtt_ms_minute: u64,
/// Maximum RTT of successful requests in the last minute (ms)
pub plugin_authn_service_succ_max_rtt_ms_minute: u64,
/// Total requests count in the last full minute
pub plugin_authn_service_total_requests_minute: u64,
/// Time in milliseconds since last successful IAM data sync
pub since_last_sync_millis: u64,
/// Number of failed IAM data syncs since server start
pub sync_failures: u64,
/// Number of successful IAM data syncs since server start
pub sync_successes: u64,
}
/// Collects IAM metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for IAM statistics.
pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
stats.plugin_authn_service_last_fail_seconds as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD,
stats.plugin_authn_service_last_succ_seconds as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD,
stats.plugin_authn_service_succ_avg_rtt_ms_minute as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD,
stats.plugin_authn_service_succ_max_rtt_ms_minute as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD,
stats.plugin_authn_service_total_requests_minute as f64,
),
PrometheusMetric::from_descriptor(&SINCE_LAST_SYNC_MILLIS_MD, stats.since_last_sync_millis as f64),
PrometheusMetric::from_descriptor(&SYNC_FAILURES_MD, stats.sync_failures as f64),
PrometheusMetric::from_descriptor(&SYNC_SUCCESSES_MD, stats.sync_successes as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_iam_metrics() {
let stats = IamStats {
plugin_authn_service_last_fail_seconds: 3600,
plugin_authn_service_last_succ_seconds: 10,
plugin_authn_service_succ_avg_rtt_ms_minute: 50,
plugin_authn_service_succ_max_rtt_ms_minute: 200,
plugin_authn_service_total_requests_minute: 1000,
since_last_sync_millis: 5000,
sync_failures: 5,
sync_successes: 1000,
};
let metrics = collect_iam_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name();
let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name);
assert!(sync_successes.is_some());
assert_eq!(sync_successes.map(|m| m.value), Some(1000.0));
}
#[test]
fn test_collect_iam_metrics_default() {
let stats = IamStats::default();
let metrics = collect_iam_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/cluster_usage.rs
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster usage metrics collector.
//!
//! Collects cluster-wide and per-bucket usage metrics including
//! object counts, sizes, versions, and distributions.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_usage::*;
/// Cluster-wide usage statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterUsageStats {
/// Total bytes used in the cluster
pub total_bytes: u64,
/// Total number of objects
pub objects_count: u64,
/// Total number of object versions (including delete markers)
pub versions_count: u64,
/// Total number of delete markers
pub delete_markers_count: u64,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
pub versions_distribution: Vec<(String, u64)>,
}
/// Per-bucket usage statistics.
#[derive(Debug, Clone, Default)]
pub struct BucketUsageStats {
/// Bucket name
pub bucket: String,
/// Total bytes used in the bucket
pub total_bytes: u64,
/// Total number of objects in the bucket
pub objects_count: u64,
/// Total number of object versions (including delete markers)
pub versions_count: u64,
/// Total number of delete markers
pub delete_markers_count: u64,
/// Bucket quota in bytes (0 if no quota)
pub quota_bytes: u64,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
pub version_count_distribution: Vec<(String, u64)>,
}
/// Collects cluster-wide usage metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for cluster usage.
pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len());
metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64));
metrics.push(PrometheusMetric::from_descriptor(
&USAGE_DELETE_MARKERS_COUNT_MD,
stats.delete_markers_count as f64,
));
// Object size distribution
for (range, count) in &stats.object_size_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_OBJECTS_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone()),
);
}
// Version distribution
for (range, count) in &stats.versions_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_VERSIONS_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone()),
);
}
metrics
}
/// Collects per-bucket usage metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for bucket usage.
pub fn collect_bucket_usage_metrics(stats: &[BucketUsageStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
for stat in stats {
let bucket_label = stat.bucket.clone();
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, stat.total_bytes as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, stat.objects_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, stat.versions_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, stat.delete_markers_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, stat.quota_bytes as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
// Object size distribution per bucket
for (range, count) in &stat.object_size_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone())
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
}
// Version count distribution per bucket
for (range, count) in &stat.version_count_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone())
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
}
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_cluster_usage_metrics() {
let stats = ClusterUsageStats {
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
objects_count: 10000,
versions_count: 15000,
delete_markers_count: 500,
object_size_distribution: vec![
("0-1KB".to_string(), 5000),
("1KB-1MB".to_string(), 3000),
("1MB-100MB".to_string(), 1500),
("100MB+".to_string(), 500),
],
versions_distribution: vec![("1".to_string(), 8000), ("2-5".to_string(), 1500), ("6+".to_string(), 500)],
};
let metrics = collect_cluster_usage_metrics(&stats);
report_metrics(&metrics);
// 4 base metrics + 4 size distribution + 3 version distribution = 11
assert_eq!(metrics.len(), 11);
let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
assert!(total_bytes.is_some());
}
#[test]
fn test_collect_bucket_usage_metrics() {
let stats = vec![BucketUsageStats {
bucket: "test-bucket".to_string(),
total_bytes: 1024 * 1024 * 1024 * 10, // 10 GB
objects_count: 1000,
versions_count: 1200,
delete_markers_count: 50,
quota_bytes: 1024 * 1024 * 1024 * 100, // 100 GB quota
object_size_distribution: vec![("0-1KB".to_string(), 500)],
version_count_distribution: vec![("1".to_string(), 800)],
}];
let metrics = collect_bucket_usage_metrics(&stats);
report_metrics(&metrics);
// 5 base metrics + 1 size distribution + 1 version distribution = 7
assert_eq!(metrics.len(), 7);
}
}
crates/metrics/src/collectors/global.rs
+77 -228
View File
@@ -12,9 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Global metrics collector initialization.
//!
//! This module provides the entry point for initializing all metrics collectors.
//! The actual statistics collection functions are in `stats_collector.rs`.
use crate::collectors::stats_collector::{
collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats,
collect_process_stats,
};
use crate::collectors::{
BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics,
collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, collect_resource_metrics,
collect_bucket_metrics, collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics,
collect_resource_metrics,
};
use crate::constants::{
DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL,
@@ -23,245 +32,85 @@ use crate::constants::{
ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
};
use crate::format::report_metrics;
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use rustfs_utils::get_env_opt_u64;
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{instrument, warn};
use tracing::warn;
/// Process start time for calculating uptime.
static PROCESS_START: OnceLock<Instant> = OnceLock::new();
/// Get the process start time, initializing it on first call.
#[inline]
fn get_process_start() -> &'static Instant {
PROCESS_START.get_or_init(Instant::now)
}
/// Collect cluster statistics from the storage layer.
#[instrument]
async fn collect_cluster_stats() -> ClusterStats {
let Some(store) = new_object_layer_fn() else {
return ClusterStats::default();
};
let storage_info = store.storage_info().await;
let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
// Get bucket and object counts from data usage info
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!("Failed to load data usage from backend: {}", e);
// Fall back to bucket list for buckets_count, objects_count stays 0
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|e| {
warn!("Failed to list buckets for cluster metrics: {}", e);
Vec::new()
});
(buckets.len() as u64, 0)
}
};
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
}
}
/// Collect bucket statistics from the storage layer.
async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_data_usage_from_backend(store.clone()).await {
Ok(info) => Some(info),
Err(e) => {
warn!("Failed to load data usage from backend for bucket metrics: {}", e);
None
}
};
let buckets = match store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
{
Ok(b) => b,
Err(e) => {
warn!("Failed to list buckets for metrics: {}", e);
return Vec::new();
}
};
// Build bucket stats with real data from DataUsageInfo
let mut stats = Vec::with_capacity(buckets.len());
for bucket in buckets {
if bucket.name.starts_with('.') {
continue;
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
// Get quota from bucket metadata
let quota_bytes = match get_quota_config(&bucket.name).await {
Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
Err(_) => 0, // No quota configured or error
};
stats.push(BucketStats {
name: bucket.name,
size_bytes,
objects_count,
quota_bytes,
});
}
stats
}
/// Collect bucket replication bandwidth stats from the global monitor.
fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
let Some(monitor) = get_global_bucket_monitor() else {
return Vec::new();
};
monitor
.get_report(|_| true)
.bucket_stats
.into_iter()
.map(|(opts, details)| BucketReplicationBandwidthStats {
bucket: opts.name,
target_arn: opts.replication_arn,
limit_bytes_per_sec: details.limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
})
.collect()
}
/// Collect disk statistics from the storage layer.
async fn collect_disk_stats() -> Vec<DiskStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
let storage_info = store.storage_info().await;
storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
})
.collect()
}
/// Collect resource statistics for the current process.
///
/// Collects:
/// - Uptime: Calculated from process start time
/// - Memory: Process resident set size from sysinfo
/// - CPU: Process CPU usage percentage from sysinfo
#[inline]
fn collect_process_stats() -> ResourceStats {
let uptime_seconds = get_process_start().elapsed().as_secs();
// Use sysinfo for process metrics
let mut sys = System::new();
let pid = Pid::from_u32(std::process::id());
sys.refresh_processes_specifics(
ProcessesToUpdate::Some(&[pid]),
true,
ProcessRefreshKind::nothing().with_cpu().with_memory(),
);
if let Some(process) = sys.process(pid) {
ResourceStats {
cpu_percent: process.cpu_usage() as f64,
memory_bytes: process.memory(),
uptime_seconds,
}
} else {
// Fallback if process not found
ResourceStats {
cpu_percent: 0.0,
memory_bytes: 0,
uptime_seconds,
}
}
}
/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics.
/// Initialize all metrics collectors.
///
/// This function spawns background tasks that periodically collect metrics
/// and report them using the `metrics` crate.
/// from various sources and report them to the metrics system.
///
/// # Arguments
/// * `token` - A `CancellationToken` that can be used to gracefully shut down
/// all metrics collection tasks.
///
/// * `token` - A cancellation token to gracefully stop the metrics collection tasks.
/// # Environment Variables
/// The collection intervals can be configured via environment variables:
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60)
/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300)
/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60)
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30)
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15)
/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds.
///
/// Legacy interval names without `_SEC` are still accepted for backward compatibility:
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL`
/// - `RUSTFS_METRICS_BUCKET_INTERVAL`
/// - `RUSTFS_METRICS_NODE_INTERVAL`
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL`
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL`
pub fn init_metrics_collectors(token: CancellationToken) {
// Initialize process start time
get_process_start();
const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL";
const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL";
const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL";
const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL";
const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
// Helper closure to determine interval for a specific metric type
let get_interval = |env_key: &str, type_default: Duration| -> Duration {
// 1. Try specific env var
// 2. Fallback to global default env var (if set differently from hardcoded default)
// 3. Fallback to type specific default
fn parse_interval(msc: &str, legacy_msc: &str) -> Option<u64> {
get_env_opt_u64(msc)
.or_else(|| get_env_opt_u64(legacy_msc))
.filter(|&v| v > 0)
}
// Helper to check if value is valid (non-zero)
let is_valid = |v: u64| v > 0;
// Read intervals from environment or use defaults, ensuring zero is ignored.
let cluster_interval = parse_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_CLUSTER_METRICS_INTERVAL);
if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) {
Duration::from_secs(val)
} else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) {
Duration::from_secs(val)
} else {
type_default
}
};
let bucket_interval = parse_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_BUCKET_METRICS_INTERVAL);
let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
let bucket_replication_bandwidth_interval = get_interval(
ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
);
let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
let bucket_replication_bandwidth_interval =
parse_interval(ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, LEGACY_REPLICATION_BANDWIDTH_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL);
let node_interval = parse_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_NODE_METRICS_INTERVAL);
let resource_interval = parse_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_RESOURCE_METRICS_INTERVAL);
// Spawn task for cluster metrics
let token_clone = token.clone();
crates/metrics/src/collectors/ilm.rs
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! ILM (Information Lifecycle Management) metrics collector.
//!
//! Collects ILM metrics including pending tasks, active tasks,
//! and scanned versions.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::ilm`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::ilm::*;
/// ILM statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct IlmStats {
/// Number of pending ILM expiry tasks
pub expiry_pending_tasks: u64,
/// Number of active ILM transition tasks
pub transition_active_tasks: u64,
/// Number of pending ILM transition tasks
pub transition_pending_tasks: u64,
/// Number of missed immediate ILM transition tasks
pub transition_missed_immediate_tasks: u64,
/// Total number of object versions scanned for ILM
pub versions_scanned: u64,
}
/// Collects ILM metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::ilm` module.
/// Returns a vector of Prometheus metrics for ILM statistics.
pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&ILM_EXPIRY_PENDING_TASKS_MD, stats.expiry_pending_tasks as f64),
PrometheusMetric::from_descriptor(&ILM_TRANSITION_ACTIVE_TASKS_MD, stats.transition_active_tasks as f64),
PrometheusMetric::from_descriptor(&ILM_TRANSITION_PENDING_TASKS_MD, stats.transition_pending_tasks as f64),
PrometheusMetric::from_descriptor(
&ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD,
stats.transition_missed_immediate_tasks as f64,
),
PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_ilm_metrics() {
let stats = IlmStats {
expiry_pending_tasks: 100,
transition_active_tasks: 5,
transition_pending_tasks: 50,
transition_missed_immediate_tasks: 10,
versions_scanned: 1000000,
};
let metrics = collect_ilm_metrics(&stats);
assert_eq!(metrics.len(), 5);
let pending = metrics.iter().find(|m| m.value == 100.0);
assert!(pending.is_some());
let scanned = metrics.iter().find(|m| m.value == 1000000.0);
assert!(scanned.is_some());
}
#[test]
fn test_collect_ilm_metrics_default() {
let stats = IlmStats::default();
let metrics = collect_ilm_metrics(&stats);
assert_eq!(metrics.len(), 5);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/logger_webhook.rs
+122
View File
@@ -0,0 +1,122 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Logger webhook metrics collector.
//!
//! Collects webhook metrics including failed messages, queue length,
//! and total messages per webhook target.
use crate::format::PrometheusMetric;
use crate::metrics_type::logger_webhook::{
ENDPOINT_LABEL, NAME_LABEL, WEBHOOK_FAILED_MESSAGES_MD, WEBHOOK_QUEUE_LENGTH_MD, WEBHOOK_TOTAL_MESSAGES_MD,
};
use std::borrow::Cow;
/// Webhook target statistics.
#[derive(Debug, Clone, Default)]
pub struct WebhookTargetStats {
/// Webhook name
pub name: String,
/// Webhook endpoint URL
pub endpoint: String,
/// Number of failed messages
pub failed_messages: u64,
/// Number of messages in queue
pub queue_length: u64,
/// Total number of messages sent
pub total_messages: u64,
}
/// Collects webhook metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::logger_webhook` module.
/// Returns a vector of Prometheus metrics for webhook statistics.
pub fn collect_webhook_metrics(stats: &[WebhookTargetStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(stats.len() * 3);
for stat in stats {
let name_label: Cow<'static, str> = Cow::Owned(stat.name.clone());
let endpoint_label: Cow<'static, str> = Cow::Owned(stat.endpoint.clone());
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_webhook_metrics() {
let stats = vec![WebhookTargetStats {
name: "alert-webhook".to_string(),
endpoint: "https://hooks.example.com/alert".to_string(),
failed_messages: 3,
queue_length: 15,
total_messages: 5000,
}];
let metrics = collect_webhook_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 3);
// Verify we have metrics with the expected values
let failed = metrics.iter().find(|m| m.value == 3.0);
assert!(failed.is_some());
// Verify labels
let total = metrics.iter().find(|m| m.value == 5000.0);
assert!(total.is_some());
let total_metric = total.unwrap();
assert!(
total_metric
.labels
.iter()
.any(|(k, v)| *k == NAME_LABEL && v == "alert-webhook")
);
assert!(
total_metric
.labels
.iter()
.any(|(k, v)| *k == ENDPOINT_LABEL && v == "https://hooks.example.com/alert")
);
}
#[test]
fn test_collect_webhook_metrics_empty() {
let stats: Vec<WebhookTargetStats> = vec![];
let metrics = collect_webhook_metrics(&stats);
assert!(metrics.is_empty());
}
}
crates/metrics/src/collectors/mod.rs
+35
View File
@@ -61,16 +61,51 @@
//! report_metrics(&metrics);
//! ```
mod audit;
mod bucket;
mod bucket_replication;
mod cluster;
mod cluster_config;
mod cluster_erasure_set;
mod cluster_health;
mod cluster_iam;
mod cluster_usage;
pub(crate) mod global;
mod ilm;
mod logger_webhook;
mod node;
mod notification;
mod replication;
mod request;
mod resource;
mod scanner;
mod stats_collector;
mod system_cpu;
mod system_drive;
mod system_memory;
mod system_network;
mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
pub use cluster::{ClusterStats, collect_cluster_metrics};
pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics};
pub use cluster_iam::{IamStats, collect_iam_metrics};
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
pub use global::init_metrics_collectors;
pub use ilm::{IlmStats, collect_ilm_metrics};
pub use logger_webhook::{WebhookTargetStats, collect_webhook_metrics};
pub use node::{DiskStats, collect_node_metrics};
pub use notification::{NotificationStats, collect_notification_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub use request::{ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub use scanner::{ScannerStats, collect_scanner_metrics};
pub use system_cpu::{CpuStats, collect_cpu_metrics};
pub use system_drive::{DriveCountStats, DriveDetailedStats, collect_drive_count_metrics, collect_drive_detailed_metrics};
pub use system_memory::{MemoryStats, collect_memory_metrics};
pub use system_network::{NetworkStats, collect_network_metrics};
pub use system_process::{ProcessStats, collect_process_metrics};
crates/metrics/src/collectors/node.rs
+24 -54
View File
@@ -16,93 +16,61 @@
//!
//! Collects storage metrics for each disk/drive in the cluster,
//! including capacity, usage, and health status.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::node_disk`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::node_disk::*;
use std::borrow::Cow;
/// Statistics for a single disk/drive.
/// Disk statistics for metrics collection.
///
/// This struct provides a decoupled interface for collecting disk metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct DiskStats {
/// Server endpoint (e.g., "node1:9000")
pub server: String,
/// Drive path (e.g., "/data/disk1")
pub drive: String,
/// Total capacity in bytes
/// Total disk capacity in bytes
pub total_bytes: u64,
/// Used space in bytes
/// Used disk space in bytes
pub used_bytes: u64,
/// Free space in bytes
/// Free disk space in bytes
pub free_bytes: u64,
}
// Static metric definitions
const METRIC_TOTAL: &str = "rustfs_node_disk_total_bytes";
const METRIC_USED: &str = "rustfs_node_disk_used_bytes";
const METRIC_FREE: &str = "rustfs_node_disk_free_bytes";
const HELP_TOTAL: &str = "Total disk capacity in bytes";
const HELP_USED: &str = "Used disk space in bytes";
const HELP_FREE: &str = "Free disk space in bytes";
/// Collects per-node disk metrics from the provided disk statistics.
/// Collects per-disk metrics from the provided disk statistics.
///
/// # Metrics Produced
///
/// For each disk, the following metrics are produced with `server` and `drive` labels:
///
/// - `rustfs_node_disk_total_bytes`: Total capacity of the disk
/// - `rustfs_node_disk_used_bytes`: Used space on the disk
/// - `rustfs_node_disk_free_bytes`: Free space on the disk
///
/// # Arguments
///
/// * `disks` - Slice of disk statistics
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_node_metrics, DiskStats};
///
/// let disks = vec![
/// DiskStats {
/// server: "node1:9000".to_string(),
/// drive: "/data/disk1".to_string(),
/// total_bytes: 1_000_000_000,
/// used_bytes: 400_000_000,
/// free_bytes: 600_000_000,
/// },
/// ];
/// let metrics = collect_node_metrics(&disks);
/// assert_eq!(metrics.len(), 3);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::node_disk` module.
/// Returns a vector of Prometheus metrics for all disks.
pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec<PrometheusMetric> {
if disks.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(disks.len() * 3);
for disk in disks {
let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone());
let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone());
metrics.push(
PrometheusMetric::new(METRIC_TOTAL, MetricType::Gauge, HELP_TOTAL, disk.total_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_TOTAL_BYTES_MD, disk.total_bytes as f64)
.with_label("server", server_label.clone())
.with_label("drive", drive_label.clone()),
);
metrics.push(
PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, disk.used_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_USED_BYTES_MD, disk.used_bytes as f64)
.with_label("server", server_label.clone())
.with_label("drive", drive_label.clone()),
);
metrics.push(
PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, disk.free_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_FREE_BYTES_MD, disk.free_bytes as f64)
.with_label("server", server_label)
.with_label("drive", drive_label),
);
@@ -140,22 +108,24 @@ mod tests {
assert_eq!(metrics.len(), 6);
// Verify node1 disk1 total bytes
let node1_total_name = NODE_DISK_TOTAL_BYTES_MD.get_full_metric_name();
let node1_total = metrics.iter().find(|m| {
m.name == METRIC_TOTAL
m.name == node1_total_name
&& m.value == 1000000.0
&& m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000")
&& m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1")
});
assert!(node1_total.is_some());
assert_eq!(node1_total.map(|m| m.value), Some(1000000.0));
// Verify node2 disk2 used bytes
let node2_used_name = NODE_DISK_USED_BYTES_MD.get_full_metric_name();
let node2_used = metrics.iter().find(|m| {
m.name == METRIC_USED
m.name == node2_used_name
&& m.value == 800000.0
&& m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000")
&& m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2")
});
assert!(node2_used.is_some());
assert_eq!(node2_used.map(|m| m.value), Some(800000.0));
}
#[test]
crates/metrics/src/collectors/notification.rs
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Notification metrics collector.
//!
//! Collects notification system metrics including events sent,
//! errors, and skipped events.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_notification::{
NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
};
/// Notification statistics.
#[derive(Debug, Clone, Default)]
pub struct NotificationStats {
/// Number of concurrent send operations in progress
pub current_send_in_progress: u64,
/// Total number of events that encountered errors
pub events_errors_total: u64,
/// Total number of events successfully sent
pub events_sent_total: u64,
/// Total number of events skipped
pub events_skipped_total: u64,
}
/// Collects notification metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_notification` module.
/// Returns a vector of Prometheus metrics for notification statistics.
pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, stats.current_send_in_progress as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, stats.events_errors_total as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_MD, stats.events_sent_total as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, stats.events_skipped_total as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_notification_metrics() {
let stats = NotificationStats {
current_send_in_progress: 5,
events_errors_total: 10,
events_sent_total: 10000,
events_skipped_total: 50,
};
let metrics = collect_notification_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 4);
let sent = metrics.iter().find(|m| m.value == 10000.0);
assert!(sent.is_some());
let errors = metrics.iter().find(|m| m.value == 10.0);
assert!(errors.is_some());
}
#[test]
fn test_collect_notification_metrics_default() {
let stats = NotificationStats::default();
let metrics = collect_notification_metrics(&stats);
assert_eq!(metrics.len(), 4);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/replication.rs
+107
View File
@@ -0,0 +1,107 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Replication metrics collector.
//!
//! Collects cluster-wide replication metrics including queue stats,
//! data transfer rates, and worker information.
use crate::format::PrometheusMetric;
use crate::metrics_type::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
/// Number of active replication workers
pub active_workers: u64,
/// Current data transfer rate in bytes/sec
pub current_data_transfer_rate: f64,
/// Bytes queued in the last full minute
pub last_minute_queued_bytes: u64,
/// Objects queued in the last full minute
pub last_minute_queued_count: u64,
/// Maximum active workers seen since server start
pub max_active_workers: u64,
/// Maximum bytes queued since server start
pub max_queued_bytes: u64,
/// Maximum objects queued since server start
pub max_queued_count: u64,
/// Maximum data transfer rate seen since server start
pub max_data_transfer_rate: f64,
/// Objects in replication backlog in the last 5 minutes
pub recent_backlog_count: u64,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD, stats.last_minute_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_MD, stats.max_active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_MD, stats.max_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_MD, stats.max_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_DATA_TRANSFER_RATE_MD, stats.max_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_RECENT_BACKLOG_COUNT_MD, stats.recent_backlog_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
active_workers: 10,
current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
last_minute_queued_bytes: 1024 * 1024 * 100, // 100 MB
last_minute_queued_count: 500,
max_active_workers: 20,
max_queued_bytes: 1024 * 1024 * 500, // 500 MB
max_queued_count: 2000,
max_data_transfer_rate: 1024.0 * 1024.0 * 10.0, // 10 MB/s
recent_backlog_count: 1500,
};
let metrics = collect_replication_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 9);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
let active = metrics.iter().find(|m| m.name == active_name);
assert!(active.is_some());
assert_eq!(active.map(|m| m.value), Some(10.0));
}
#[test]
fn test_collect_replication_metrics_default() {
let stats = ReplicationStats::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 9);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/request.rs
+174
View File
@@ -0,0 +1,174 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! API request metrics collector.
//!
//! Collects API request metrics including request counts, errors,
//! latency, and traffic statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::request::*;
/// API request statistics for a specific API endpoint.
#[derive(Debug, Clone, Default)]
pub struct ApiRequestStats {
/// API name (e.g., "GetObject", "PutObject")
pub name: String,
/// Request type (e.g., "s3", "admin")
pub req_type: String,
/// Number of requests currently in flight
pub in_flight: u64,
/// Total number of requests
pub total: u64,
/// Total number of errors (4xx + 5xx)
pub errors_total: u64,
/// Total number of 5xx errors
pub errors_5xx: u64,
/// Total number of 4xx errors
pub errors_4xx: u64,
/// Total number of canceled requests
pub canceled: u64,
/// TTFB distribution by bucket (le label)
pub ttfb_distribution: Vec<(String, f64)>,
/// Bytes sent
pub sent_bytes: u64,
/// Bytes received
pub recv_bytes: u64,
}
/// Collects API request metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for API request statistics.
pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
for stat in stats {
// In-flight requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 5xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 4xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Canceled requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// TTFB distribution (histogram buckets)
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
// Traffic metrics
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_MD, stat.sent_bytes as f64)
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_MD, stat.recv_bytes as f64)
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_request_metrics() {
let stats = vec![ApiRequestStats {
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
total: 10000,
errors_total: 50,
errors_5xx: 10,
errors_4xx: 40,
canceled: 5,
ttfb_distribution: vec![
("0.1".to_string(), 5000.0),
("0.5".to_string(), 8000.0),
("1.0".to_string(), 9500.0),
("+Inf".to_string(), 10000.0),
],
sent_bytes: 1024 * 1024 * 500, // 500 MB
recv_bytes: 1024 * 1024 * 100, // 100 MB
}];
let metrics = collect_request_metrics(&stats);
report_metrics(&metrics);
// 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
assert_eq!(metrics.len(), 12);
let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
let total = metrics.iter().find(|m| m.name == total_name);
assert!(total.is_some());
assert_eq!(total.map(|m| m.value), Some(10000.0));
let in_flight_name = API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name();
let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
assert!(in_flight.is_some());
assert_eq!(in_flight.map(|m| m.value), Some(10.0));
}
#[test]
fn test_collect_request_metrics_empty() {
let stats: Vec<ApiRequestStats> = vec![];
let metrics = collect_request_metrics(&stats);
assert!(metrics.is_empty());
}
}
crates/metrics/src/collectors/resource.rs
+27 -70
View File
@@ -16,17 +16,21 @@
//!
//! Collects system-level metrics for the RustFS process including
//! CPU usage, memory consumption, and process uptime.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::process_resource`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::process_resource::*;
/// Resource statistics for a RustFS process.
/// Resource statistics for metrics collection.
///
/// This struct encapsulates the resource usage data that can be
/// collected from the operating system for the current process.
/// This struct provides a decoupled interface for collecting resource metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct ResourceStats {
/// CPU usage as a percentage (0.0 to 100.0+)
/// CPU usage as a percentage (can exceed 100% on multi-core systems)
pub cpu_percent: f64,
/// Resident memory usage in bytes
pub memory_bytes: u64,
@@ -34,65 +38,16 @@ pub struct ResourceStats {
pub uptime_seconds: u64,
}
// Static metric definitions
const METRIC_CPU: &str = "rustfs_process_cpu_percent";
const METRIC_MEMORY: &str = "rustfs_process_memory_bytes";
const METRIC_UPTIME: &str = "rustfs_process_uptime_seconds";
const HELP_CPU: &str = "CPU usage of the RustFS process as a percentage";
const HELP_MEMORY: &str = "Resident memory usage of the RustFS process in bytes";
const HELP_UPTIME: &str = "Uptime of the RustFS process in seconds";
/// Number of metrics produced by this collector.
const METRIC_COUNT: usize = 3;
/// Collects system resource metrics from the provided statistics.
/// Collects resource metrics from the provided resource statistics.
///
/// # Metrics Produced
///
/// - `rustfs_process_cpu_percent`: CPU usage as a percentage
/// - `rustfs_process_memory_bytes`: Resident memory usage in bytes
/// - `rustfs_process_uptime_seconds`: Process uptime in seconds
///
/// # Arguments
///
/// * `stats` - Resource statistics for the current process
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_resource_metrics, ResourceStats};
///
/// let stats = ResourceStats {
/// cpu_percent: 25.5,
/// memory_bytes: 1024 * 1024 * 512, // 512 MB
/// uptime_seconds: 3600,
/// };
/// let metrics = collect_resource_metrics(&stats);
/// assert_eq!(metrics.len(), 3);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::process_resource` module.
/// Returns a vector of Prometheus metrics for resource statistics.
pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(METRIC_COUNT);
metrics.push(PrometheusMetric::new(METRIC_CPU, MetricType::Gauge, HELP_CPU, stats.cpu_percent));
metrics.push(PrometheusMetric::new(
METRIC_MEMORY,
MetricType::Gauge,
HELP_MEMORY,
stats.memory_bytes as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_UPTIME,
MetricType::Gauge,
HELP_UPTIME,
stats.uptime_seconds as f64,
));
metrics
vec![
PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent),
PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
]
}
#[cfg(test)]
@@ -114,19 +69,21 @@ mod tests {
assert_eq!(metrics.len(), 3);
// Verify CPU metric
let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 45.5);
assert!(cpu.is_some());
assert_eq!(cpu.map(|m| m.value), Some(45.5));
// Verify memory metric
let memory = metrics.iter().find(|m| m.name == METRIC_MEMORY);
let memory_metric_name = PROCESS_MEMORY_BYTES_MD.get_full_metric_name();
let memory = metrics
.iter()
.find(|m| m.name == memory_metric_name && m.value == (1024 * 1024 * 256) as f64);
assert!(memory.is_some());
assert_eq!(memory.map(|m| m.value), Some((1024 * 1024 * 256) as f64));
// Verify uptime metric
let uptime = metrics.iter().find(|m| m.name == METRIC_UPTIME);
let uptime_metric_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
let uptime = metrics.iter().find(|m| m.name == uptime_metric_name && m.value == 7200.0);
assert!(uptime.is_some());
assert_eq!(uptime.map(|m| m.value), Some(7200.0));
}
#[test]
@@ -155,9 +112,9 @@ mod tests {
let metrics = collect_resource_metrics(&stats);
report_metrics(&metrics);
let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 150.0);
assert!(cpu.is_some());
assert_eq!(cpu.map(|m| m.value), Some(150.0));
}
#[test]
crates/metrics/src/collectors/scanner.rs
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Scanner metrics collector.
//!
//! Collects background scanner metrics including bucket scans,
//! directory scans, and object scans.
use crate::format::PrometheusMetric;
use crate::metrics_type::scanner::{
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
};
/// Scanner statistics.
#[derive(Debug, Clone, Default)]
pub struct ScannerStats {
/// Number of bucket scans finished
pub bucket_scans_finished: u64,
/// Number of bucket scans started
pub bucket_scans_started: u64,
/// Number of directories scanned
pub directories_scanned: u64,
/// Number of objects scanned
pub objects_scanned: u64,
/// Number of object versions scanned
pub versions_scanned: u64,
/// Seconds since last scanner activity
pub last_activity_seconds: u64,
}
/// Collects scanner metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::scanner` module.
/// Returns a vector of Prometheus metrics for scanner statistics.
pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_scanner_metrics() {
let stats = ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
};
let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 6);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
let last_activity = metrics.iter().find(|m| m.value == 30.0);
assert!(last_activity.is_some());
}
#[test]
fn test_collect_scanner_metrics_default() {
let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 6);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/stats_collector.rs
+233
View File
@@ -0,0 +1,233 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Statistics collection functions for metrics.
//!
//! This module contains functions that collect statistics from various
//! RustFS internal sources (storage layer, bucket monitor, system info)
//! and convert them to the Stats structs used by collectors.
use crate::collectors::{BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats};
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use std::sync::OnceLock;
use std::time::Instant;
use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
use tracing::{instrument, warn};
/// Process start time for calculating uptime.
static PROCESS_START: OnceLock<Instant> = OnceLock::new();
/// Get the process start time, initializing it on first call.
#[inline]
fn get_process_start() -> &'static Instant {
PROCESS_START.get_or_init(Instant::now)
}
/// Collect cluster statistics from the storage layer.
#[instrument]
pub async fn collect_cluster_stats() -> ClusterStats {
let Some(store) = new_object_layer_fn() else {
return ClusterStats::default();
};
let storage_info = store.storage_info().await;
let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
// Get bucket and object counts from data usage info
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!("Failed to load data usage from backend: {}", e);
// Fall back to bucket list for buckets_count, objects_count stays 0
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|e| {
warn!("Failed to list buckets for cluster metrics: {}", e);
Vec::new()
});
(buckets.len() as u64, 0)
}
};
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
}
}
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_data_usage_from_backend(store.clone()).await {
Ok(info) => Some(info),
Err(e) => {
warn!("Failed to load data usage for bucket metrics: {}", e);
None
}
};
// List all buckets
let buckets = match store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
{
Ok(buckets) => buckets,
Err(e) => {
warn!("Failed to list buckets for bucket metrics: {}", e);
return Vec::new();
}
};
let mut stats = Vec::with_capacity(buckets.len());
for bucket in buckets {
if bucket.name.starts_with('.') {
continue;
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
// Get quota from bucket metadata
let quota_bytes = match get_quota_config(&bucket.name).await {
Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
Err(_) => 0, // No quota configured or error
};
stats.push(BucketStats {
name: bucket.name,
size_bytes,
objects_count,
quota_bytes,
});
}
stats
}
/// Collect bucket replication bandwidth stats from the global monitor.
pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
let Some(monitor) = get_global_bucket_monitor() else {
return Vec::new();
};
monitor
.get_report(|_| true)
.bucket_stats
.into_iter()
.map(|(opts, details)| {
let target_arn = opts.replication_arn;
let limit_bytes_per_sec = u64::try_from(details.limit_bytes_per_sec).unwrap_or_else(|_| {
warn!(
"Invalid bandwidth limit value for target {:?}: {}",
target_arn, details.limit_bytes_per_sec
);
0
});
BucketReplicationBandwidthStats {
bucket: opts.name,
target_arn,
limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
}
})
.collect()
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
let storage_info = store.storage_info().await;
storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
})
.collect()
}
/// Collect resource statistics for the current process.
///
/// Collects:
/// - Uptime: Calculated from process start time
/// - Memory: Process resident set size from sysinfo
/// - CPU: Process CPU usage percentage from sysinfo
#[inline]
pub fn collect_process_stats() -> ResourceStats {
let uptime_seconds = get_process_start().elapsed().as_secs();
// Use sysinfo for process metrics
let mut sys = System::new();
let pid = Pid::from_u32(std::process::id());
sys.refresh_processes_specifics(
ProcessesToUpdate::Some(&[pid]),
true,
ProcessRefreshKind::nothing().with_cpu().with_memory(),
);
if let Some(process) = sys.process(pid) {
ResourceStats {
cpu_percent: process.cpu_usage() as f64,
memory_bytes: process.memory(),
uptime_seconds,
}
} else {
// Fallback if process info unavailable
ResourceStats {
cpu_percent: 0.0,
memory_bytes: 0,
uptime_seconds,
}
}
}
crates/metrics/src/collectors/system_cpu.rs
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System CPU metrics collector.
//!
//! Collects CPU-related metrics including load average, idle time,
//! I/O wait, and CPU usage percentages.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_cpu`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_cpu::*;
/// CPU statistics for a node.
#[derive(Debug, Clone, Default)]
pub struct CpuStats {
/// Average CPU idle time (percentage, 0-100)
pub avg_idle: f64,
/// Average CPU I/O wait time (percentage, 0-100)
pub avg_iowait: f64,
/// CPU load average over 1 minute
pub load_avg: f64,
/// CPU load average as percentage
pub load_avg_perc: f64,
/// CPU nice time (percentage, 0-100)
pub nice: f64,
/// CPU steal time (percentage, 0-100)
pub steal: f64,
/// CPU system time (percentage, 0-100)
pub system: f64,
/// CPU user time (percentage, 0-100)
pub user: f64,
}
/// Collects CPU metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_cpu` module.
/// Returns a vector of Prometheus metrics for CPU statistics.
pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle),
PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IOWAIT_MD, stats.avg_iowait),
PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg),
PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc),
PrometheusMetric::from_descriptor(&SYS_CPU_NICE_MD, stats.nice),
PrometheusMetric::from_descriptor(&SYS_CPU_STEAL_MD, stats.steal),
PrometheusMetric::from_descriptor(&SYS_CPU_SYSTEM_MD, stats.system),
PrometheusMetric::from_descriptor(&SYS_CPU_USER_MD, stats.user),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_cpu_metrics() {
let stats = CpuStats {
avg_idle: 75.5,
avg_iowait: 2.3,
load_avg: 1.5,
load_avg_perc: 37.5,
nice: 0.5,
steal: 0.1,
system: 10.0,
user: 15.0,
};
let metrics = collect_cpu_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
// Verify that metric names are properly generated from descriptors
assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_cpu_")));
}
#[test]
fn test_collect_cpu_metrics_default() {
let stats = CpuStats::default();
let metrics = collect_cpu_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/system_drive.rs
+228
View File
@@ -0,0 +1,228 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System drive metrics collector.
//!
//! Collects detailed drive/disk metrics including capacity, I/O statistics,
//! error counts, and health status.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_drive::*;
/// Detailed drive statistics for a single drive.
#[derive(Debug, Clone, Default)]
pub struct DriveDetailedStats {
/// Server identifier (e.g., "node1:9000")
pub server: String,
/// Drive path (e.g., "/data/disk1")
pub drive: String,
/// Total capacity in bytes
pub total_bytes: u64,
/// Used capacity in bytes
pub used_bytes: u64,
/// Free capacity in bytes
pub free_bytes: u64,
/// Used inodes
pub used_inodes: u64,
/// Free inodes
pub free_inodes: u64,
/// Total inodes
pub total_inodes: u64,
/// Total timeout errors
pub timeout_errors_total: u64,
/// Total I/O errors
pub io_errors_total: u64,
/// Total availability errors
pub availability_errors_total: u64,
/// Number of I/O operations waiting
pub waiting_io: u64,
/// API latency in microseconds
pub api_latency_micros: u64,
/// Health status (1=healthy, 0=unhealthy)
pub health: u8,
/// Reads per second
pub reads_per_sec: f64,
/// Kilobytes read per second
pub reads_kb_per_sec: f64,
/// Average read await time
pub reads_await: f64,
/// Writes per second
pub writes_per_sec: f64,
/// Kilobytes written per second
pub writes_kb_per_sec: f64,
/// Average write await time
pub writes_await: f64,
/// Percentage utilization
pub perc_util: f64,
}
/// Aggregate drive count statistics.
#[derive(Debug, Clone, Default)]
pub struct DriveCountStats {
/// Number of offline Drives
pub offline_count: u64,
/// Number of online drives
pub online_count: u64,
/// Total number of drives
pub total_count: u64,
}
/// Collects detailed drive metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for each drive.
pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
fn push_drive_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::MetricDescriptor,
value: f64,
server_label: &str,
drive_label: &str,
) {
metrics.push(
PrometheusMetric::from_descriptor(descriptor, value)
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
.with_label_owned(SERVER_LABEL, server_label.to_string()),
);
}
let mut metrics = Vec::with_capacity(stats.len() * 19);
for stat in stats {
let server_label = stat.server.as_str();
let drive_label = stat.drive.as_str();
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_TIMEOUT_ERRORS_MD,
stat.timeout_errors_total as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, stat.io_errors_total as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_AVAILABILITY_ERRORS_MD,
stat.availability_errors_total as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, stat.waiting_io as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_API_LATENCY_MD,
stat.api_latency_micros as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, stat.reads_per_sec, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, stat.reads_kb_per_sec, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, stat.reads_await, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, stat.writes_per_sec, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_WRITES_KB_PER_SEC_MD,
stat.writes_kb_per_sec,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, stat.writes_await, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, stat.perc_util, server_label, drive_label);
}
metrics
}
/// Collects drive count metrics (offline, online, total).
///
/// Returns a vector of Prometheus metrics for drive counts.
pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&DRIVE_OFFLINE_COUNT_MD, stats.offline_count as f64),
PrometheusMetric::from_descriptor(&DRIVE_ONLINE_COUNT_MD, stats.online_count as f64),
PrometheusMetric::from_descriptor(&DRIVE_COUNT_MD, stats.total_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_drive_detailed_metrics() {
let stats = vec![DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
used_inodes: 100000,
free_inodes: 900000,
total_inodes: 1000000,
timeout_errors_total: 5,
io_errors_total: 10,
availability_errors_total: 2,
waiting_io: 3,
api_latency_micros: 1500,
health: 1,
reads_per_sec: 100.0,
reads_kb_per_sec: 1024.0,
reads_await: 5.5,
writes_per_sec: 50.0,
writes_kb_per_sec: 512.0,
writes_await: 10.2,
perc_util: 75.5,
}];
let metrics = collect_drive_detailed_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 19);
// Verify total bytes metric
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
assert!(total_bytes.is_some());
assert_eq!(total_bytes.map(|m| m.value), Some(1024.0 * 1024.0 * 1024.0 * 100.0));
}
#[test]
fn test_collect_drive_count_metrics() {
let stats = DriveCountStats {
offline_count: 2,
online_count: 8,
total_count: 10,
};
let metrics = collect_drive_count_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 3);
// Verify offline count
let offline_name = DRIVE_OFFLINE_COUNT_MD.get_full_metric_name();
let offline = metrics.iter().find(|m| m.name == offline_name);
assert!(offline.is_some());
assert_eq!(offline.map(|m| m.value), Some(2.0));
}
}
crates/metrics/src/collectors/system_memory.rs
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System memory metrics collector.
//!
//! Collects memory-related metrics including total, used, free,
//! buffers, cache, shared, and available memory.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_memory`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_memory::*;
/// Memory statistics for a node.
#[derive(Debug, Clone, Default)]
pub struct MemoryStats {
/// Total memory in bytes
pub total: u64,
/// Used memory in bytes
pub used: u64,
/// Used memory percentage (0-100)
pub used_perc: f64,
/// Free memory in bytes
pub free: u64,
/// Buffer memory in bytes
pub buffers: u64,
/// Cache memory in bytes
pub cache: u64,
/// Shared memory in bytes
pub shared: u64,
/// Available memory in bytes
pub available: u64,
}
/// Collects memory metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_memory` module.
/// Returns a vector of Prometheus metrics for memory statistics.
pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64),
PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64),
PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc),
PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64),
PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64),
PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64),
PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64),
PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_memory_metrics() {
let stats = MemoryStats {
total: 16 * 1024 * 1024 * 1024, // 16 GB
used: 8 * 1024 * 1024 * 1024, // 8 GB
used_perc: 50.0,
free: 4 * 1024 * 1024 * 1024, // 4 GB
buffers: 1024 * 1024 * 512, // 512 MB
cache: 2 * 1024 * 1024 * 1024, // 2 GB
shared: 1024 * 1024 * 256, // 256 MB
available: 6 * 1024 * 1024 * 1024, // 6 GB
};
let metrics = collect_memory_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_memory_")));
}
#[test]
fn test_collect_memory_metrics_default() {
let stats = MemoryStats::default();
let metrics = collect_memory_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/system_network.rs
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System network metrics collector.
//!
//! Collects internode network metrics including errors, dial times,
//! and bytes sent/received.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_network`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_network::*;
/// Network statistics for internode communication.
#[derive(Debug, Clone, Default)]
pub struct NetworkStats {
/// Total number of failed internode calls
pub internode_errors_total: u64,
/// Total number of TCP dial timeouts and errors
pub internode_dial_errors_total: u64,
/// Average dial time in nanoseconds
pub internode_dial_avg_time_nanos: u64,
/// Total bytes sent to other nodes
pub internode_sent_bytes_total: u64,
/// Total bytes received from other nodes
pub internode_recv_bytes_total: u64,
}
/// Collects network metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_network` module.
/// Returns a vector of Prometheus metrics for network statistics.
pub fn collect_network_metrics(stats: &NetworkStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64),
PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_network_metrics() {
let stats = NetworkStats {
internode_errors_total: 10,
internode_dial_errors_total: 5,
internode_dial_avg_time_nanos: 1_500_000, // 1.5ms
internode_sent_bytes_total: 1024 * 1024 * 100, // 100 MB
internode_recv_bytes_total: 1024 * 1024 * 200, // 200 MB
};
let metrics = collect_network_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 5);
assert!(metrics.iter().all(|m| m.name.contains("internode")));
}
#[test]
fn test_collect_network_metrics_default() {
let stats = NetworkStats::default();
let metrics = collect_network_metrics(&stats);
assert_eq!(metrics.len(), 5);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/collectors/system_process.rs
+145
View File
@@ -0,0 +1,145 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System process metrics collector.
//!
//! Collects process-level metrics including file descriptors, memory,
//! syscalls, and runtime statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_process::*;
/// Process statistics for the RustFS server process.
#[derive(Debug, Clone, Default)]
pub struct ProcessStats {
/// Total read locks held
pub locks_read_total: u64,
/// Total write locks held
pub locks_write_total: u64,
/// Total CPU time in seconds
pub cpu_total_seconds: f64,
/// Total number of async tasks (goroutines equivalent)
pub go_routine_total: u64,
/// Total bytes read via read syscalls (rchar)
pub io_rchar_bytes: u64,
/// Total bytes actually read from storage
pub io_read_bytes: u64,
/// Total bytes written via write syscalls (wchar)
pub io_wchar_bytes: u64,
/// Total bytes actually written to storage
pub io_write_bytes: u64,
/// Process start time in seconds since Unix epoch
pub start_time_seconds: u64,
/// Process uptime in seconds
pub uptime_seconds: u64,
/// File descriptor limit
pub file_descriptor_limit_total: u64,
/// Open file descriptors count
pub file_descriptor_open_total: u64,
/// Total read syscalls
pub syscall_read_total: u64,
/// Total write syscalls
pub syscall_write_total: u64,
/// Resident memory size in bytes
pub resident_memory_bytes: u64,
/// Virtual memory size in bytes
pub virtual_memory_bytes: u64,
/// Maximum virtual memory size in bytes
pub virtual_memory_max_bytes: u64,
}
/// Collects process metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for process statistics.
pub fn collect_process_metrics(stats: &ProcessStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds),
PrometheusMetric::from_descriptor(&PROCESS_GO_ROUTINE_TOTAL_MD, stats.go_routine_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_RCHAR_BYTES_MD, stats.io_rchar_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_READ_BYTES_MD, stats.io_read_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_WCHAR_BYTES_MD, stats.io_wchar_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_WRITE_BYTES_MD, stats.io_write_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_START_TIME_SECONDS_MD, stats.start_time_seconds as f64),
PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD, stats.file_descriptor_limit_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD, stats.file_descriptor_open_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_READ_TOTAL_MD, stats.syscall_read_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_WRITE_TOTAL_MD, stats.syscall_write_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_process_metrics() {
let stats = ProcessStats {
locks_read_total: 100,
locks_write_total: 50,
cpu_total_seconds: 1234.56,
go_routine_total: 200,
io_rchar_bytes: 1024 * 1024 * 500,
io_read_bytes: 1024 * 1024 * 400,
io_wchar_bytes: 1024 * 1024 * 300,
io_write_bytes: 1024 * 1024 * 250,
start_time_seconds: 1700000000,
uptime_seconds: 86400,
file_descriptor_limit_total: 65536,
file_descriptor_open_total: 1500,
syscall_read_total: 100000,
syscall_write_total: 50000,
resident_memory_bytes: 1024 * 1024 * 512,
virtual_memory_bytes: 1024 * 1024 * 1024,
virtual_memory_max_bytes: 1024 * 1024 * 2048,
};
let metrics = collect_process_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 17);
// Verify uptime
let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
let uptime = metrics.iter().find(|m| m.name == uptime_name);
assert!(uptime.is_some());
assert_eq!(uptime.map(|m| m.value), Some(86400.0));
// Verify file descriptors
let fd_open_name = PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD.get_full_metric_name();
let fd_open = metrics.iter().find(|m| m.name == fd_open_name);
assert!(fd_open.is_some());
assert_eq!(fd_open.map(|m| m.value), Some(1500.0));
}
#[test]
fn test_collect_process_metrics_default() {
let stats = ProcessStats::default();
let metrics = collect_process_metrics(&stats);
assert_eq!(metrics.len(), 17);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
crates/metrics/src/format.rs
+71 -12
View File
@@ -20,21 +20,50 @@
use crate::MetricType;
use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge};
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
static NAME_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
static HELP_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
fn intern_string(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
let cache = cache.get_or_init(Default::default);
let mut cache = match cache.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if let Some(existing) = cache.get(value) {
existing
} else {
let value = Box::leak(value.to_string().into_boxed_str());
cache.insert(value.to_string(), value);
value
}
}
fn into_static_str(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
intern_string(cache, value)
}
/// Report metrics using the `metrics` crate.
///
/// This function iterates over the provided metrics and reports them using
/// the `metrics` crate's API. This allows integration with various metrics
/// exporters (e.g., Prometheus) that are configured globally.
///
pub fn report_metrics(metrics: &[PrometheusMetric]) {
for metric in metrics {
let name = into_static_str(&NAME_CACHE, &metric.name);
let help = into_static_str(&HELP_CACHE, &metric.help);
// Register metric description (help text)
// Note: In a real-world scenario, descriptions should ideally be registered once at startup.
// However, the `metrics` crate handles duplicate registrations gracefully.
match metric.metric_type {
MetricType::Counter => describe_counter!(metric.name, metric.help),
MetricType::Gauge => describe_gauge!(metric.name, metric.help),
MetricType::Histogram => describe_histogram!(metric.name, metric.help),
MetricType::Counter => describe_counter!(name, help),
MetricType::Gauge => describe_gauge!(name, help),
MetricType::Histogram => describe_histogram!(name, help),
}
// Convert labels to the format expected by `metrics` crate
@@ -50,15 +79,15 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) {
//
// Since `metrics` 0.21+, `Counter` has an `absolute` method which sets the counter to a specific value.
// This is useful for mirroring an external counter.
let counter = counter!(metric.name, &labels);
let counter = counter!(name, &labels);
counter.absolute(metric.value as u64);
}
MetricType::Gauge => {
let gauge = gauge!(metric.name, &labels);
let gauge = gauge!(name, &labels);
gauge.set(metric.value);
}
MetricType::Histogram => {
let histogram = metrics::histogram!(metric.name, &labels);
let histogram = metrics::histogram!(name, &labels);
histogram.record(metric.value);
}
}
@@ -67,17 +96,17 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) {
/// A single Prometheus metric with labels and value.
///
/// This struct is optimized for performance by using `&'static str` for
/// the name and help text, which are typically compile-time constants.
/// This struct is optimized for performance by using `Cow<'static, str>` for
/// the name and help text, which allows both static strings and owned strings.
/// Labels use `Cow<'static, str>` to avoid allocations when possible.
#[derive(Debug, Clone)]
pub struct PrometheusMetric {
/// The metric name (e.g., "http_requests_total").
pub name: &'static str,
pub name: Cow<'static, str>,
/// The type of this metric (counter, gauge, or histogram).
pub metric_type: MetricType,
/// Human-readable description shown in Prometheus UI.
pub help: &'static str,
pub help: Cow<'static, str>,
/// Key-value label pairs for this metric instance.
/// Uses Cow to avoid allocations for static label keys.
pub labels: Vec<(&'static str, Cow<'static, str>)>,
@@ -92,9 +121,39 @@ impl PrometheusMetric {
#[inline]
pub const fn new(name: &'static str, metric_type: MetricType, help: &'static str, value: f64) -> Self {
Self {
name,
name: Cow::Borrowed(name),
metric_type,
help,
help: Cow::Borrowed(help),
labels: Vec::new(),
value,
}
}
/// Creates a new metric with owned strings for name and help.
///
/// Use this when the metric name or help text is dynamically generated.
#[inline]
pub fn new_owned(name: String, metric_type: MetricType, help: String, value: f64) -> Self {
Self {
name: Cow::Owned(name),
metric_type,
help: Cow::Owned(help),
labels: Vec::new(),
value,
}
}
/// Creates a new metric from a MetricDescriptor.
///
/// This is the recommended way to create metrics when using MetricDescriptor
/// from the metrics_type module.
#[inline]
pub fn from_descriptor(descriptor: &crate::MetricDescriptor, value: f64) -> Self {
let help = intern_string(&HELP_CACHE, &descriptor.help);
Self {
name: Cow::Owned(descriptor.get_full_metric_name()),
metric_type: descriptor.metric_type,
help: Cow::Borrowed(help),
labels: Vec::new(),
value,
}
crates/metrics/src/metrics_type/cluster.rs
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
/// Total raw storage capacity across all disks in bytes
pub static CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_raw_total_bytes".to_string()),
"Total raw storage capacity in bytes across all disks",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total usable storage capacity in bytes (accounting for erasure coding)
pub static CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_usable_total_bytes".to_string()),
"Total usable storage capacity in bytes (accounting for erasure coding)",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total used storage capacity in bytes
pub static CLUSTER_CAPACITY_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_used_bytes".to_string()),
"Total used storage capacity in bytes",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total free storage capacity in bytes
pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_free_bytes".to_string()),
"Total free storage capacity in bytes",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total number of objects in the cluster
pub static CLUSTER_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("objects_total".to_string()),
"Total number of objects in the cluster",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total number of buckets in the cluster
pub static CLUSTER_BUCKETS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("buckets_total".to_string()),
"Total number of buckets in the cluster",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
crates/metrics/src/metrics_type/cluster_erasure_set.rs
+27
View File
@@ -22,6 +22,33 @@ pub const POOL_ID_L: &str = "pool_id";
/// The label for the pool ID
pub const SET_ID_L: &str = "set_id";
pub static ERASURE_SET_SIZE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetSize,
"Total number of drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetParity,
"Parity drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_DATA_SHARDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetDataShards,
"Data shards in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_OVERALL_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetOverallWriteQuorum,
crates/metrics/src/metrics_type/entry/metric_name.rs
+6
View File
@@ -183,6 +183,9 @@ pub enum MetricName {
ConfigStandardParity,
// Erasure coding set related metrics
ErasureSetSize,
ErasureSetParity,
ErasureSetDataShards,
ErasureSetOverallWriteQuorum,
ErasureSetOverallHealth,
ErasureSetReadQuorum,
@@ -502,6 +505,9 @@ impl MetricName {
Self::ConfigStandardParity => "standard_parity".to_string(),
// Erasure coding set related metrics
Self::ErasureSetSize => "size".to_string(),
Self::ErasureSetParity => "parity".to_string(),
Self::ErasureSetDataShards => "data_shards".to_string(),
Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(),
Self::ErasureSetOverallHealth => "overall_health".to_string(),
Self::ErasureSetReadQuorum => "read_quorum".to_string(),
crates/metrics/src/metrics_type/mod.rs
+4
View File
@@ -15,6 +15,7 @@
pub mod audit;
pub mod bucket;
pub mod bucket_replication;
pub mod cluster;
pub mod cluster_config;
pub mod cluster_erasure_set;
pub mod cluster_health;
@@ -24,6 +25,9 @@ pub mod cluster_usage;
pub mod entry;
pub mod ilm;
pub mod logger_webhook;
pub mod node_bucket;
pub mod node_disk;
pub mod process_resource;
pub mod replication;
pub mod request;
pub mod scanner;
crates/metrics/src/metrics_type/node_bucket.rs
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
const BUCKET_LABEL: &str = "bucket";
/// Total bytes used by the bucket
pub static BUCKET_USAGE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("usage_bytes".to_string()),
"Total bytes used by the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
/// Total number of objects in the bucket
pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("objects_total".to_string()),
"Total number of objects in the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
/// Quota limit in bytes for the bucket
pub static BUCKET_QUOTA_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("quota_bytes".to_string()),
"Quota limit in bytes for the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
crates/metrics/src/metrics_type/node_disk.rs
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
use std::sync::LazyLock;
const SERVER_LABEL: &str = "server";
const DRIVE_LABEL: &str = "drive";
/// Total disk capacity in bytes
pub static NODE_DISK_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_total_bytes".to_string()),
"Total disk capacity in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
/// Used disk space in bytes
pub static NODE_DISK_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_used_bytes".to_string()),
"Used disk space in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
/// Free disk space in bytes
pub static NODE_DISK_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_free_bytes".to_string()),
"Free disk space in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
crates/metrics/src/metrics_type/process_resource.rs
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
use std::sync::LazyLock;
/// CPU usage of the RustFS process as a percentage
pub static PROCESS_CPU_PERCENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cpu_percent".to_string()),
"CPU usage of the RustFS process as a percentage",
&[],
MetricSubsystem::new("/process"),
)
});
/// Resident memory usage of the RustFS process in bytes
pub static PROCESS_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("memory_bytes".to_string()),
"Resident memory usage of the RustFS process in bytes",
&[],
MetricSubsystem::new("/process"),
)
});
/// Uptime of the RustFS process in seconds
pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("uptime_seconds".to_string()),
"Uptime of the RustFS process in seconds",
&[],
MetricSubsystem::new("/process"),
)
});
crates/metrics/src/metrics_type/request.rs
+6
View File
@@ -16,6 +16,12 @@
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
/// name label
pub const NAME_LABEL: &str = "name";
/// type label
pub const TYPE_LABEL: &str = "type";
/// le label (for histogram buckets)
pub const LE_LABEL: &str = "le";
pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
crates/metrics/src/metrics_type/system_drive.rs
+2
View File
@@ -19,6 +19,8 @@ use std::sync::LazyLock;
/// drive related labels
pub const DRIVE_LABEL: &str = "drive";
/// server label
pub const SERVER_LABEL: &str = "server";
/// pool index label
pub const POOL_INDEX_LABEL: &str = "pool_index";
/// set index label