Compare commits

...

6 Commits

Author SHA1 Message Date
Trent Houliston e1a278aaf8 fix(iam): preserve trailing slash in OIDC issuer URL (#2228) 2026-03-19 13:52:59 +08:00
weisd 35e1f28f23 fix: support legacy bucket metadata decoding (#2227) 2026-03-19 12:43:53 +08:00
weisd c6715259b1 fix(ecstore): keep bucket created time when metadata is epoch (#2220)
Co-authored-by: momoda693 <momoda693@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-03-19 01:02:34 +08:00
weisd df8f0edaea fix(ecstore): support legacy pool meta decoding (#2218)
Co-authored-by: momoda693 <momoda693@gmail.com>
2026-03-19 00:20:24 +08:00
houseme a4411a24d6 refactor(metrics): modularize collectors and add comprehensive metrics support (#2208)
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
2026-03-18 23:52:42 +08:00
houseme 4a807d80e3 build(deps): bump the dependencies group with 2 updates (#2211) 2026-03-18 21:39:01 +08:00
44 changed files with 3481 additions and 595 deletions
Generated
+4 -4
View File
@@ -717,9 +717,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.126.0"
version = "1.127.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7878050a2321d215eec9db8be09f8db59418b53860ae86cc7042b4094d6cb2bb"
checksum = "151783f64e0dcddeb4965d08e36c276b4400a46caa88805a2e36d497deaf031a"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -1012,9 +1012,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.4.6"
version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2b1117b3b2bbe166d11199b540ceed0d0f7676e36e7b962b5a437a9971eac75"
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
dependencies = [
"base64-simd",
"bytes",
+2 -2
View File
@@ -186,9 +186,9 @@ atoi = "2.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.15" }
aws-credential-types = { version = "1.2.14" }
aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-s3 = { version = "1.127.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
aws-smithy-types = { version = "1.4.6" }
aws-smithy-types = { version = "1.4.7" }
backtrace = "0.3.76"
base64 = "0.22.1"
base64-simd = "0.8.0"
+180 -18
View File
@@ -33,7 +33,7 @@ use serde::Serializer;
use std::collections::HashMap;
use std::io::{Read, Write};
use std::sync::Arc;
use time::OffsetDateTime;
use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset};
use tracing::error;
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
@@ -43,19 +43,177 @@ fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
Ok(String::from_utf8(buf)?)
}
fn read_msgp_bool<R: Read>(rd: &mut R) -> Result<bool> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::True => Ok(true),
rmp::Marker::False => Ok(false),
rmp::Marker::FixPos(v) => Ok(v != 0),
rmp::Marker::U8 => Ok(read_u8(rd)? != 0),
rmp::Marker::U16 => Ok(read_u16_raw(rd)? != 0),
rmp::Marker::U32 => Ok(read_u32_raw(rd)? != 0),
rmp::Marker::U64 => Ok(read_u64_raw(rd)? != 0),
rmp::Marker::I8 => Ok(read_i8_raw(rd)? != 0),
rmp::Marker::I16 => Ok(read_i16_raw(rd)? != 0),
rmp::Marker::I32 => Ok(read_i32_raw(rd)? != 0),
rmp::Marker::I64 => Ok(read_i64_raw(rd)? != 0),
rmp::Marker::FixNeg(v) => Ok(v != 0),
_ => Err(Error::other(format!("expected bool or int-like bool, got marker: {marker:?}"))),
}
}
fn read_msgp_time_value<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH),
rmp::Marker::Ext8 => read_msgp_ext8_time(rd),
rmp::Marker::FixArray(len) => read_msgp_legacy_compact_time(rd, u32::from(len)),
rmp::Marker::Array16 => {
let len = read_u16_raw(rd)?;
read_msgp_legacy_compact_time(rd, u32::from(len))
}
rmp::Marker::Array32 => {
let len = read_u32_raw(rd)?;
read_msgp_legacy_compact_time(rd, len)
}
rmp::Marker::Bin8 => {
let len = usize::from(read_u8(rd)?);
read_msgp_time_value_from_embedded_bin(rd, len)
}
rmp::Marker::Bin16 => {
let len = usize::from(read_u16_raw(rd)?);
read_msgp_time_value_from_embedded_bin(rd, len)
}
rmp::Marker::Bin32 => {
let len = read_u32_raw(rd)? as usize;
read_msgp_time_value_from_embedded_bin(rd, len)
}
_ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))),
}
}
fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
let len = rmp::decode::read_bin_len(rd)? as usize;
fn read_u8<R: Read>(rd: &mut R) -> Result<u8> {
let mut buf = [0u8; 1];
rd.read_exact(&mut buf)?;
Ok(buf[0])
}
fn read_u16_raw<R: Read>(rd: &mut R) -> Result<u16> {
let mut buf = [0u8; 2];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_u16(&buf))
}
fn read_u32_raw<R: Read>(rd: &mut R) -> Result<u32> {
let mut buf = [0u8; 4];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_u32(&buf))
}
fn read_u64_raw<R: Read>(rd: &mut R) -> Result<u64> {
let mut buf = [0u8; 8];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_u64(&buf))
}
fn read_i8_raw<R: Read>(rd: &mut R) -> Result<i8> {
Ok(read_u8(rd)? as i8)
}
fn read_i16_raw<R: Read>(rd: &mut R) -> Result<i16> {
let mut buf = [0u8; 2];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_i16(&buf))
}
fn read_i32_raw<R: Read>(rd: &mut R) -> Result<i32> {
let mut buf = [0u8; 4];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_i32(&buf))
}
fn read_i64_raw<R: Read>(rd: &mut R) -> Result<i64> {
let mut buf = [0u8; 8];
rd.read_exact(&mut buf)?;
Ok(BigEndian::read_i64(&buf))
}
fn read_msgp_time_value_from_embedded_bin<R: Read>(rd: &mut R, len: usize) -> Result<OffsetDateTime> {
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
let mut cur = std::io::Cursor::new(buf);
read_msgp_time_value(&mut cur)
}
fn read_msgp_legacy_compact_time<R: Read>(rd: &mut R, len: u32) -> Result<OffsetDateTime> {
if len != 9 {
return Err(Error::other(format!("invalid legacy compact time len: {len}")));
}
let year: i32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let ordinal: u16 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let hour: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let minute: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let second: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let nanosecond: u32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let offset_hour: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let offset_minute: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let offset_second: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let date =
Date::from_ordinal_date(year, ordinal).map_err(|e| Error::other(format!("invalid legacy compact time date: {e}")))?;
let time = CivilTime::from_hms_nano(hour, minute, second, nanosecond)
.map_err(|e| Error::other(format!("invalid legacy compact time time: {e}")))?;
let offset = UtcOffset::from_hms(offset_hour, offset_minute, offset_second)
.map_err(|e| Error::other(format!("invalid legacy compact time offset: {e}")))?;
Ok(PrimitiveDateTime::new(date, time)
.assume_offset(offset)
.to_offset(UtcOffset::UTC))
}
fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(Vec::new()),
rmp::Marker::Bin8 => {
let len = usize::from(read_u8(rd)?);
read_exact_bytes(rd, len)
}
rmp::Marker::Bin16 => {
let len = usize::from(read_u16_raw(rd)?);
read_exact_bytes(rd, len)
}
rmp::Marker::Bin32 => {
let len = read_u32_raw(rd)? as usize;
read_exact_bytes(rd, len)
}
rmp::Marker::FixArray(len) => read_msgp_legacy_byte_array(rd, u32::from(len)),
rmp::Marker::Array16 => {
let len = read_u16_raw(rd)?;
read_msgp_legacy_byte_array(rd, u32::from(len))
}
rmp::Marker::Array32 => {
let len = read_u32_raw(rd)?;
read_msgp_legacy_byte_array(rd, len)
}
_ => Err(Error::other(format!("expected bin or byte array, got marker: {marker:?}"))),
}
}
fn read_exact_bytes<R: Read>(rd: &mut R, len: usize) -> Result<Vec<u8>> {
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(buf)
}
fn read_msgp_legacy_byte_array<R: Read>(rd: &mut R, len: u32) -> Result<Vec<u8>> {
let mut buf = Vec::with_capacity(len as usize);
for _ in 0..len {
let value: i64 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let byte = u8::try_from(value).map_err(|_| Error::other(format!("byte value out of range: {value}")))?;
buf.push(byte);
}
Ok(buf)
}
@@ -227,18 +385,20 @@ impl BucketMetadata {
match key.as_str() {
"Name" => self.name = read_msgp_str(rd)?,
"Created" => self.created = read_msgp_time_value(rd)?,
"LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?,
"PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?,
"NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?,
"LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
"ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?,
"VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?,
"EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?,
"TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?,
"QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?,
"ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?,
"BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
"BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?,
"LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?,
"PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
"NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
"LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
"ObjectLockConfigXML" | "ObjectLockConfigXml" => self.object_lock_config_xml = read_msgp_bin(rd)?,
"VersioningConfigXML" | "VersioningConfigXml" => self.versioning_config_xml = read_msgp_bin(rd)?,
"EncryptionConfigXML" | "EncryptionConfigXml" => self.encryption_config_xml = read_msgp_bin(rd)?,
"TaggingConfigXML" | "TaggingConfigXml" => self.tagging_config_xml = read_msgp_bin(rd)?,
"QuotaConfigJSON" | "QuotaConfigJson" => self.quota_config_json = read_msgp_bin(rd)?,
"ReplicationConfigXML" | "ReplicationConfigXml" => self.replication_config_xml = read_msgp_bin(rd)?,
"BucketTargetsConfigJSON" | "BucketTargetsConfigJson" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
"BucketTargetsConfigMetaJSON" | "BucketTargetsConfigMetaJson" => {
self.bucket_targets_config_meta_json = read_msgp_bin(rd)?
}
"PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?,
"ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?,
"EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?,
@@ -250,9 +410,11 @@ impl BucketMetadata {
"NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
"CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?,
"PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?,
"BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
"CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?,
"PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => {
self.public_access_block_config_xml = read_msgp_bin(rd)?
}
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
+137
View File
@@ -98,6 +98,143 @@ async fn unmarshal_test_bucket_metadata() {
assert!(bm.bucket_acl_config_json.is_empty());
}
#[test]
fn unmarshal_legacy_compact_time_bucket_metadata() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"83",
"a44e616d65",
"a474657374",
"a743726561746564",
"99cd07e9cd01100c1021ce2026b1fa000000",
"ab4c6f636b456e61626c6564",
"c2"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy compact time should decode");
assert_eq!(bm.name, "test");
assert_eq!(bm.created.unix_timestamp(), 1759148193);
assert_eq!(bm.created.nanosecond(), 539406842);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_bin_wrapped_ext_time_bucket_metadata() {
use faster_hex::hex_decode;
let wrapped_hex = concat!(
"83",
"a44e616d65",
"a464616461",
"a743726561746564",
"c40fc70c05fffffff1886e090000000000",
"ab4c6f636b456e61626c6564",
"c2"
);
let mut bytes = vec![0u8; wrapped_hex.len() / 2];
hex_decode(wrapped_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("bin-wrapped ext time should decode");
assert_eq!(bm.name, "dada");
assert_eq!(bm.created.unix_timestamp(), -62135596800);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_legacy_rmp_serde_field_aliases_and_byte_arrays() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"85",
"a44e616d65",
"a474657374",
"a743726561746564",
"99cd07e9cd01100c1021ce2026b1fa000000",
"ab4c6f636b456e61626c6564",
"c2",
"b0506f6c696379436f6e6669674a736f6e",
"93010203",
"bb4275636b657454617267657473436f6e6669674d6574614a736f6e",
"920405"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy field aliases and byte arrays should decode");
assert_eq!(bm.name, "test");
assert_eq!(bm.created.unix_timestamp(), 1759148193);
assert_eq!(bm.created.nanosecond(), 539406842);
assert_eq!(bm.policy_config_json, vec![1, 2, 3]);
assert_eq!(bm.bucket_targets_config_meta_json, vec![4, 5]);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_legacy_bin16_and_array16_bucket_metadata() {
let policy = vec![b'x'; 257];
let targets_meta: Vec<u8> = (0u8..=16).collect();
let mut bytes = Vec::new();
rmp::encode::write_map_len(&mut bytes, 5).unwrap();
rmp::encode::write_str(&mut bytes, "Name").unwrap();
rmp::encode::write_str(&mut bytes, "test-bucket").unwrap();
rmp::encode::write_str(&mut bytes, "Created").unwrap();
bytes.extend_from_slice(&[
0xc7, 0x0c, 0x05, 0x00, 0x00, 0x00, 0x00, 0x65, 0x92, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
]);
rmp::encode::write_str(&mut bytes, "LockEnabled").unwrap();
bytes.push(0x01);
rmp::encode::write_str(&mut bytes, "PolicyConfigJson").unwrap();
rmp::encode::write_bin(&mut bytes, &policy).unwrap();
rmp::encode::write_str(&mut bytes, "BucketTargetsConfigMetaJson").unwrap();
rmp::encode::write_array_len(&mut bytes, targets_meta.len() as u32).unwrap();
for byte in &targets_meta {
rmp::encode::write_uint(&mut bytes, u64::from(*byte)).unwrap();
}
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy bin16 and array16 should decode");
assert_eq!(bm.name, "test-bucket");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert!(bm.lock_enabled);
assert_eq!(bm.policy_config_json, policy);
assert_eq!(bm.bucket_targets_config_meta_json, targets_meta);
}
#[test]
fn unmarshal_legacy_numeric_bool_bucket_metadata() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"83",
"a44e616d65",
"ab746573742d6275636b6574",
"a743726561746564",
"c70c05000000006592008000000000",
"ab4c6f636b456e61626c6564",
"01"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy numeric bool should decode");
assert_eq!(bm.name, "test-bucket");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert_eq!(bm.created.nanosecond(), 0);
assert!(bm.lock_enabled);
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
+77 -4
View File
@@ -46,7 +46,9 @@ use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use bytes::Bytes;
use futures::future::BoxFuture;
use http::HeaderMap;
use rmp_serde::{Deserializer, Serializer};
#[cfg(test)]
use rmp_serde::Deserializer;
use rmp_serde::Serializer;
use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
@@ -235,6 +237,22 @@ impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
}
impl PoolMeta {
fn decode_pool_meta_payload(payload: &[u8]) -> Result<Self> {
match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
Ok(meta) => Ok(meta.into()),
Err(persisted_err) => {
let mut legacy: PoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
Error::other(format!(
"PoolMeta decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
))
})?;
// Runtime-only flag must not be restored from on-disk payload.
legacy.dont_save = false;
Ok(legacy)
}
}
}
pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
let mut new_meta = Self {
version: POOL_META_VERSION,
@@ -302,9 +320,7 @@ impl PoolMeta {
return Err(Error::other(format!("PoolMeta: unknown version: {version}")));
}
let mut buf = Deserializer::new(Cursor::new(&data[4..]));
let meta: PersistedPoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta.into();
*self = Self::decode_pool_meta_payload(&data[4..])?;
if self.version != POOL_META_VERSION {
return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version)));
@@ -1959,6 +1975,63 @@ mod tests {
assert_eq!(restored_decommission.bytes_done, 1024);
assert_eq!(restored_decommission.bytes_failed, 128);
}
#[test]
fn pool_meta_decode_supports_legacy_payload() {
let start_time = OffsetDateTime::now_utc();
let legacy_meta = PoolMeta {
version: POOL_META_VERSION,
pools: vec![PoolStatus {
id: 3,
cmd_line: "/legacy/pool".to_string(),
last_update: start_time,
decommission: Some(PoolDecommissionInfo {
start_time: Some(start_time),
items_decommissioned: 9,
items_decommission_failed: 2,
bytes_done: 2048,
bytes_failed: 256,
queued_buckets: vec!["not-persisted".to_string()],
decommissioned_buckets: vec!["not-persisted".to_string()],
bucket: "not-persisted".to_string(),
prefix: "not-persisted".to_string(),
object: "not-persisted".to_string(),
..Default::default()
}),
}],
dont_save: true,
};
let mut legacy_payload = Vec::new();
legacy_meta
.serialize(&mut Serializer::new(&mut legacy_payload))
.expect("legacy payload should serialize");
// New persisted schema has fewer top-level fields and should not decode this legacy struct payload.
let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
assert!(persisted_decode.is_err());
let decoded = PoolMeta::decode_pool_meta_payload(&legacy_payload).expect("legacy payload should decode");
assert_eq!(decoded.version, POOL_META_VERSION);
assert!(!decoded.dont_save, "runtime-only flag should reset on load");
assert_eq!(decoded.pools.len(), 1);
assert_eq!(decoded.pools[0].id, 3);
assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
assert_eq!(decoded.pools[0].last_update, start_time);
let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
assert_eq!(decommission.start_time, Some(start_time));
assert_eq!(decommission.items_decommissioned, 9);
assert_eq!(decommission.items_decommission_failed, 2);
assert_eq!(decommission.bytes_done, 2048);
assert_eq!(decommission.bytes_failed, 256);
// These fields were skipped in legacy payload and should be defaulted.
assert!(decommission.queued_buckets.is_empty());
assert!(decommission.decommissioned_buckets.is_empty());
assert!(decommission.bucket.is_empty());
assert!(decommission.prefix.is_empty());
assert!(decommission.object.is_empty());
}
}
// impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>
+27 -2
View File
@@ -14,6 +14,10 @@
use super::*;
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
created != OffsetDateTime::UNIX_EPOCH
}
impl ECStore {
#[instrument(skip(self))]
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
@@ -68,7 +72,9 @@ impl ECStore {
let mut info = self.peer_sys.get_bucket_info(bucket, opts).await?;
if let Ok(sys) = metadata_sys::get(bucket).await {
info.created = Some(sys.created);
if should_override_created_from_metadata(sys.created) {
info.created = Some(sys.created);
}
info.versioning = sys.versioning();
info.object_locking = sys.object_locking();
}
@@ -84,7 +90,9 @@ impl ECStore {
if !opts.no_metadata {
for bucket in buckets.iter_mut() {
if let Ok(created) = metadata_sys::created_at(&bucket.name).await {
if let Ok(created) = metadata_sys::created_at(&bucket.name).await
&& should_override_created_from_metadata(created)
{
bucket.created = Some(created);
}
}
@@ -148,3 +156,20 @@ impl ECStore {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::should_override_created_from_metadata;
use time::OffsetDateTime;
#[test]
fn should_not_override_when_metadata_created_is_unix_epoch() {
assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
}
#[test]
fn should_override_when_metadata_created_is_valid_time() {
let created = OffsetDateTime::from_unix_timestamp(1704067200).expect("valid timestamp");
assert!(should_override_created_from_metadata(created));
}
}
+46 -4
View File
@@ -74,6 +74,14 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
}
}
fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
if version_id.is_none() || version_id == Some(Uuid::nil()) {
Some(Uuid::nil().to_string())
} else {
None
}
}
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -164,6 +172,21 @@ impl FileMeta {
});
}
fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
let key = data_key_for_version(version_id);
if let Some(data) = self.data.find(key.as_str())? {
return Ok(Some(data));
}
if let Some(legacy_key) = legacy_data_key_for_version(version_id)
&& legacy_key != key
{
return self.data.find(legacy_key.as_str());
}
Ok(None)
}
// Find version
pub fn find_version(&self, vid: Option<Uuid>) -> Result<(usize, FileMetaVersion)> {
let vid = vid.unwrap_or_default();
@@ -709,10 +732,7 @@ impl FileMeta {
}
if read_data && fi.inline_data() {
fi.data = self
.data
.find(data_key_for_version(fi.version_id).as_str())?
.map(bytes::Bytes::from);
fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
}
found_fi = Some(fi);
@@ -1071,6 +1091,28 @@ mod test {
assert!(!inline_data.is_empty(), "Inline data should not be empty");
}
#[test]
fn test_into_fileinfo_reads_legacy_nil_uuid_inline_key() {
let mut fm = FileMeta::new();
let mut fi = FileInfo::new("test", 2, 1);
fi.mod_time = Some(OffsetDateTime::now_utc());
fi.data = Some(Bytes::from_static(b"legacy-inline"));
fi.set_inline_data();
fm.add_version(fi).unwrap();
let current_key = data_key_for_version(Some(Uuid::nil()));
let legacy_key = legacy_data_key_for_version(Some(Uuid::nil())).unwrap();
let payload = fm.data.find(current_key.as_str()).unwrap().unwrap();
fm.data.remove_key(current_key.as_str()).unwrap();
fm.data.replace(legacy_key.as_str(), payload.clone()).unwrap();
let restored = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
assert!(restored.inline_data());
assert_eq!(restored.data, Some(Bytes::from(payload)));
}
#[test]
fn test_error_handling_and_recovery() {
// Test error handling and recovery
+1 -5
View File
@@ -43,11 +43,7 @@ impl FileMeta {
let vid = version_id.unwrap_or_default();
if self.data.entries().unwrap_or_default() > 0
&& self
.data
.find(super::data_key_for_version(version_id).as_str())
.unwrap_or_default()
.is_some()
&& self.find_inline_data_for_version(version_id).unwrap_or_default().is_some()
{
return 0;
}
+48 -4
View File
@@ -761,14 +761,21 @@ fn normalize_config_url(config_url: &str) -> Result<String, String> {
return Err(format!("invalid config_url scheme: {}", url.scheme()));
}
let host = url.host_str().ok_or_else(|| "config_url missing host".to_string())?;
let path = url.path().trim_end_matches('/');
let path = url.path();
if path.contains("/.well-known/") && !path.ends_with("/.well-known/openid-configuration") {
// Strip `/.well-known/openid-configuration` (with optional trailing slash) if present.
// Everything else is preserved exactly so the issuer URL matches the provider's discovery
// document (e.g. Authentik includes a trailing slash, Keycloak does not).
let normalized_path = path
.strip_suffix('/')
.unwrap_or(path)
.strip_suffix("/.well-known/openid-configuration")
.unwrap_or(if path == "/" { "" } else { path });
if normalized_path.contains("/.well-known/") {
return Err("config_url uses an unsupported .well-known discovery URL".into());
}
let normalized_path = path.strip_suffix("/.well-known/openid-configuration").unwrap_or(path);
let mut issuer = format!("{}://{host}", url.scheme());
if let Some(port) = url.port() {
issuer.push(':');
@@ -886,20 +893,57 @@ mod tests {
#[test]
fn test_normalize_config_url() {
// --- Well-known suffix stripping ---
// Bare well-known URL → stripped to just the host
assert_eq!(
normalize_config_url("https://idp.example.com/.well-known/openid-configuration").unwrap(),
"https://idp.example.com"
);
// Trailing slash after well-known suffix is also stripped
assert_eq!(
normalize_config_url("https://idp.example.com/.well-known/openid-configuration/").unwrap(),
"https://idp.example.com"
);
// Well-known under a sub-path (Keycloak realms)
assert_eq!(
normalize_config_url("https://keycloak.example.com/realms/myrealm/.well-known/openid-configuration").unwrap(),
"https://keycloak.example.com/realms/myrealm"
);
// --- Providers WITHOUT trailing slash (Keycloak, Auth0, Okta, Google) ---
assert_eq!(
normalize_config_url("https://keycloak.example.com/realms/myrealm").unwrap(),
"https://keycloak.example.com/realms/myrealm"
);
assert_eq!(
normalize_config_url("https://idp.example.com/custom/realm").unwrap(),
"https://idp.example.com/custom/realm"
);
// --- Providers WITH trailing slash (Authentik) ---
assert_eq!(
normalize_config_url("https://auth.example.com/application/o/myapp/").unwrap(),
"https://auth.example.com/application/o/myapp/"
);
// --- Root-level issuer (bare host) ---
assert_eq!(normalize_config_url("https://idp.example.com").unwrap(), "https://idp.example.com");
assert_eq!(normalize_config_url("https://idp.example.com/").unwrap(), "https://idp.example.com");
// --- Custom port ---
assert_eq!(
normalize_config_url("https://idp.example.com:8443/auth/realms/test").unwrap(),
"https://idp.example.com:8443/auth/realms/test"
);
assert_eq!(
normalize_config_url("http://localhost:8080/application/o/app/").unwrap(),
"http://localhost:8080/application/o/app/"
);
// --- Error cases ---
assert!(normalize_config_url("https://idp.example.com/.well-known/invalid").is_err());
assert!(normalize_config_url("gopher://idp.example.com").is_err());
assert!(normalize_config_url("not-a-url").is_err());
}
#[test]
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Audit metrics collector.
//!
//! Collects audit log metrics including failed messages, queue length,
//! and total messages per target.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::audit`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::audit::*;
use std::borrow::Cow;
/// Audit target statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct AuditTargetStats {
/// Target identifier
pub target_id: String,
/// Number of messages that failed to send
pub failed_messages: u64,
/// Number of unsent messages in queue
pub queue_length: u64,
/// Total number of messages sent
pub total_messages: u64,
}
/// Collects audit metrics from the provided audit target statistics.
///
/// Uses the metric descriptors from `metrics_type::audit` module.
/// Returns a vector of Prometheus metrics for audit statistics.
pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * 3);
for stat in stats {
let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label("target_id", target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label("target_id", target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label("target_id", target_id_label),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_audit_metrics() {
let stats = vec![
AuditTargetStats {
target_id: "target-1".to_string(),
failed_messages: 5,
queue_length: 10,
total_messages: 1000,
},
AuditTargetStats {
target_id: "target-2".to_string(),
failed_messages: 2,
queue_length: 5,
total_messages: 500,
},
];
let metrics = collect_audit_metrics(&stats);
assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each
let failed = metrics
.iter()
.find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1"));
assert!(failed.is_some());
}
#[test]
fn test_collect_audit_metrics_empty() {
let stats: Vec<AuditTargetStats> = vec![];
let metrics = collect_audit_metrics(&stats);
assert!(metrics.is_empty());
}
}
+38 -63
View File
@@ -16,90 +16,56 @@
//!
//! Collects usage metrics for each bucket in the cluster, including
//! size, object counts, and quota information.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::node_bucket`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::node_bucket::*;
use std::borrow::Cow;
/// Usage statistics for a single bucket.
/// Bucket statistics for metrics collection.
///
/// This struct provides a decoupled interface for collecting bucket metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct BucketStats {
/// Bucket name
/// Name of the bucket
pub name: String,
/// Total bytes used by the bucket
/// Total size of all objects in the bucket (bytes)
pub size_bytes: u64,
/// Total number of objects in the bucket
/// Number of objects in the bucket
pub objects_count: u64,
/// Quota limit in bytes (0 means no quota)
/// Quota limit for the bucket (bytes), 0 if no quota
pub quota_bytes: u64,
}
// Static metric definitions
const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes";
const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total";
const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes";
const HELP_SIZE: &str = "Total bytes used by the bucket";
const HELP_OBJECTS: &str = "Total number of objects in the bucket";
const HELP_QUOTA: &str = "Quota limit in bytes for the bucket";
/// Collects per-bucket usage metrics from the provided bucket statistics.
/// Collects per-bucket metrics from the provided bucket statistics.
///
/// # Metrics Produced
///
/// For each bucket, the following metrics are produced with a `bucket` label:
///
/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket
/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket
/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured)
///
/// # Arguments
///
/// * `buckets` - Slice of bucket statistics
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats};
///
/// let buckets = vec![
/// BucketStats {
/// name: "my-bucket".to_string(),
/// size_bytes: 1_000_000,
/// objects_count: 100,
/// quota_bytes: 10_000_000,
/// },
/// ];
/// let metrics = collect_bucket_metrics(&buckets);
/// assert_eq!(metrics.len(), 3); // size, objects, quota
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::node_bucket` module.
/// Returns a vector of Prometheus metrics for all buckets.
pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
if buckets.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(buckets.len() * 3);
for bucket in buckets {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
// Bucket size in bytes
metrics.push(
PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64)
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
// Object count
metrics.push(
PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64)
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
// Quota (always emit, 0 when no quota configured for consistent PromQL queries)
metrics.push(
PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64)
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
.with_label("bucket", bucket_label),
);
}
@@ -118,27 +84,28 @@ mod tests {
BucketStats {
name: "test-bucket".to_string(),
size_bytes: 1000,
objects_count: 50,
objects_count: 100,
quota_bytes: 0,
},
BucketStats {
name: "other-bucket".to_string(),
name: "another-bucket".to_string(),
size_bytes: 2000,
objects_count: 100,
objects_count: 200,
quota_bytes: 0,
},
];
let metrics = collect_bucket_metrics(&buckets);
report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here.
report_metrics(&metrics);
// 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
assert_eq!(metrics.len(), 6);
// Verify test-bucket metrics
// Verify test-bucket metrics have correct labels
let test_bucket_size_name = BUCKET_USAGE_BYTES_MD.get_full_metric_name();
let test_bucket_size = metrics
.iter()
.find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
.find(|m| m.name == test_bucket_size_name && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
assert!(test_bucket_size.is_some());
assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
}
@@ -159,9 +126,13 @@ mod tests {
assert_eq!(metrics.len(), 3);
// Verify quota metric exists
let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
let quota_metric = metrics.iter().find(|m| {
m.name == quota_metric_name
&& m.value == 10000.0
&& m.labels.iter().any(|(k, v)| *k == "bucket" && v == "quota-bucket")
});
assert!(quota_metric.is_some());
assert_eq!(quota_metric.map(|m| m.value), Some(10000.0));
}
#[test]
@@ -185,9 +156,13 @@ mod tests {
// Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
assert_eq!(metrics.len(), 3);
let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
let quota_metric_name = BUCKET_QUOTA_BYTES_MD.get_full_metric_name();
let quota_metric = metrics.iter().find(|m| {
m.name == quota_metric_name
&& m.value == 0.0
&& m.labels.iter().any(|(k, v)| *k == "bucket" && v == "no-quota-bucket")
});
assert!(quota_metric.is_some());
assert_eq!(quota_metric.map(|m| m.value), Some(0.0));
}
#[test]
@@ -13,32 +13,33 @@
// limitations under the License.
//! Bucket replication bandwidth metrics collector.
//!
//! Collects bandwidth metrics for bucket replication targets.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
use std::borrow::Cow;
/// Bucket replication bandwidth stats for one replication target.
/// Bucket replication bandwidth statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
/// Name of the bucket
pub bucket: String,
/// Target ARN for replication
pub target_arn: String,
pub limit_bytes_per_sec: i64,
/// Configured bandwidth limit in bytes per second
pub limit_bytes_per_sec: u64,
/// Current bandwidth in bytes per second (EWMA)
pub current_bandwidth_bytes_per_sec: f64,
}
const BUCKET_LABEL: &str = "bucket";
const TARGET_ARN_LABEL: &str = "targetArn";
const METRIC_BANDWIDTH_LIMIT: &str = "rustfs_bucket_replication_bandwidth_limit_bytes_per_second";
const METRIC_BANDWIDTH_CURRENT: &str = "rustfs_bucket_replication_bandwidth_current_bytes_per_second";
const HELP_BANDWIDTH_LIMIT: &str = "Configured bandwidth limit for replication in bytes per second";
const HELP_BANDWIDTH_CURRENT: &str = "Current replication bandwidth in bytes per second (EWMA)";
/// Collect bucket replication bandwidth metrics for Prometheus/OpenTelemetry export.
#[must_use]
#[inline]
/// Collects bucket replication bandwidth metrics from the provided statistics.
///
/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
/// Returns a vector of Prometheus metrics for replication bandwidth.
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -50,25 +51,15 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone());
metrics.push(
PrometheusMetric::new(
METRIC_BANDWIDTH_LIMIT,
MetricType::Gauge,
HELP_BANDWIDTH_LIMIT,
stat.limit_bytes_per_sec as f64,
)
.with_label(BUCKET_LABEL, bucket_label.clone())
.with_label(TARGET_ARN_LABEL, target_arn_label.clone()),
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
.with_label("bucket", bucket_label.clone())
.with_label("targetArn", target_arn_label.clone()),
);
metrics.push(
PrometheusMetric::new(
METRIC_BANDWIDTH_CURRENT,
MetricType::Gauge,
HELP_BANDWIDTH_CURRENT,
stat.current_bandwidth_bytes_per_sec,
)
.with_label(BUCKET_LABEL, bucket_label)
.with_label(TARGET_ARN_LABEL, target_arn_label),
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
.with_label("bucket", bucket_label)
.with_label("targetArn", target_arn_label),
);
}
@@ -91,23 +82,27 @@ mod tests {
let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
assert_eq!(metrics.len(), 2);
let limit_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_LIMIT);
let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
let limit_metric = metrics.iter().find(|m| {
m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
});
assert!(limit_metric.is_some());
assert_eq!(limit_metric.map(|m| m.value), Some(1_048_576.0));
assert!(
limit_metric
.and_then(|m| {
m.labels
.iter()
.find(|(k, _)| *k == TARGET_ARN_LABEL)
.find(|(k, _)| *k == "targetArn")
.map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
})
.unwrap_or(false)
);
let current_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_CURRENT);
let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
let current_metric = metrics.iter().find(|m| {
m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
});
assert!(current_metric.is_some());
assert_eq!(current_metric.map(|m| m.value), Some(204_800.0));
}
#[test]
+23 -89
View File
@@ -16,9 +16,12 @@
//!
//! Collects aggregate metrics across the entire RustFS cluster including
//! total capacity, usage, and object counts.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster::*;
/// Cluster capacity and usage statistics for metrics collection.
///
@@ -41,88 +44,19 @@ pub struct ClusterStats {
pub buckets_count: u64,
}
// Static metric definitions to avoid allocations
const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes";
const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes";
const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes";
const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes";
const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total";
const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total";
const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks";
const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)";
const HELP_USED: &str = "Total used storage capacity in bytes";
const HELP_FREE: &str = "Total free storage capacity in bytes";
const HELP_OBJECTS: &str = "Total number of objects in the cluster";
const HELP_BUCKETS: &str = "Total number of buckets in the cluster";
/// Number of metrics produced by this collector.
const METRIC_COUNT: usize = 6;
/// Collects cluster-wide metrics from the provided statistics.
/// Collects cluster-wide metrics from the provided cluster statistics.
///
/// # Metrics Produced
///
/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks
/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead
/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity
/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity
/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster
/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster
///
/// # Arguments
///
/// * `stats` - Cluster statistics containing capacity and usage data
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats};
///
/// let stats = ClusterStats {
/// raw_capacity_bytes: 10_000_000_000,
/// usable_capacity_bytes: 8_000_000_000,
/// used_bytes: 2_000_000_000,
/// free_bytes: 6_000_000_000,
/// objects_count: 1000,
/// buckets_count: 10,
/// };
/// let metrics = collect_cluster_metrics(&stats);
/// assert_eq!(metrics.len(), 6);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::cluster` module.
/// Returns a vector of Prometheus metrics for cluster statistics.
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(METRIC_COUNT);
metrics.push(PrometheusMetric::new(
METRIC_RAW_CAPACITY,
MetricType::Gauge,
HELP_RAW_CAPACITY,
stats.raw_capacity_bytes as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_USABLE_CAPACITY,
MetricType::Gauge,
HELP_USABLE_CAPACITY,
stats.usable_capacity_bytes as f64,
));
metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64));
metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64));
metrics.push(PrometheusMetric::new(
METRIC_OBJECTS,
MetricType::Gauge,
HELP_OBJECTS,
stats.objects_count as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_BUCKETS,
MetricType::Gauge,
HELP_BUCKETS,
stats.buckets_count as f64,
));
metrics
vec![
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
}
#[cfg(test)]
@@ -147,24 +81,24 @@ mod tests {
assert_eq!(metrics.len(), 6);
// Verify raw capacity
let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY);
let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name();
let raw_capacity = metrics.iter().find(|m| m.name == raw_capacity_name && m.value == 3000.0);
assert!(raw_capacity.is_some());
assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0));
// Verify used capacity
let used = metrics.iter().find(|m| m.name == METRIC_USED);
let used_name = CLUSTER_CAPACITY_USED_BYTES_MD.get_full_metric_name();
let used = metrics.iter().find(|m| m.name == used_name && m.value == 1200.0);
assert!(used.is_some());
assert_eq!(used.map(|m| m.value), Some(1200.0));
// Verify object count
let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS);
let objects_name = CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name();
let objects = metrics.iter().find(|m| m.name == objects_name && m.value == 100.0);
assert!(objects.is_some());
assert_eq!(objects.map(|m| m.value), Some(100.0));
// Verify bucket count
let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS);
let buckets_name = CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name();
let buckets = metrics.iter().find(|m| m.name == buckets_name && m.value == 5.0);
assert!(buckets.is_some());
assert_eq!(buckets.map(|m| m.value), Some(5.0));
}
#[test]
@@ -0,0 +1,81 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster config metrics collector.
//!
//! Collects cluster configuration metrics including storage class
//! parity settings.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster_config`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_config::*;
/// Cluster configuration statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterConfigStats {
/// Parity for reduced redundancy storage (RRS) class
pub rrs_parity: u32,
/// Parity for standard storage class
pub standard_parity: u32,
}
/// Collects cluster config metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_config` module.
/// Returns a vector of Prometheus metrics for cluster configuration.
pub fn collect_cluster_config_metrics(stats: &ClusterConfigStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&CONFIG_RRS_PARITY_MD, stats.rrs_parity as f64),
PrometheusMetric::from_descriptor(&CONFIG_STANDARD_PARITY_MD, stats.standard_parity as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_cluster_config_metrics() {
let stats = ClusterConfigStats {
rrs_parity: 2,
standard_parity: 4,
};
let metrics = collect_cluster_config_metrics(&stats);
assert_eq!(metrics.len(), 2);
let rrs = metrics.iter().find(|m| m.value == 2.0);
assert!(rrs.is_some());
let standard = metrics.iter().find(|m| m.value == 4.0);
assert!(standard.is_some());
}
#[test]
fn test_collect_cluster_config_metrics_default() {
let stats = ClusterConfigStats::default();
let metrics = collect_cluster_config_metrics(&stats);
assert_eq!(metrics.len(), 2);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,179 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster erasure set metrics collector.
//!
//! Collects erasure coding set metrics including parity, quorum,
//! drive counts, and health status.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_erasure_set::*;
/// Erasure set statistics.
#[derive(Debug, Clone, Default)]
pub struct ErasureSetStats {
/// Pool ID
pub pool_id: u32,
/// Set ID within the pool
pub set_id: u32,
/// Total number of drives in the set
pub size: u32,
/// Number of parity drives
pub parity: u32,
/// Number of data shards
pub data_shards: u32,
/// Read quorum
pub read_quorum: u32,
/// Write quorum
pub write_quorum: u32,
/// Number of online drives
pub online_drives_count: u32,
/// Number of healing drives
pub healing_drives_count: u32,
/// Health status (1=healthy, 0=unhealthy)
pub health: u8,
/// Read tolerance (number of drive failures tolerated for reads)
pub read_tolerance: u32,
/// Write tolerance (number of drive failures tolerated for writes)
pub write_tolerance: u32,
/// Read health status (1=healthy, 0=unhealthy)
pub read_health: u8,
/// Write health status (1=healthy, 0=unhealthy)
pub write_health: u8,
}
/// Collects erasure set metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for erasure sets.
pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(stats.len() * 12);
for stat in stats {
let pool_id_label = stat.pool_id.to_string();
let set_id_label = stat.set_id.to_string();
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_SIZE_MD, stat.size as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_PARITY_MD, stat.parity as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_DATA_SHARDS_MD, stat.data_shards as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_QUORUM_MD, stat.read_quorum as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_QUORUM_MD, stat.write_quorum as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_ONLINE_DRIVES_COUNT_MD, stat.online_drives_count as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_HEALING_DRIVES_COUNT_MD, stat.healing_drives_count as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_HEALTH_MD, stat.health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_TOLERANCE_MD, stat.read_tolerance as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_TOLERANCE_MD, stat.write_tolerance as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_READ_HEALTH_MD, stat.read_health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_erasure_set_metrics() {
let stats = vec![ErasureSetStats {
pool_id: 1,
set_id: 0,
size: 16,
parity: 4,
data_shards: 12,
read_quorum: 13,
write_quorum: 13,
online_drives_count: 16,
healing_drives_count: 0,
health: 1,
read_tolerance: 4,
write_tolerance: 4,
read_health: 1,
write_health: 1,
}];
let metrics = collect_erasure_set_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 12);
let size_name = ERASURE_SET_SIZE_MD.get_full_metric_name();
let size = metrics.iter().find(|m| m.name == size_name);
assert!(size.is_some());
assert_eq!(size.map(|m| m.value), Some(16.0));
let health_name = ERASURE_SET_HEALTH_MD.get_full_metric_name();
let health = metrics.iter().find(|m| m.name == health_name);
assert!(health.is_some());
assert_eq!(health.map(|m| m.value), Some(1.0));
}
#[test]
fn test_collect_erasure_set_metrics_empty() {
let stats: Vec<ErasureSetStats> = vec![];
let metrics = collect_erasure_set_metrics(&stats);
assert!(metrics.is_empty());
}
}
@@ -0,0 +1,85 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster health metrics collector.
//!
//! Collects cluster-wide health metrics including drive counts
//! (offline, online, total).
//!
//! This collector reuses the metric descriptors defined in `metrics_type::cluster_health`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_health::*;
/// Cluster health statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterHealthStats {
/// Number of offline drives in the cluster
pub drives_offline_count: u64,
/// Number of online drives in the cluster
pub drives_online_count: u64,
/// Total number of drives in the cluster
pub drives_count: u64,
}
/// Collects cluster health metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_health` module.
/// Returns a vector of Prometheus metrics for cluster health.
pub fn collect_cluster_health_metrics(stats: &ClusterHealthStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_OFFLINE_COUNT_MD, stats.drives_offline_count as f64),
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_ONLINE_COUNT_MD, stats.drives_online_count as f64),
PrometheusMetric::from_descriptor(&HEALTH_DRIVES_COUNT_MD, stats.drives_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_cluster_health_metrics() {
let stats = ClusterHealthStats {
drives_offline_count: 2,
drives_online_count: 18,
drives_count: 20,
};
let metrics = collect_cluster_health_metrics(&stats);
assert_eq!(metrics.len(), 3);
let offline = metrics.iter().find(|m| m.value == 2.0);
assert!(offline.is_some());
let online = metrics.iter().find(|m| m.value == 18.0);
assert!(online.is_some());
}
#[test]
fn test_collect_cluster_health_metrics_default() {
let stats = ClusterHealthStats::default();
let metrics = collect_cluster_health_metrics(&stats);
assert_eq!(metrics.len(), 3);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,117 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster IAM metrics collector.
//!
//! Collects IAM (Identity and Access Management) metrics including
//! plugin authentication service stats and sync statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_iam::*;
/// IAM statistics.
#[derive(Debug, Clone, Default)]
pub struct IamStats {
/// Time in seconds since last failed authn service request
pub plugin_authn_service_last_fail_seconds: u64,
/// Time in seconds since last successful authn service request
pub plugin_authn_service_last_succ_seconds: u64,
/// Average RTT of successful requests in the last minute (ms)
pub plugin_authn_service_succ_avg_rtt_ms_minute: u64,
/// Maximum RTT of successful requests in the last minute (ms)
pub plugin_authn_service_succ_max_rtt_ms_minute: u64,
/// Total requests count in the last full minute
pub plugin_authn_service_total_requests_minute: u64,
/// Time in milliseconds since last successful IAM data sync
pub since_last_sync_millis: u64,
/// Number of failed IAM data syncs since server start
pub sync_failures: u64,
/// Number of successful IAM data syncs since server start
pub sync_successes: u64,
}
/// Collects IAM metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for IAM statistics.
pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
stats.plugin_authn_service_last_fail_seconds as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD,
stats.plugin_authn_service_last_succ_seconds as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD,
stats.plugin_authn_service_succ_avg_rtt_ms_minute as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD,
stats.plugin_authn_service_succ_max_rtt_ms_minute as f64,
),
PrometheusMetric::from_descriptor(
&PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD,
stats.plugin_authn_service_total_requests_minute as f64,
),
PrometheusMetric::from_descriptor(&SINCE_LAST_SYNC_MILLIS_MD, stats.since_last_sync_millis as f64),
PrometheusMetric::from_descriptor(&SYNC_FAILURES_MD, stats.sync_failures as f64),
PrometheusMetric::from_descriptor(&SYNC_SUCCESSES_MD, stats.sync_successes as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_iam_metrics() {
let stats = IamStats {
plugin_authn_service_last_fail_seconds: 3600,
plugin_authn_service_last_succ_seconds: 10,
plugin_authn_service_succ_avg_rtt_ms_minute: 50,
plugin_authn_service_succ_max_rtt_ms_minute: 200,
plugin_authn_service_total_requests_minute: 1000,
since_last_sync_millis: 5000,
sync_failures: 5,
sync_successes: 1000,
};
let metrics = collect_iam_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name();
let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name);
assert!(sync_successes.is_some());
assert_eq!(sync_successes.map(|m| m.value), Some(1000.0));
}
#[test]
fn test_collect_iam_metrics_default() {
let stats = IamStats::default();
let metrics = collect_iam_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,199 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Cluster usage metrics collector.
//!
//! Collects cluster-wide and per-bucket usage metrics including
//! object counts, sizes, versions, and distributions.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_usage::*;
/// Cluster-wide usage statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterUsageStats {
/// Total bytes used in the cluster
pub total_bytes: u64,
/// Total number of objects
pub objects_count: u64,
/// Total number of object versions (including delete markers)
pub versions_count: u64,
/// Total number of delete markers
pub delete_markers_count: u64,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
pub versions_distribution: Vec<(String, u64)>,
}
/// Per-bucket usage statistics.
#[derive(Debug, Clone, Default)]
pub struct BucketUsageStats {
/// Bucket name
pub bucket: String,
/// Total bytes used in the bucket
pub total_bytes: u64,
/// Total number of objects in the bucket
pub objects_count: u64,
/// Total number of object versions (including delete markers)
pub versions_count: u64,
/// Total number of delete markers
pub delete_markers_count: u64,
/// Bucket quota in bytes (0 if no quota)
pub quota_bytes: u64,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
pub version_count_distribution: Vec<(String, u64)>,
}
/// Collects cluster-wide usage metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for cluster usage.
pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len());
metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64));
metrics.push(PrometheusMetric::from_descriptor(
&USAGE_DELETE_MARKERS_COUNT_MD,
stats.delete_markers_count as f64,
));
// Object size distribution
for (range, count) in &stats.object_size_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_OBJECTS_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone()),
);
}
// Version distribution
for (range, count) in &stats.versions_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_VERSIONS_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone()),
);
}
metrics
}
/// Collects per-bucket usage metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for bucket usage.
pub fn collect_bucket_usage_metrics(stats: &[BucketUsageStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
for stat in stats {
let bucket_label = stat.bucket.clone();
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, stat.total_bytes as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, stat.objects_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, stat.versions_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, stat.delete_markers_count as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, stat.quota_bytes as f64)
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
// Object size distribution per bucket
for (range, count) in &stat.object_size_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone())
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
}
// Version count distribution per bucket
for (range, count) in &stat.version_count_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, *count as f64)
.with_label_owned(RANGE_LABEL, range.clone())
.with_label_owned(BUCKET_LABEL, bucket_label.clone()),
);
}
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_cluster_usage_metrics() {
let stats = ClusterUsageStats {
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
objects_count: 10000,
versions_count: 15000,
delete_markers_count: 500,
object_size_distribution: vec![
("0-1KB".to_string(), 5000),
("1KB-1MB".to_string(), 3000),
("1MB-100MB".to_string(), 1500),
("100MB+".to_string(), 500),
],
versions_distribution: vec![("1".to_string(), 8000), ("2-5".to_string(), 1500), ("6+".to_string(), 500)],
};
let metrics = collect_cluster_usage_metrics(&stats);
report_metrics(&metrics);
// 4 base metrics + 4 size distribution + 3 version distribution = 11
assert_eq!(metrics.len(), 11);
let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
assert!(total_bytes.is_some());
}
#[test]
fn test_collect_bucket_usage_metrics() {
let stats = vec![BucketUsageStats {
bucket: "test-bucket".to_string(),
total_bytes: 1024 * 1024 * 1024 * 10, // 10 GB
objects_count: 1000,
versions_count: 1200,
delete_markers_count: 50,
quota_bytes: 1024 * 1024 * 1024 * 100, // 100 GB quota
object_size_distribution: vec![("0-1KB".to_string(), 500)],
version_count_distribution: vec![("1".to_string(), 800)],
}];
let metrics = collect_bucket_usage_metrics(&stats);
report_metrics(&metrics);
// 5 base metrics + 1 size distribution + 1 version distribution = 7
assert_eq!(metrics.len(), 7);
}
}
+77 -228
View File
@@ -12,9 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Global metrics collector initialization.
//!
//! This module provides the entry point for initializing all metrics collectors.
//! The actual statistics collection functions are in `stats_collector.rs`.
use crate::collectors::stats_collector::{
collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats,
collect_process_stats,
};
use crate::collectors::{
BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics,
collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, collect_resource_metrics,
collect_bucket_metrics, collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics,
collect_resource_metrics,
};
use crate::constants::{
DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL,
@@ -23,245 +32,85 @@ use crate::constants::{
ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
};
use crate::format::report_metrics;
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use rustfs_utils::get_env_opt_u64;
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{instrument, warn};
use tracing::warn;
/// Process start time for calculating uptime.
static PROCESS_START: OnceLock<Instant> = OnceLock::new();
/// Get the process start time, initializing it on first call.
#[inline]
fn get_process_start() -> &'static Instant {
PROCESS_START.get_or_init(Instant::now)
}
/// Collect cluster statistics from the storage layer.
#[instrument]
async fn collect_cluster_stats() -> ClusterStats {
let Some(store) = new_object_layer_fn() else {
return ClusterStats::default();
};
let storage_info = store.storage_info().await;
let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
// Get bucket and object counts from data usage info
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!("Failed to load data usage from backend: {}", e);
// Fall back to bucket list for buckets_count, objects_count stays 0
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|e| {
warn!("Failed to list buckets for cluster metrics: {}", e);
Vec::new()
});
(buckets.len() as u64, 0)
}
};
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
}
}
/// Collect bucket statistics from the storage layer.
async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_data_usage_from_backend(store.clone()).await {
Ok(info) => Some(info),
Err(e) => {
warn!("Failed to load data usage from backend for bucket metrics: {}", e);
None
}
};
let buckets = match store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
{
Ok(b) => b,
Err(e) => {
warn!("Failed to list buckets for metrics: {}", e);
return Vec::new();
}
};
// Build bucket stats with real data from DataUsageInfo
let mut stats = Vec::with_capacity(buckets.len());
for bucket in buckets {
if bucket.name.starts_with('.') {
continue;
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
// Get quota from bucket metadata
let quota_bytes = match get_quota_config(&bucket.name).await {
Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
Err(_) => 0, // No quota configured or error
};
stats.push(BucketStats {
name: bucket.name,
size_bytes,
objects_count,
quota_bytes,
});
}
stats
}
/// Collect bucket replication bandwidth stats from the global monitor.
fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
let Some(monitor) = get_global_bucket_monitor() else {
return Vec::new();
};
monitor
.get_report(|_| true)
.bucket_stats
.into_iter()
.map(|(opts, details)| BucketReplicationBandwidthStats {
bucket: opts.name,
target_arn: opts.replication_arn,
limit_bytes_per_sec: details.limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
})
.collect()
}
/// Collect disk statistics from the storage layer.
async fn collect_disk_stats() -> Vec<DiskStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
let storage_info = store.storage_info().await;
storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
})
.collect()
}
/// Collect resource statistics for the current process.
///
/// Collects:
/// - Uptime: Calculated from process start time
/// - Memory: Process resident set size from sysinfo
/// - CPU: Process CPU usage percentage from sysinfo
#[inline]
fn collect_process_stats() -> ResourceStats {
let uptime_seconds = get_process_start().elapsed().as_secs();
// Use sysinfo for process metrics
let mut sys = System::new();
let pid = Pid::from_u32(std::process::id());
sys.refresh_processes_specifics(
ProcessesToUpdate::Some(&[pid]),
true,
ProcessRefreshKind::nothing().with_cpu().with_memory(),
);
if let Some(process) = sys.process(pid) {
ResourceStats {
cpu_percent: process.cpu_usage() as f64,
memory_bytes: process.memory(),
uptime_seconds,
}
} else {
// Fallback if process not found
ResourceStats {
cpu_percent: 0.0,
memory_bytes: 0,
uptime_seconds,
}
}
}
/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics.
/// Initialize all metrics collectors.
///
/// This function spawns background tasks that periodically collect metrics
/// and report them using the `metrics` crate.
/// from various sources and report them to the metrics system.
///
/// # Arguments
/// * `token` - A `CancellationToken` that can be used to gracefully shut down
/// all metrics collection tasks.
///
/// * `token` - A cancellation token to gracefully stop the metrics collection tasks.
/// # Environment Variables
/// The collection intervals can be configured via environment variables:
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60)
/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300)
/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60)
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30)
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15)
/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds.
///
/// Legacy interval names without `_SEC` are still accepted for backward compatibility:
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL`
/// - `RUSTFS_METRICS_BUCKET_INTERVAL`
/// - `RUSTFS_METRICS_NODE_INTERVAL`
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL`
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL`
pub fn init_metrics_collectors(token: CancellationToken) {
// Initialize process start time
get_process_start();
const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL";
const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL";
const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL";
const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL";
const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
// Helper closure to determine interval for a specific metric type
let get_interval = |env_key: &str, type_default: Duration| -> Duration {
// 1. Try specific env var
// 2. Fallback to global default env var (if set differently from hardcoded default)
// 3. Fallback to type specific default
fn parse_interval(msc: &str, legacy_msc: &str) -> Option<u64> {
get_env_opt_u64(msc)
.or_else(|| get_env_opt_u64(legacy_msc))
.filter(|&v| v > 0)
}
// Helper to check if value is valid (non-zero)
let is_valid = |v: u64| v > 0;
// Read intervals from environment or use defaults, ensuring zero is ignored.
let cluster_interval = parse_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_CLUSTER_METRICS_INTERVAL);
if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) {
Duration::from_secs(val)
} else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) {
Duration::from_secs(val)
} else {
type_default
}
};
let bucket_interval = parse_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_BUCKET_METRICS_INTERVAL);
let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
let bucket_replication_bandwidth_interval = get_interval(
ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
);
let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
let bucket_replication_bandwidth_interval =
parse_interval(ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, LEGACY_REPLICATION_BANDWIDTH_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL);
let node_interval = parse_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_NODE_METRICS_INTERVAL);
let resource_interval = parse_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL)
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
.filter(|&v| v > 0)
.map(Duration::from_secs)
.unwrap_or(DEFAULT_RESOURCE_METRICS_INTERVAL);
// Spawn task for cluster metrics
let token_clone = token.clone();
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! ILM (Information Lifecycle Management) metrics collector.
//!
//! Collects ILM metrics including pending tasks, active tasks,
//! and scanned versions.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::ilm`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::ilm::*;
/// ILM statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct IlmStats {
/// Number of pending ILM expiry tasks
pub expiry_pending_tasks: u64,
/// Number of active ILM transition tasks
pub transition_active_tasks: u64,
/// Number of pending ILM transition tasks
pub transition_pending_tasks: u64,
/// Number of missed immediate ILM transition tasks
pub transition_missed_immediate_tasks: u64,
/// Total number of object versions scanned for ILM
pub versions_scanned: u64,
}
/// Collects ILM metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::ilm` module.
/// Returns a vector of Prometheus metrics for ILM statistics.
pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&ILM_EXPIRY_PENDING_TASKS_MD, stats.expiry_pending_tasks as f64),
PrometheusMetric::from_descriptor(&ILM_TRANSITION_ACTIVE_TASKS_MD, stats.transition_active_tasks as f64),
PrometheusMetric::from_descriptor(&ILM_TRANSITION_PENDING_TASKS_MD, stats.transition_pending_tasks as f64),
PrometheusMetric::from_descriptor(
&ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD,
stats.transition_missed_immediate_tasks as f64,
),
PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_ilm_metrics() {
let stats = IlmStats {
expiry_pending_tasks: 100,
transition_active_tasks: 5,
transition_pending_tasks: 50,
transition_missed_immediate_tasks: 10,
versions_scanned: 1000000,
};
let metrics = collect_ilm_metrics(&stats);
assert_eq!(metrics.len(), 5);
let pending = metrics.iter().find(|m| m.value == 100.0);
assert!(pending.is_some());
let scanned = metrics.iter().find(|m| m.value == 1000000.0);
assert!(scanned.is_some());
}
#[test]
fn test_collect_ilm_metrics_default() {
let stats = IlmStats::default();
let metrics = collect_ilm_metrics(&stats);
assert_eq!(metrics.len(), 5);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,122 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Logger webhook metrics collector.
//!
//! Collects webhook metrics including failed messages, queue length,
//! and total messages per webhook target.
use crate::format::PrometheusMetric;
use crate::metrics_type::logger_webhook::{
ENDPOINT_LABEL, NAME_LABEL, WEBHOOK_FAILED_MESSAGES_MD, WEBHOOK_QUEUE_LENGTH_MD, WEBHOOK_TOTAL_MESSAGES_MD,
};
use std::borrow::Cow;
/// Webhook target statistics.
#[derive(Debug, Clone, Default)]
pub struct WebhookTargetStats {
/// Webhook name
pub name: String,
/// Webhook endpoint URL
pub endpoint: String,
/// Number of failed messages
pub failed_messages: u64,
/// Number of messages in queue
pub queue_length: u64,
/// Total number of messages sent
pub total_messages: u64,
}
/// Collects webhook metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::logger_webhook` module.
/// Returns a vector of Prometheus metrics for webhook statistics.
pub fn collect_webhook_metrics(stats: &[WebhookTargetStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(stats.len() * 3);
for stat in stats {
let name_label: Cow<'static, str> = Cow::Owned(stat.name.clone());
let endpoint_label: Cow<'static, str> = Cow::Owned(stat.endpoint.clone());
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&WEBHOOK_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label(NAME_LABEL, name_label.clone())
.with_label(ENDPOINT_LABEL, endpoint_label.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_webhook_metrics() {
let stats = vec![WebhookTargetStats {
name: "alert-webhook".to_string(),
endpoint: "https://hooks.example.com/alert".to_string(),
failed_messages: 3,
queue_length: 15,
total_messages: 5000,
}];
let metrics = collect_webhook_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 3);
// Verify we have metrics with the expected values
let failed = metrics.iter().find(|m| m.value == 3.0);
assert!(failed.is_some());
// Verify labels
let total = metrics.iter().find(|m| m.value == 5000.0);
assert!(total.is_some());
let total_metric = total.unwrap();
assert!(
total_metric
.labels
.iter()
.any(|(k, v)| *k == NAME_LABEL && v == "alert-webhook")
);
assert!(
total_metric
.labels
.iter()
.any(|(k, v)| *k == ENDPOINT_LABEL && v == "https://hooks.example.com/alert")
);
}
#[test]
fn test_collect_webhook_metrics_empty() {
let stats: Vec<WebhookTargetStats> = vec![];
let metrics = collect_webhook_metrics(&stats);
assert!(metrics.is_empty());
}
}
+35
View File
@@ -61,16 +61,51 @@
//! report_metrics(&metrics);
//! ```
mod audit;
mod bucket;
mod bucket_replication;
mod cluster;
mod cluster_config;
mod cluster_erasure_set;
mod cluster_health;
mod cluster_iam;
mod cluster_usage;
pub(crate) mod global;
mod ilm;
mod logger_webhook;
mod node;
mod notification;
mod replication;
mod request;
mod resource;
mod scanner;
mod stats_collector;
mod system_cpu;
mod system_drive;
mod system_memory;
mod system_network;
mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
pub use cluster::{ClusterStats, collect_cluster_metrics};
pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics};
pub use cluster_iam::{IamStats, collect_iam_metrics};
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
pub use global::init_metrics_collectors;
pub use ilm::{IlmStats, collect_ilm_metrics};
pub use logger_webhook::{WebhookTargetStats, collect_webhook_metrics};
pub use node::{DiskStats, collect_node_metrics};
pub use notification::{NotificationStats, collect_notification_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub use request::{ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub use scanner::{ScannerStats, collect_scanner_metrics};
pub use system_cpu::{CpuStats, collect_cpu_metrics};
pub use system_drive::{DriveCountStats, DriveDetailedStats, collect_drive_count_metrics, collect_drive_detailed_metrics};
pub use system_memory::{MemoryStats, collect_memory_metrics};
pub use system_network::{NetworkStats, collect_network_metrics};
pub use system_process::{ProcessStats, collect_process_metrics};
+24 -54
View File
@@ -16,93 +16,61 @@
//!
//! Collects storage metrics for each disk/drive in the cluster,
//! including capacity, usage, and health status.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::node_disk`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::node_disk::*;
use std::borrow::Cow;
/// Statistics for a single disk/drive.
/// Disk statistics for metrics collection.
///
/// This struct provides a decoupled interface for collecting disk metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct DiskStats {
/// Server endpoint (e.g., "node1:9000")
pub server: String,
/// Drive path (e.g., "/data/disk1")
pub drive: String,
/// Total capacity in bytes
/// Total disk capacity in bytes
pub total_bytes: u64,
/// Used space in bytes
/// Used disk space in bytes
pub used_bytes: u64,
/// Free space in bytes
/// Free disk space in bytes
pub free_bytes: u64,
}
// Static metric definitions
const METRIC_TOTAL: &str = "rustfs_node_disk_total_bytes";
const METRIC_USED: &str = "rustfs_node_disk_used_bytes";
const METRIC_FREE: &str = "rustfs_node_disk_free_bytes";
const HELP_TOTAL: &str = "Total disk capacity in bytes";
const HELP_USED: &str = "Used disk space in bytes";
const HELP_FREE: &str = "Free disk space in bytes";
/// Collects per-node disk metrics from the provided disk statistics.
/// Collects per-disk metrics from the provided disk statistics.
///
/// # Metrics Produced
///
/// For each disk, the following metrics are produced with `server` and `drive` labels:
///
/// - `rustfs_node_disk_total_bytes`: Total capacity of the disk
/// - `rustfs_node_disk_used_bytes`: Used space on the disk
/// - `rustfs_node_disk_free_bytes`: Free space on the disk
///
/// # Arguments
///
/// * `disks` - Slice of disk statistics
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_node_metrics, DiskStats};
///
/// let disks = vec![
/// DiskStats {
/// server: "node1:9000".to_string(),
/// drive: "/data/disk1".to_string(),
/// total_bytes: 1_000_000_000,
/// used_bytes: 400_000_000,
/// free_bytes: 600_000_000,
/// },
/// ];
/// let metrics = collect_node_metrics(&disks);
/// assert_eq!(metrics.len(), 3);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::node_disk` module.
/// Returns a vector of Prometheus metrics for all disks.
pub fn collect_node_metrics(disks: &[DiskStats]) -> Vec<PrometheusMetric> {
if disks.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(disks.len() * 3);
for disk in disks {
let server_label: Cow<'static, str> = Cow::Owned(disk.server.clone());
let drive_label: Cow<'static, str> = Cow::Owned(disk.drive.clone());
metrics.push(
PrometheusMetric::new(METRIC_TOTAL, MetricType::Gauge, HELP_TOTAL, disk.total_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_TOTAL_BYTES_MD, disk.total_bytes as f64)
.with_label("server", server_label.clone())
.with_label("drive", drive_label.clone()),
);
metrics.push(
PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, disk.used_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_USED_BYTES_MD, disk.used_bytes as f64)
.with_label("server", server_label.clone())
.with_label("drive", drive_label.clone()),
);
metrics.push(
PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, disk.free_bytes as f64)
PrometheusMetric::from_descriptor(&NODE_DISK_FREE_BYTES_MD, disk.free_bytes as f64)
.with_label("server", server_label)
.with_label("drive", drive_label),
);
@@ -140,22 +108,24 @@ mod tests {
assert_eq!(metrics.len(), 6);
// Verify node1 disk1 total bytes
let node1_total_name = NODE_DISK_TOTAL_BYTES_MD.get_full_metric_name();
let node1_total = metrics.iter().find(|m| {
m.name == METRIC_TOTAL
m.name == node1_total_name
&& m.value == 1000000.0
&& m.labels.iter().any(|(k, v)| *k == "server" && v == "node1:9000")
&& m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk1")
});
assert!(node1_total.is_some());
assert_eq!(node1_total.map(|m| m.value), Some(1000000.0));
// Verify node2 disk2 used bytes
let node2_used_name = NODE_DISK_USED_BYTES_MD.get_full_metric_name();
let node2_used = metrics.iter().find(|m| {
m.name == METRIC_USED
m.name == node2_used_name
&& m.value == 800000.0
&& m.labels.iter().any(|(k, v)| *k == "server" && v == "node2:9000")
&& m.labels.iter().any(|(k, v)| *k == "drive" && v == "/data/disk2")
});
assert!(node2_used.is_some());
assert_eq!(node2_used.map(|m| m.value), Some(800000.0));
}
#[test]
@@ -0,0 +1,91 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Notification metrics collector.
//!
//! Collects notification system metrics including events sent,
//! errors, and skipped events.
use crate::format::PrometheusMetric;
use crate::metrics_type::cluster_notification::{
NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
};
/// Notification statistics.
#[derive(Debug, Clone, Default)]
pub struct NotificationStats {
/// Number of concurrent send operations in progress
pub current_send_in_progress: u64,
/// Total number of events that encountered errors
pub events_errors_total: u64,
/// Total number of events successfully sent
pub events_sent_total: u64,
/// Total number of events skipped
pub events_skipped_total: u64,
}
/// Collects notification metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::cluster_notification` module.
/// Returns a vector of Prometheus metrics for notification statistics.
pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, stats.current_send_in_progress as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, stats.events_errors_total as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_MD, stats.events_sent_total as f64),
PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, stats.events_skipped_total as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_notification_metrics() {
let stats = NotificationStats {
current_send_in_progress: 5,
events_errors_total: 10,
events_sent_total: 10000,
events_skipped_total: 50,
};
let metrics = collect_notification_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 4);
let sent = metrics.iter().find(|m| m.value == 10000.0);
assert!(sent.is_some());
let errors = metrics.iter().find(|m| m.value == 10.0);
assert!(errors.is_some());
}
#[test]
fn test_collect_notification_metrics_default() {
let stats = NotificationStats::default();
let metrics = collect_notification_metrics(&stats);
assert_eq!(metrics.len(), 4);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,107 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Replication metrics collector.
//!
//! Collects cluster-wide replication metrics including queue stats,
//! data transfer rates, and worker information.
use crate::format::PrometheusMetric;
use crate::metrics_type::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
/// Number of active replication workers
pub active_workers: u64,
/// Current data transfer rate in bytes/sec
pub current_data_transfer_rate: f64,
/// Bytes queued in the last full minute
pub last_minute_queued_bytes: u64,
/// Objects queued in the last full minute
pub last_minute_queued_count: u64,
/// Maximum active workers seen since server start
pub max_active_workers: u64,
/// Maximum bytes queued since server start
pub max_queued_bytes: u64,
/// Maximum objects queued since server start
pub max_queued_count: u64,
/// Maximum data transfer rate seen since server start
pub max_data_transfer_rate: f64,
/// Objects in replication backlog in the last 5 minutes
pub recent_backlog_count: u64,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD, stats.last_minute_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_MD, stats.max_active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_MD, stats.max_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_MD, stats.max_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_DATA_TRANSFER_RATE_MD, stats.max_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_RECENT_BACKLOG_COUNT_MD, stats.recent_backlog_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
active_workers: 10,
current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
last_minute_queued_bytes: 1024 * 1024 * 100, // 100 MB
last_minute_queued_count: 500,
max_active_workers: 20,
max_queued_bytes: 1024 * 1024 * 500, // 500 MB
max_queued_count: 2000,
max_data_transfer_rate: 1024.0 * 1024.0 * 10.0, // 10 MB/s
recent_backlog_count: 1500,
};
let metrics = collect_replication_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 9);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
let active = metrics.iter().find(|m| m.name == active_name);
assert!(active.is_some());
assert_eq!(active.map(|m| m.value), Some(10.0));
}
#[test]
fn test_collect_replication_metrics_default() {
let stats = ReplicationStats::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 9);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
+174
View File
@@ -0,0 +1,174 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! API request metrics collector.
//!
//! Collects API request metrics including request counts, errors,
//! latency, and traffic statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::request::*;
/// API request statistics for a specific API endpoint.
#[derive(Debug, Clone, Default)]
pub struct ApiRequestStats {
/// API name (e.g., "GetObject", "PutObject")
pub name: String,
/// Request type (e.g., "s3", "admin")
pub req_type: String,
/// Number of requests currently in flight
pub in_flight: u64,
/// Total number of requests
pub total: u64,
/// Total number of errors (4xx + 5xx)
pub errors_total: u64,
/// Total number of 5xx errors
pub errors_5xx: u64,
/// Total number of 4xx errors
pub errors_4xx: u64,
/// Total number of canceled requests
pub canceled: u64,
/// TTFB distribution by bucket (le label)
pub ttfb_distribution: Vec<(String, f64)>,
/// Bytes sent
pub sent_bytes: u64,
/// Bytes received
pub recv_bytes: u64,
}
/// Collects API request metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for API request statistics.
pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
for stat in stats {
// In-flight requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 5xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 4xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Canceled requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// TTFB distribution (histogram buckets)
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
// Traffic metrics
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_MD, stat.sent_bytes as f64)
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_MD, stat.recv_bytes as f64)
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_request_metrics() {
let stats = vec![ApiRequestStats {
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
total: 10000,
errors_total: 50,
errors_5xx: 10,
errors_4xx: 40,
canceled: 5,
ttfb_distribution: vec![
("0.1".to_string(), 5000.0),
("0.5".to_string(), 8000.0),
("1.0".to_string(), 9500.0),
("+Inf".to_string(), 10000.0),
],
sent_bytes: 1024 * 1024 * 500, // 500 MB
recv_bytes: 1024 * 1024 * 100, // 100 MB
}];
let metrics = collect_request_metrics(&stats);
report_metrics(&metrics);
// 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
assert_eq!(metrics.len(), 12);
let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
let total = metrics.iter().find(|m| m.name == total_name);
assert!(total.is_some());
assert_eq!(total.map(|m| m.value), Some(10000.0));
let in_flight_name = API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name();
let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
assert!(in_flight.is_some());
assert_eq!(in_flight.map(|m| m.value), Some(10.0));
}
#[test]
fn test_collect_request_metrics_empty() {
let stats: Vec<ApiRequestStats> = vec![];
let metrics = collect_request_metrics(&stats);
assert!(metrics.is_empty());
}
}
+27 -70
View File
@@ -16,17 +16,21 @@
//!
//! Collects system-level metrics for the RustFS process including
//! CPU usage, memory consumption, and process uptime.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::process_resource`
//! to avoid duplication of metric names, types, and help text.
use crate::MetricType;
use crate::format::PrometheusMetric;
use crate::metrics_type::process_resource::*;
/// Resource statistics for a RustFS process.
/// Resource statistics for metrics collection.
///
/// This struct encapsulates the resource usage data that can be
/// collected from the operating system for the current process.
/// This struct provides a decoupled interface for collecting resource metrics
/// without depending on specific internal types. HTTP handlers should populate
/// this struct from their available data sources.
#[derive(Debug, Clone, Default)]
pub struct ResourceStats {
/// CPU usage as a percentage (0.0 to 100.0+)
/// CPU usage as a percentage (can exceed 100% on multi-core systems)
pub cpu_percent: f64,
/// Resident memory usage in bytes
pub memory_bytes: u64,
@@ -34,65 +38,16 @@ pub struct ResourceStats {
pub uptime_seconds: u64,
}
// Static metric definitions
const METRIC_CPU: &str = "rustfs_process_cpu_percent";
const METRIC_MEMORY: &str = "rustfs_process_memory_bytes";
const METRIC_UPTIME: &str = "rustfs_process_uptime_seconds";
const HELP_CPU: &str = "CPU usage of the RustFS process as a percentage";
const HELP_MEMORY: &str = "Resident memory usage of the RustFS process in bytes";
const HELP_UPTIME: &str = "Uptime of the RustFS process in seconds";
/// Number of metrics produced by this collector.
const METRIC_COUNT: usize = 3;
/// Collects system resource metrics from the provided statistics.
/// Collects resource metrics from the provided resource statistics.
///
/// # Metrics Produced
///
/// - `rustfs_process_cpu_percent`: CPU usage as a percentage
/// - `rustfs_process_memory_bytes`: Resident memory usage in bytes
/// - `rustfs_process_uptime_seconds`: Process uptime in seconds
///
/// # Arguments
///
/// * `stats` - Resource statistics for the current process
///
/// # Example
///
/// ```
/// use rustfs_metrics::collectors::{collect_resource_metrics, ResourceStats};
///
/// let stats = ResourceStats {
/// cpu_percent: 25.5,
/// memory_bytes: 1024 * 1024 * 512, // 512 MB
/// uptime_seconds: 3600,
/// };
/// let metrics = collect_resource_metrics(&stats);
/// assert_eq!(metrics.len(), 3);
/// ```
#[must_use]
#[inline]
/// Uses the metric descriptors from `metrics_type::process_resource` module.
/// Returns a vector of Prometheus metrics for resource statistics.
pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(METRIC_COUNT);
metrics.push(PrometheusMetric::new(METRIC_CPU, MetricType::Gauge, HELP_CPU, stats.cpu_percent));
metrics.push(PrometheusMetric::new(
METRIC_MEMORY,
MetricType::Gauge,
HELP_MEMORY,
stats.memory_bytes as f64,
));
metrics.push(PrometheusMetric::new(
METRIC_UPTIME,
MetricType::Gauge,
HELP_UPTIME,
stats.uptime_seconds as f64,
));
metrics
vec![
PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent),
PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
]
}
#[cfg(test)]
@@ -114,19 +69,21 @@ mod tests {
assert_eq!(metrics.len(), 3);
// Verify CPU metric
let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 45.5);
assert!(cpu.is_some());
assert_eq!(cpu.map(|m| m.value), Some(45.5));
// Verify memory metric
let memory = metrics.iter().find(|m| m.name == METRIC_MEMORY);
let memory_metric_name = PROCESS_MEMORY_BYTES_MD.get_full_metric_name();
let memory = metrics
.iter()
.find(|m| m.name == memory_metric_name && m.value == (1024 * 1024 * 256) as f64);
assert!(memory.is_some());
assert_eq!(memory.map(|m| m.value), Some((1024 * 1024 * 256) as f64));
// Verify uptime metric
let uptime = metrics.iter().find(|m| m.name == METRIC_UPTIME);
let uptime_metric_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
let uptime = metrics.iter().find(|m| m.name == uptime_metric_name && m.value == 7200.0);
assert!(uptime.is_some());
assert_eq!(uptime.map(|m| m.value), Some(7200.0));
}
#[test]
@@ -155,9 +112,9 @@ mod tests {
let metrics = collect_resource_metrics(&stats);
report_metrics(&metrics);
let cpu = metrics.iter().find(|m| m.name == METRIC_CPU);
let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name();
let cpu = metrics.iter().find(|m| m.name == cpu_metric_name && m.value == 150.0);
assert!(cpu.is_some());
assert_eq!(cpu.map(|m| m.value), Some(150.0));
}
#[test]
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Scanner metrics collector.
//!
//! Collects background scanner metrics including bucket scans,
//! directory scans, and object scans.
use crate::format::PrometheusMetric;
use crate::metrics_type::scanner::{
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
};
/// Scanner statistics.
#[derive(Debug, Clone, Default)]
pub struct ScannerStats {
/// Number of bucket scans finished
pub bucket_scans_finished: u64,
/// Number of bucket scans started
pub bucket_scans_started: u64,
/// Number of directories scanned
pub directories_scanned: u64,
/// Number of objects scanned
pub objects_scanned: u64,
/// Number of object versions scanned
pub versions_scanned: u64,
/// Seconds since last scanner activity
pub last_activity_seconds: u64,
}
/// Collects scanner metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::scanner` module.
/// Returns a vector of Prometheus metrics for scanner statistics.
pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_scanner_metrics() {
let stats = ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
};
let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 6);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
let last_activity = metrics.iter().find(|m| m.value == 30.0);
assert!(last_activity.is_some());
}
#[test]
fn test_collect_scanner_metrics_default() {
let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 6);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,233 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! Statistics collection functions for metrics.
//!
//! This module contains functions that collect statistics from various
//! RustFS internal sources (storage layer, bucket monitor, system info)
//! and convert them to the Stats structs used by collectors.
use crate::collectors::{BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats};
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use std::sync::OnceLock;
use std::time::Instant;
use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
use tracing::{instrument, warn};
/// Process start time for calculating uptime.
static PROCESS_START: OnceLock<Instant> = OnceLock::new();
/// Get the process start time, initializing it on first call.
#[inline]
fn get_process_start() -> &'static Instant {
PROCESS_START.get_or_init(Instant::now)
}
/// Collect cluster statistics from the storage layer.
#[instrument]
pub async fn collect_cluster_stats() -> ClusterStats {
let Some(store) = new_object_layer_fn() else {
return ClusterStats::default();
};
let storage_info = store.storage_info().await;
let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
// Get bucket and object counts from data usage info
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!("Failed to load data usage from backend: {}", e);
// Fall back to bucket list for buckets_count, objects_count stays 0
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|e| {
warn!("Failed to list buckets for cluster metrics: {}", e);
Vec::new()
});
(buckets.len() as u64, 0)
}
};
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
}
}
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_data_usage_from_backend(store.clone()).await {
Ok(info) => Some(info),
Err(e) => {
warn!("Failed to load data usage for bucket metrics: {}", e);
None
}
};
// List all buckets
let buckets = match store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
{
Ok(buckets) => buckets,
Err(e) => {
warn!("Failed to list buckets for bucket metrics: {}", e);
return Vec::new();
}
};
let mut stats = Vec::with_capacity(buckets.len());
for bucket in buckets {
if bucket.name.starts_with('.') {
continue;
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
// Get quota from bucket metadata
let quota_bytes = match get_quota_config(&bucket.name).await {
Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
Err(_) => 0, // No quota configured or error
};
stats.push(BucketStats {
name: bucket.name,
size_bytes,
objects_count,
quota_bytes,
});
}
stats
}
/// Collect bucket replication bandwidth stats from the global monitor.
pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
let Some(monitor) = get_global_bucket_monitor() else {
return Vec::new();
};
monitor
.get_report(|_| true)
.bucket_stats
.into_iter()
.map(|(opts, details)| {
let target_arn = opts.replication_arn;
let limit_bytes_per_sec = u64::try_from(details.limit_bytes_per_sec).unwrap_or_else(|_| {
warn!(
"Invalid bandwidth limit value for target {:?}: {}",
target_arn, details.limit_bytes_per_sec
);
0
});
BucketReplicationBandwidthStats {
bucket: opts.name,
target_arn,
limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
}
})
.collect()
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let Some(store) = new_object_layer_fn() else {
return Vec::new();
};
let storage_info = store.storage_info().await;
storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
})
.collect()
}
/// Collect resource statistics for the current process.
///
/// Collects:
/// - Uptime: Calculated from process start time
/// - Memory: Process resident set size from sysinfo
/// - CPU: Process CPU usage percentage from sysinfo
#[inline]
pub fn collect_process_stats() -> ResourceStats {
let uptime_seconds = get_process_start().elapsed().as_secs();
// Use sysinfo for process metrics
let mut sys = System::new();
let pid = Pid::from_u32(std::process::id());
sys.refresh_processes_specifics(
ProcessesToUpdate::Some(&[pid]),
true,
ProcessRefreshKind::nothing().with_cpu().with_memory(),
);
if let Some(process) = sys.process(pid) {
ResourceStats {
cpu_percent: process.cpu_usage() as f64,
memory_bytes: process.memory(),
uptime_seconds,
}
} else {
// Fallback if process info unavailable
ResourceStats {
cpu_percent: 0.0,
memory_bytes: 0,
uptime_seconds,
}
}
}
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System CPU metrics collector.
//!
//! Collects CPU-related metrics including load average, idle time,
//! I/O wait, and CPU usage percentages.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_cpu`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_cpu::*;
/// CPU statistics for a node.
#[derive(Debug, Clone, Default)]
pub struct CpuStats {
/// Average CPU idle time (percentage, 0-100)
pub avg_idle: f64,
/// Average CPU I/O wait time (percentage, 0-100)
pub avg_iowait: f64,
/// CPU load average over 1 minute
pub load_avg: f64,
/// CPU load average as percentage
pub load_avg_perc: f64,
/// CPU nice time (percentage, 0-100)
pub nice: f64,
/// CPU steal time (percentage, 0-100)
pub steal: f64,
/// CPU system time (percentage, 0-100)
pub system: f64,
/// CPU user time (percentage, 0-100)
pub user: f64,
}
/// Collects CPU metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_cpu` module.
/// Returns a vector of Prometheus metrics for CPU statistics.
pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle),
PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IOWAIT_MD, stats.avg_iowait),
PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg),
PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc),
PrometheusMetric::from_descriptor(&SYS_CPU_NICE_MD, stats.nice),
PrometheusMetric::from_descriptor(&SYS_CPU_STEAL_MD, stats.steal),
PrometheusMetric::from_descriptor(&SYS_CPU_SYSTEM_MD, stats.system),
PrometheusMetric::from_descriptor(&SYS_CPU_USER_MD, stats.user),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_cpu_metrics() {
let stats = CpuStats {
avg_idle: 75.5,
avg_iowait: 2.3,
load_avg: 1.5,
load_avg_perc: 37.5,
nice: 0.5,
steal: 0.1,
system: 10.0,
user: 15.0,
};
let metrics = collect_cpu_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
// Verify that metric names are properly generated from descriptors
assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_cpu_")));
}
#[test]
fn test_collect_cpu_metrics_default() {
let stats = CpuStats::default();
let metrics = collect_cpu_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,228 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System drive metrics collector.
//!
//! Collects detailed drive/disk metrics including capacity, I/O statistics,
//! error counts, and health status.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_drive::*;
/// Detailed drive statistics for a single drive.
#[derive(Debug, Clone, Default)]
pub struct DriveDetailedStats {
/// Server identifier (e.g., "node1:9000")
pub server: String,
/// Drive path (e.g., "/data/disk1")
pub drive: String,
/// Total capacity in bytes
pub total_bytes: u64,
/// Used capacity in bytes
pub used_bytes: u64,
/// Free capacity in bytes
pub free_bytes: u64,
/// Used inodes
pub used_inodes: u64,
/// Free inodes
pub free_inodes: u64,
/// Total inodes
pub total_inodes: u64,
/// Total timeout errors
pub timeout_errors_total: u64,
/// Total I/O errors
pub io_errors_total: u64,
/// Total availability errors
pub availability_errors_total: u64,
/// Number of I/O operations waiting
pub waiting_io: u64,
/// API latency in microseconds
pub api_latency_micros: u64,
/// Health status (1=healthy, 0=unhealthy)
pub health: u8,
/// Reads per second
pub reads_per_sec: f64,
/// Kilobytes read per second
pub reads_kb_per_sec: f64,
/// Average read await time
pub reads_await: f64,
/// Writes per second
pub writes_per_sec: f64,
/// Kilobytes written per second
pub writes_kb_per_sec: f64,
/// Average write await time
pub writes_await: f64,
/// Percentage utilization
pub perc_util: f64,
}
/// Aggregate drive count statistics.
#[derive(Debug, Clone, Default)]
pub struct DriveCountStats {
/// Number of offline Drives
pub offline_count: u64,
/// Number of online drives
pub online_count: u64,
/// Total number of drives
pub total_count: u64,
}
/// Collects detailed drive metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for each drive.
pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
fn push_drive_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::MetricDescriptor,
value: f64,
server_label: &str,
drive_label: &str,
) {
metrics.push(
PrometheusMetric::from_descriptor(descriptor, value)
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
.with_label_owned(SERVER_LABEL, server_label.to_string()),
);
}
let mut metrics = Vec::with_capacity(stats.len() * 19);
for stat in stats {
let server_label = stat.server.as_str();
let drive_label = stat.drive.as_str();
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_TIMEOUT_ERRORS_MD,
stat.timeout_errors_total as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, stat.io_errors_total as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_AVAILABILITY_ERRORS_MD,
stat.availability_errors_total as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, stat.waiting_io as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_API_LATENCY_MD,
stat.api_latency_micros as f64,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, stat.reads_per_sec, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, stat.reads_kb_per_sec, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, stat.reads_await, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, stat.writes_per_sec, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_WRITES_KB_PER_SEC_MD,
stat.writes_kb_per_sec,
server_label,
drive_label,
);
push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, stat.writes_await, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, stat.perc_util, server_label, drive_label);
}
metrics
}
/// Collects drive count metrics (offline, online, total).
///
/// Returns a vector of Prometheus metrics for drive counts.
pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&DRIVE_OFFLINE_COUNT_MD, stats.offline_count as f64),
PrometheusMetric::from_descriptor(&DRIVE_ONLINE_COUNT_MD, stats.online_count as f64),
PrometheusMetric::from_descriptor(&DRIVE_COUNT_MD, stats.total_count as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_drive_detailed_metrics() {
let stats = vec![DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
used_inodes: 100000,
free_inodes: 900000,
total_inodes: 1000000,
timeout_errors_total: 5,
io_errors_total: 10,
availability_errors_total: 2,
waiting_io: 3,
api_latency_micros: 1500,
health: 1,
reads_per_sec: 100.0,
reads_kb_per_sec: 1024.0,
reads_await: 5.5,
writes_per_sec: 50.0,
writes_kb_per_sec: 512.0,
writes_await: 10.2,
perc_util: 75.5,
}];
let metrics = collect_drive_detailed_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 19);
// Verify total bytes metric
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
assert!(total_bytes.is_some());
assert_eq!(total_bytes.map(|m| m.value), Some(1024.0 * 1024.0 * 1024.0 * 100.0));
}
#[test]
fn test_collect_drive_count_metrics() {
let stats = DriveCountStats {
offline_count: 2,
online_count: 8,
total_count: 10,
};
let metrics = collect_drive_count_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 3);
// Verify offline count
let offline_name = DRIVE_OFFLINE_COUNT_MD.get_full_metric_name();
let offline = metrics.iter().find(|m| m.name == offline_name);
assert!(offline.is_some());
assert_eq!(offline.map(|m| m.value), Some(2.0));
}
}
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System memory metrics collector.
//!
//! Collects memory-related metrics including total, used, free,
//! buffers, cache, shared, and available memory.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_memory`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_memory::*;
/// Memory statistics for a node.
#[derive(Debug, Clone, Default)]
pub struct MemoryStats {
/// Total memory in bytes
pub total: u64,
/// Used memory in bytes
pub used: u64,
/// Used memory percentage (0-100)
pub used_perc: f64,
/// Free memory in bytes
pub free: u64,
/// Buffer memory in bytes
pub buffers: u64,
/// Cache memory in bytes
pub cache: u64,
/// Shared memory in bytes
pub shared: u64,
/// Available memory in bytes
pub available: u64,
}
/// Collects memory metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_memory` module.
/// Returns a vector of Prometheus metrics for memory statistics.
pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64),
PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64),
PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc),
PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64),
PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64),
PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64),
PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64),
PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_memory_metrics() {
let stats = MemoryStats {
total: 16 * 1024 * 1024 * 1024, // 16 GB
used: 8 * 1024 * 1024 * 1024, // 8 GB
used_perc: 50.0,
free: 4 * 1024 * 1024 * 1024, // 4 GB
buffers: 1024 * 1024 * 512, // 512 MB
cache: 2 * 1024 * 1024 * 1024, // 2 GB
shared: 1024 * 1024 * 256, // 256 MB
available: 6 * 1024 * 1024 * 1024, // 6 GB
};
let metrics = collect_memory_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
assert!(metrics.iter().all(|m| m.name.starts_with("gauge.rustfs_system_memory_")));
}
#[test]
fn test_collect_memory_metrics_default() {
let stats = MemoryStats::default();
let metrics = collect_memory_metrics(&stats);
assert_eq!(metrics.len(), 8);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,90 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System network metrics collector.
//!
//! Collects internode network metrics including errors, dial times,
//! and bytes sent/received.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::system_network`
//! to avoid duplication of metric names, types, and help text.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_network::*;
/// Network statistics for internode communication.
#[derive(Debug, Clone, Default)]
pub struct NetworkStats {
/// Total number of failed internode calls
pub internode_errors_total: u64,
/// Total number of TCP dial timeouts and errors
pub internode_dial_errors_total: u64,
/// Average dial time in nanoseconds
pub internode_dial_avg_time_nanos: u64,
/// Total bytes sent to other nodes
pub internode_sent_bytes_total: u64,
/// Total bytes received from other nodes
pub internode_recv_bytes_total: u64,
}
/// Collects network metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::system_network` module.
/// Returns a vector of Prometheus metrics for network statistics.
pub fn collect_network_metrics(stats: &NetworkStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64),
PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64),
PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_network_metrics() {
let stats = NetworkStats {
internode_errors_total: 10,
internode_dial_errors_total: 5,
internode_dial_avg_time_nanos: 1_500_000, // 1.5ms
internode_sent_bytes_total: 1024 * 1024 * 100, // 100 MB
internode_recv_bytes_total: 1024 * 1024 * 200, // 200 MB
};
let metrics = collect_network_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 5);
assert!(metrics.iter().all(|m| m.name.contains("internode")));
}
#[test]
fn test_collect_network_metrics_default() {
let stats = NetworkStats::default();
let metrics = collect_network_metrics(&stats);
assert_eq!(metrics.len(), 5);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
@@ -0,0 +1,145 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
//! System process metrics collector.
//!
//! Collects process-level metrics including file descriptors, memory,
//! syscalls, and runtime statistics.
use crate::format::PrometheusMetric;
use crate::metrics_type::system_process::*;
/// Process statistics for the RustFS server process.
#[derive(Debug, Clone, Default)]
pub struct ProcessStats {
/// Total read locks held
pub locks_read_total: u64,
/// Total write locks held
pub locks_write_total: u64,
/// Total CPU time in seconds
pub cpu_total_seconds: f64,
/// Total number of async tasks (goroutines equivalent)
pub go_routine_total: u64,
/// Total bytes read via read syscalls (rchar)
pub io_rchar_bytes: u64,
/// Total bytes actually read from storage
pub io_read_bytes: u64,
/// Total bytes written via write syscalls (wchar)
pub io_wchar_bytes: u64,
/// Total bytes actually written to storage
pub io_write_bytes: u64,
/// Process start time in seconds since Unix epoch
pub start_time_seconds: u64,
/// Process uptime in seconds
pub uptime_seconds: u64,
/// File descriptor limit
pub file_descriptor_limit_total: u64,
/// Open file descriptors count
pub file_descriptor_open_total: u64,
/// Total read syscalls
pub syscall_read_total: u64,
/// Total write syscalls
pub syscall_write_total: u64,
/// Resident memory size in bytes
pub resident_memory_bytes: u64,
/// Virtual memory size in bytes
pub virtual_memory_bytes: u64,
/// Maximum virtual memory size in bytes
pub virtual_memory_max_bytes: u64,
}
/// Collects process metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for process statistics.
pub fn collect_process_metrics(stats: &ProcessStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds),
PrometheusMetric::from_descriptor(&PROCESS_GO_ROUTINE_TOTAL_MD, stats.go_routine_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_RCHAR_BYTES_MD, stats.io_rchar_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_READ_BYTES_MD, stats.io_read_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_WCHAR_BYTES_MD, stats.io_wchar_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_IO_WRITE_BYTES_MD, stats.io_write_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_START_TIME_SECONDS_MD, stats.start_time_seconds as f64),
PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64),
PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD, stats.file_descriptor_limit_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD, stats.file_descriptor_open_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_READ_TOTAL_MD, stats.syscall_read_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_SYSCALL_WRITE_TOTAL_MD, stats.syscall_write_total as f64),
PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64),
PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64),
]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::report_metrics;
#[test]
fn test_collect_process_metrics() {
let stats = ProcessStats {
locks_read_total: 100,
locks_write_total: 50,
cpu_total_seconds: 1234.56,
go_routine_total: 200,
io_rchar_bytes: 1024 * 1024 * 500,
io_read_bytes: 1024 * 1024 * 400,
io_wchar_bytes: 1024 * 1024 * 300,
io_write_bytes: 1024 * 1024 * 250,
start_time_seconds: 1700000000,
uptime_seconds: 86400,
file_descriptor_limit_total: 65536,
file_descriptor_open_total: 1500,
syscall_read_total: 100000,
syscall_write_total: 50000,
resident_memory_bytes: 1024 * 1024 * 512,
virtual_memory_bytes: 1024 * 1024 * 1024,
virtual_memory_max_bytes: 1024 * 1024 * 2048,
};
let metrics = collect_process_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 17);
// Verify uptime
let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name();
let uptime = metrics.iter().find(|m| m.name == uptime_name);
assert!(uptime.is_some());
assert_eq!(uptime.map(|m| m.value), Some(86400.0));
// Verify file descriptors
let fd_open_name = PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD.get_full_metric_name();
let fd_open = metrics.iter().find(|m| m.name == fd_open_name);
assert!(fd_open.is_some());
assert_eq!(fd_open.map(|m| m.value), Some(1500.0));
}
#[test]
fn test_collect_process_metrics_default() {
let stats = ProcessStats::default();
let metrics = collect_process_metrics(&stats);
assert_eq!(metrics.len(), 17);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
}
}
}
+71 -12
View File
@@ -20,21 +20,50 @@
use crate::MetricType;
use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge};
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
static NAME_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
static HELP_CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
fn intern_string(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
let cache = cache.get_or_init(Default::default);
let mut cache = match cache.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if let Some(existing) = cache.get(value) {
existing
} else {
let value = Box::leak(value.to_string().into_boxed_str());
cache.insert(value.to_string(), value);
value
}
}
fn into_static_str(cache: &OnceLock<Mutex<HashMap<String, &'static str>>>, value: &str) -> &'static str {
intern_string(cache, value)
}
/// Report metrics using the `metrics` crate.
///
/// This function iterates over the provided metrics and reports them using
/// the `metrics` crate's API. This allows integration with various metrics
/// exporters (e.g., Prometheus) that are configured globally.
///
pub fn report_metrics(metrics: &[PrometheusMetric]) {
for metric in metrics {
let name = into_static_str(&NAME_CACHE, &metric.name);
let help = into_static_str(&HELP_CACHE, &metric.help);
// Register metric description (help text)
// Note: In a real-world scenario, descriptions should ideally be registered once at startup.
// However, the `metrics` crate handles duplicate registrations gracefully.
match metric.metric_type {
MetricType::Counter => describe_counter!(metric.name, metric.help),
MetricType::Gauge => describe_gauge!(metric.name, metric.help),
MetricType::Histogram => describe_histogram!(metric.name, metric.help),
MetricType::Counter => describe_counter!(name, help),
MetricType::Gauge => describe_gauge!(name, help),
MetricType::Histogram => describe_histogram!(name, help),
}
// Convert labels to the format expected by `metrics` crate
@@ -50,15 +79,15 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) {
//
// Since `metrics` 0.21+, `Counter` has an `absolute` method which sets the counter to a specific value.
// This is useful for mirroring an external counter.
let counter = counter!(metric.name, &labels);
let counter = counter!(name, &labels);
counter.absolute(metric.value as u64);
}
MetricType::Gauge => {
let gauge = gauge!(metric.name, &labels);
let gauge = gauge!(name, &labels);
gauge.set(metric.value);
}
MetricType::Histogram => {
let histogram = metrics::histogram!(metric.name, &labels);
let histogram = metrics::histogram!(name, &labels);
histogram.record(metric.value);
}
}
@@ -67,17 +96,17 @@ pub fn report_metrics(metrics: &[PrometheusMetric]) {
/// A single Prometheus metric with labels and value.
///
/// This struct is optimized for performance by using `&'static str` for
/// the name and help text, which are typically compile-time constants.
/// This struct is optimized for performance by using `Cow<'static, str>` for
/// the name and help text, which allows both static strings and owned strings.
/// Labels use `Cow<'static, str>` to avoid allocations when possible.
#[derive(Debug, Clone)]
pub struct PrometheusMetric {
/// The metric name (e.g., "http_requests_total").
pub name: &'static str,
pub name: Cow<'static, str>,
/// The type of this metric (counter, gauge, or histogram).
pub metric_type: MetricType,
/// Human-readable description shown in Prometheus UI.
pub help: &'static str,
pub help: Cow<'static, str>,
/// Key-value label pairs for this metric instance.
/// Uses Cow to avoid allocations for static label keys.
pub labels: Vec<(&'static str, Cow<'static, str>)>,
@@ -92,9 +121,39 @@ impl PrometheusMetric {
#[inline]
pub const fn new(name: &'static str, metric_type: MetricType, help: &'static str, value: f64) -> Self {
Self {
name,
name: Cow::Borrowed(name),
metric_type,
help,
help: Cow::Borrowed(help),
labels: Vec::new(),
value,
}
}
/// Creates a new metric with owned strings for name and help.
///
/// Use this when the metric name or help text is dynamically generated.
#[inline]
pub fn new_owned(name: String, metric_type: MetricType, help: String, value: f64) -> Self {
Self {
name: Cow::Owned(name),
metric_type,
help: Cow::Owned(help),
labels: Vec::new(),
value,
}
}
/// Creates a new metric from a MetricDescriptor.
///
/// This is the recommended way to create metrics when using MetricDescriptor
/// from the metrics_type module.
#[inline]
pub fn from_descriptor(descriptor: &crate::MetricDescriptor, value: f64) -> Self {
let help = intern_string(&HELP_CACHE, &descriptor.help);
Self {
name: Cow::Owned(descriptor.get_full_metric_name()),
metric_type: descriptor.metric_type,
help: Cow::Borrowed(help),
labels: Vec::new(),
value,
}
@@ -0,0 +1,78 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
/// Total raw storage capacity across all disks in bytes
pub static CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_raw_total_bytes".to_string()),
"Total raw storage capacity in bytes across all disks",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total usable storage capacity in bytes (accounting for erasure coding)
pub static CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_usable_total_bytes".to_string()),
"Total usable storage capacity in bytes (accounting for erasure coding)",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total used storage capacity in bytes
pub static CLUSTER_CAPACITY_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_used_bytes".to_string()),
"Total used storage capacity in bytes",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total free storage capacity in bytes
pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_free_bytes".to_string()),
"Total free storage capacity in bytes",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total number of objects in the cluster
pub static CLUSTER_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("objects_total".to_string()),
"Total number of objects in the cluster",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total number of buckets in the cluster
pub static CLUSTER_BUCKETS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("buckets_total".to_string()),
"Total number of buckets in the cluster",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
@@ -22,6 +22,33 @@ pub const POOL_ID_L: &str = "pool_id";
/// The label for the pool ID
pub const SET_ID_L: &str = "set_id";
pub static ERASURE_SET_SIZE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetSize,
"Total number of drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_PARITY_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetParity,
"Parity drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_DATA_SHARDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetDataShards,
"Data shards in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET,
)
});
pub static ERASURE_SET_OVERALL_WRITE_QUORUM_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ErasureSetOverallWriteQuorum,
@@ -183,6 +183,9 @@ pub enum MetricName {
ConfigStandardParity,
// Erasure coding set related metrics
ErasureSetSize,
ErasureSetParity,
ErasureSetDataShards,
ErasureSetOverallWriteQuorum,
ErasureSetOverallHealth,
ErasureSetReadQuorum,
@@ -502,6 +505,9 @@ impl MetricName {
Self::ConfigStandardParity => "standard_parity".to_string(),
// Erasure coding set related metrics
Self::ErasureSetSize => "size".to_string(),
Self::ErasureSetParity => "parity".to_string(),
Self::ErasureSetDataShards => "data_shards".to_string(),
Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(),
Self::ErasureSetOverallHealth => "overall_health".to_string(),
Self::ErasureSetReadQuorum => "read_quorum".to_string(),
+4
View File
@@ -15,6 +15,7 @@
pub mod audit;
pub mod bucket;
pub mod bucket_replication;
pub mod cluster;
pub mod cluster_config;
pub mod cluster_erasure_set;
pub mod cluster_health;
@@ -24,6 +25,9 @@ pub mod cluster_usage;
pub mod entry;
pub mod ilm;
pub mod logger_webhook;
pub mod node_bucket;
pub mod node_disk;
pub mod process_resource;
pub mod replication;
pub mod request;
pub mod scanner;
@@ -0,0 +1,50 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
const BUCKET_LABEL: &str = "bucket";
/// Total bytes used by the bucket
pub static BUCKET_USAGE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("usage_bytes".to_string()),
"Total bytes used by the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
/// Total number of objects in the bucket
pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("objects_total".to_string()),
"Total number of objects in the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
/// Quota limit in bytes for the bucket
pub static BUCKET_QUOTA_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("quota_bytes".to_string()),
"Quota limit in bytes for the bucket",
&[BUCKET_LABEL],
subsystems::BUCKET_API,
)
});
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
use std::sync::LazyLock;
const SERVER_LABEL: &str = "server";
const DRIVE_LABEL: &str = "drive";
/// Total disk capacity in bytes
pub static NODE_DISK_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_total_bytes".to_string()),
"Total disk capacity in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
/// Used disk space in bytes
pub static NODE_DISK_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_used_bytes".to_string()),
"Used disk space in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
/// Free disk space in bytes
pub static NODE_DISK_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("disk_free_bytes".to_string()),
"Free disk space in bytes",
&[SERVER_LABEL, DRIVE_LABEL],
MetricSubsystem::new("/node"),
)
});
@@ -0,0 +1,48 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code)]
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
use std::sync::LazyLock;
/// CPU usage of the RustFS process as a percentage
pub static PROCESS_CPU_PERCENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cpu_percent".to_string()),
"CPU usage of the RustFS process as a percentage",
&[],
MetricSubsystem::new("/process"),
)
});
/// Resident memory usage of the RustFS process in bytes
pub static PROCESS_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("memory_bytes".to_string()),
"Resident memory usage of the RustFS process in bytes",
&[],
MetricSubsystem::new("/process"),
)
});
/// Uptime of the RustFS process in seconds
pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("uptime_seconds".to_string()),
"Uptime of the RustFS process in seconds",
&[],
MetricSubsystem::new("/process"),
)
});
@@ -16,6 +16,12 @@
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
/// name label
pub const NAME_LABEL: &str = "name";
/// type label
pub const TYPE_LABEL: &str = "type";
/// le label (for histogram buckets)
pub const LE_LABEL: &str = "le";
pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
@@ -19,6 +19,8 @@ use std::sync::LazyLock;
/// drive related labels
pub const DRIVE_LABEL: &str = "drive";
/// server label
pub const SERVER_LABEL: &str = "server";
/// pool index label
pub const POOL_INDEX_LABEL: &str = "pool_index";
/// set index label