gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(obs): label node-local metrics by server (#5465)

Browse Source 
Add stable server labels to node-local Prometheus metrics and OTLP resource attributes so dashboards can distinguish per-node CPU, memory, host network, and internode traffic series.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-30 12:57:22 +08:00
committed by GitHub
parent ad7663afd1
commit 30dc04c94b
21 changed files with 640 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/schema/process_resource.rs
+4 -3
View File
@@ -14,6 +14,7 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md};
use std::sync::LazyLock;
@@ -22,7 +23,7 @@ pub static PROCESS_CPU_PERCENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
new_gauge_md(
MetricName::Custom("cpu_percent".to_string()),
"CPU usage of the RustFS process as a percentage",
&[],
&[SERVER_LABEL],
MetricSubsystem::new("/process"),
)
});
@@ -32,7 +33,7 @@ pub static PROCESS_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
new_gauge_md(
MetricName::Custom("memory_bytes".to_string()),
"Resident memory usage of the RustFS process in bytes",
&[],
&[SERVER_LABEL],
MetricSubsystem::new("/process"),
)
});
@@ -42,7 +43,7 @@ pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_gauge_md(
MetricName::Custom("uptime_seconds".to_string()),
"Uptime of the RustFS process in seconds",
&[],
&[SERVER_LABEL],
MetricSubsystem::new("/process"),
)
});
crates/obs/src/metrics/schema/system_cpu.rs
+24 -11
View File
@@ -14,24 +14,37 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
/// CPU system-related metric descriptors
use std::sync::LazyLock;
pub static SYS_CPU_AVG_IDLE_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIdle, "Average CPU idle time", &[], subsystems::SYSTEM_CPU));
pub static SYS_CPU_AVG_IDLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::SysCPUAvgIdle,
"Average CPU idle time",
&[SERVER_LABEL],
subsystems::SYSTEM_CPU,
)
});
pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIOWait, "Average CPU IOWait time", &[], subsystems::SYSTEM_CPU));
pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::SysCPUAvgIOWait,
"Average CPU IOWait time",
&[SERVER_LABEL],
subsystems::SYSTEM_CPU,
)
});
pub static SYS_CPU_LOAD_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[], subsystems::SYSTEM_CPU));
LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[SERVER_LABEL], subsystems::SYSTEM_CPU));
pub static SYS_CPU_LOAD_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::SysCPULoadPerc,
"CPU load average 1min (percentage)",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_CPU,
)
});
@@ -40,19 +53,19 @@ pub static SYS_CPU_USAGE_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
new_gauge_md(
MetricName::Custom("usage_perc".to_string()),
"Total CPU usage percentage across all measured CPU time categories",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_CPU,
)
});
pub static SYS_CPU_NICE_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[], subsystems::SYSTEM_CPU));
LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[SERVER_LABEL], subsystems::SYSTEM_CPU));
pub static SYS_CPU_STEAL_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[], subsystems::SYSTEM_CPU));
LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[SERVER_LABEL], subsystems::SYSTEM_CPU));
pub static SYS_CPU_SYSTEM_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[], subsystems::SYSTEM_CPU));
LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[SERVER_LABEL], subsystems::SYSTEM_CPU));
pub static SYS_CPU_USER_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[], subsystems::SYSTEM_CPU));
LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[SERVER_LABEL], subsystems::SYSTEM_CPU));
crates/obs/src/metrics/schema/system_memory.rs
+44 -13
View File
@@ -14,43 +14,74 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
/// Total memory available on the node
pub static MEM_TOTAL_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemTotal, "Total memory on the node", &[], subsystems::SYSTEM_MEMORY));
pub static MEM_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemTotal,
"Total memory on the node",
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
/// Memory currently in use on the node
pub static MEM_USED_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[], subsystems::SYSTEM_MEMORY));
LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[SERVER_LABEL], subsystems::SYSTEM_MEMORY));
/// Percentage of total memory currently in use
pub static MEM_USED_PERC_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemUsedPerc,
"Used memory percentage on the node",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
/// Memory not currently in use and available for allocation
pub static MEM_FREE_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[], subsystems::SYSTEM_MEMORY));
LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[SERVER_LABEL], subsystems::SYSTEM_MEMORY));
/// Memory used for file buffers by the kernel
pub static MEM_BUFFERS_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemBuffers, "Buffers memory on the node", &[], subsystems::SYSTEM_MEMORY));
pub static MEM_BUFFERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemBuffers,
"Buffers memory on the node",
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
/// Memory used for caching file data by the kernel
pub static MEM_CACHE_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemCache, "Cache memory on the node", &[], subsystems::SYSTEM_MEMORY));
pub static MEM_CACHE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemCache,
"Cache memory on the node",
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
/// Memory shared between multiple processes
pub static MEM_SHARED_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemShared, "Shared memory on the node", &[], subsystems::SYSTEM_MEMORY));
pub static MEM_SHARED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemShared,
"Shared memory on the node",
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
/// Estimate of memory available for new applications without swapping
pub static MEM_AVAILABLE_MD: LazyLock<MetricDescriptor> =
LazyLock::new(|| new_gauge_md(MetricName::MemAvailable, "Available memory on the node", &[], subsystems::SYSTEM_MEMORY));
pub static MEM_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::MemAvailable,
"Available memory on the node",
&[SERVER_LABEL],
subsystems::SYSTEM_MEMORY,
)
});
crates/obs/src/metrics/schema/system_network.rs
+6 -5
View File
@@ -14,6 +14,7 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
@@ -22,7 +23,7 @@ pub static INTERNODE_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::InternodeErrorsTotal,
"Total number of failed internode calls",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_NETWORK_INTERNODE,
)
});
@@ -32,7 +33,7 @@ pub static INTERNODE_DIAL_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock
new_counter_md(
MetricName::InternodeDialErrorsTotal,
"Total number of internode TCP dial timeouts and errors",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_NETWORK_INTERNODE,
)
});
@@ -42,7 +43,7 @@ pub static INTERNODE_DIAL_AVG_TIME_NANOS_MD: LazyLock<MetricDescriptor> = LazyLo
new_gauge_md(
MetricName::InternodeDialAvgTimeNanos,
"Average dial time of internode TCP calls in nanoseconds",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_NETWORK_INTERNODE,
)
});
@@ -52,7 +53,7 @@ pub static INTERNODE_SENT_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock:
new_counter_md(
MetricName::InternodeSentBytesTotal,
"Total number of bytes sent to other peer nodes",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_NETWORK_INTERNODE,
)
});
@@ -62,7 +63,7 @@ pub static INTERNODE_RECV_BYTES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock:
new_counter_md(
MetricName::InternodeRecvBytesTotal,
"Total number of bytes received from other peer nodes",
&[],
&[SERVER_LABEL],
subsystems::SYSTEM_NETWORK_INTERNODE,
)
});
crates/obs/src/metrics/schema/system_network_host.rs
+12 -4
View File
@@ -14,6 +14,7 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
use std::sync::LazyLock;
@@ -25,7 +26,7 @@ pub static HOST_NETWORK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::HostNetworkIO,
"Network bytes transferred across system network interfaces",
&[DIRECTION_LABEL],
&[SERVER_LABEL, DIRECTION_LABEL],
subsystems::SYSTEM_NETWORK_HOST,
)
});
@@ -35,7 +36,7 @@ pub static HOST_NETWORK_IO_PER_INTERFACE_MD: LazyLock<MetricDescriptor> = LazyLo
new_counter_md(
MetricName::HostNetworkIOPerInterface,
"Network bytes transferred across system network interfaces (per interface)",
&[INTERFACE_LABEL, DIRECTION_LABEL],
&[SERVER_LABEL, INTERFACE_LABEL, DIRECTION_LABEL],
subsystems::SYSTEM_NETWORK_HOST,
)
});
@@ -48,12 +49,19 @@ mod tests {
#[test]
fn host_network_descriptors_export_counter_labels() {
assert_eq!(HOST_NETWORK_IO_MD.metric_type, MetricType::Counter);
assert_eq!(HOST_NETWORK_IO_MD.variable_labels, vec![DIRECTION_LABEL.to_string()]);
assert_eq!(
HOST_NETWORK_IO_MD.variable_labels,
vec![SERVER_LABEL.to_string(), DIRECTION_LABEL.to_string()]
);
assert_eq!(HOST_NETWORK_IO_PER_INTERFACE_MD.metric_type, MetricType::Counter);
assert_eq!(
HOST_NETWORK_IO_PER_INTERFACE_MD.variable_labels,
vec![INTERFACE_LABEL.to_string(), DIRECTION_LABEL.to_string()]
vec![
SERVER_LABEL.to_string(),
INTERFACE_LABEL.to_string(),
DIRECTION_LABEL.to_string()
]
);
}
}
crates/obs/src/metrics/schema/system_process.rs
+37 -21
View File
@@ -14,15 +14,31 @@
#![allow(dead_code)]
use crate::node_identity::SERVER_LABEL;
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const PROCESS_PID_LABEL: &str = "process_pid";
pub const PROCESS_EXECUTABLE_NAME_LABEL: &str = "process_executable_name";
pub const DIRECTION_LABEL: &str = "direction";
pub const STATUS_LABEL: &str = "status";
const PROCESS_LABELS: &[&str] = &[SERVER_LABEL];
const PROCESS_WITH_ATTRIBUTES_LABELS: &[&str] = &[SERVER_LABEL, PROCESS_PID_LABEL, PROCESS_EXECUTABLE_NAME_LABEL];
const PROCESS_DISK_IO_LABELS: &[&str] = &[
DIRECTION_LABEL,
SERVER_LABEL,
PROCESS_PID_LABEL,
PROCESS_EXECUTABLE_NAME_LABEL,
];
const PROCESS_STATUS_LABELS: &[&str] = &[SERVER_LABEL, STATUS_LABEL];
/// Number of current READ locks on this peer
pub static PROCESS_LOCKS_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ProcessLocksReadTotal,
"Number of current READ locks on this peer",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -32,7 +48,7 @@ pub static PROCESS_LOCKS_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::
new_gauge_md(
MetricName::ProcessLocksWriteTotal,
"Number of current WRITE locks on this peer",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -42,7 +58,7 @@ pub static PROCESS_CPU_TOTAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::
new_counter_md(
MetricName::ProcessCPUTotalSeconds,
"Total user and system CPU time spent in seconds",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -52,7 +68,7 @@ pub static PROCESS_GO_ROUTINE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::n
new_gauge_md(
MetricName::ProcessGoRoutineTotal,
"Total number of go routines running",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -62,7 +78,7 @@ pub static PROCESS_IO_RCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ProcessIORCharBytes,
"Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -72,7 +88,7 @@ pub static PROCESS_IO_READ_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
new_counter_md(
MetricName::ProcessIOReadBytes,
"Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -82,7 +98,7 @@ pub static PROCESS_IO_WCHAR_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ProcessIOWCharBytes,
"Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -92,7 +108,7 @@ pub static PROCESS_IO_WRITE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ProcessIOWriteBytes,
"Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -102,7 +118,7 @@ pub static PROCESS_START_TIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock:
new_gauge_md(
MetricName::ProcessStartTimeSeconds,
"Start time for RustFS process in seconds since Unix epoch",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -112,7 +128,7 @@ pub static PROCESS_UPTIME_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_gauge_md(
MetricName::ProcessUptimeSeconds,
"Uptime for RustFS process in seconds",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -122,7 +138,7 @@ pub static PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD: LazyLock<MetricDescriptor> =
new_gauge_md(
MetricName::ProcessFileDescriptorLimitTotal,
"Limit on total number of open file descriptors for the RustFS Server process",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -132,7 +148,7 @@ pub static PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD: LazyLock<MetricDescriptor> = L
new_gauge_md(
MetricName::ProcessFileDescriptorOpenTotal,
"Total number of open file descriptors by the RustFS Server process",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -142,7 +158,7 @@ pub static PROCESS_SYSCALL_READ_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock:
new_counter_md(
MetricName::ProcessSyscallReadTotal,
"Total read SysCalls to the kernel. /proc/[pid]/io syscr",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -152,7 +168,7 @@ pub static PROCESS_SYSCALL_WRITE_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock
new_counter_md(
MetricName::ProcessSyscallWriteTotal,
"Total write SysCalls to the kernel. /proc/[pid]/io syscw",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -162,7 +178,7 @@ pub static PROCESS_RESIDENT_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLo
new_gauge_md(
MetricName::ProcessResidentMemoryBytes,
"Resident memory size in bytes",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -172,7 +188,7 @@ pub static PROCESS_VIRTUAL_MEMORY_BYTES_MD: LazyLock<MetricDescriptor> = LazyLoc
new_gauge_md(
MetricName::ProcessVirtualMemoryBytes,
"Virtual memory size in bytes",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -182,7 +198,7 @@ pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock<MetricDescriptor> = Laz
new_gauge_md(
MetricName::ProcessVirtualMemoryMaxBytes,
"Maximum virtual memory size in bytes",
&[],
PROCESS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -196,7 +212,7 @@ pub static PROCESS_CPU_USAGE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ProcessCPUUsage,
"The percentage of CPU in use by the process",
&[],
PROCESS_WITH_ATTRIBUTES_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -206,7 +222,7 @@ pub static PROCESS_CPU_UTILIZATION_MD: LazyLock<MetricDescriptor> = LazyLock::ne
new_gauge_md(
MetricName::ProcessCPUUtilization,
"The amount of CPU in use by the process (considering multiple cores)",
&[],
PROCESS_WITH_ATTRIBUTES_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -216,7 +232,7 @@ pub static PROCESS_DISK_IO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ProcessDiskIO,
"Disk bytes transferred by the process",
&[],
PROCESS_DISK_IO_LABELS,
subsystems::SYSTEM_PROCESS,
)
});
@@ -226,7 +242,7 @@ pub static PROCESS_STATUS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ProcessStatus,
"Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)",
&[],
PROCESS_STATUS_LABELS,
subsystems::SYSTEM_PROCESS,
)
});