gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 09:18:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): improve telemetry stack, replication metrics, and Grafana alignment (#2672)

Browse Source 
Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-04-24 21:50:17 +08:00
committed by GitHub
parent 2705e3f53b
commit 13b4500212
19 changed files with 4603 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/scheduler.rs
+30 -9
View File
@@ -35,8 +35,14 @@ use crate::metrics::collectors::{
collect_audit_metrics,
collect_bucket_metrics,
collect_bucket_replication_bandwidth_metrics,
collect_bucket_replication_metrics,
collect_cluster_health_metrics,
collect_cluster_metrics,
collect_cpu_metrics,
collect_drive_count_metrics,
collect_drive_detailed_metrics,
collect_host_network_metrics,
collect_memory_metrics,
collect_node_metrics,
collect_notification_metrics,
collect_notification_target_metrics,
@@ -45,6 +51,7 @@ use crate::metrics::collectors::{
collect_process_disk_metrics,
collect_process_memory_metrics,
collect_process_metrics,
collect_replication_metrics,
collect_resource_metrics,
};
use crate::metrics::config::{
@@ -56,14 +63,16 @@ use crate::metrics::config::{
};
use crate::metrics::report::{PrometheusMetric, report_metrics};
use crate::metrics::stats_collector::{
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats,
collect_disk_stats, collect_host_network_stats, collect_process_metric_bundle,
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
collect_bucket_stats, collect_cluster_and_health_stats, collect_disk_and_system_drive_stats, collect_host_network_stats,
collect_process_metric_bundle, collect_replication_stats, collect_system_cpu_and_memory_stats_with,
};
use rustfs_audit::audit_target_metrics;
use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
use rustfs_utils::get_env_opt_u64;
use std::borrow::Cow;
use std::time::Duration;
use sysinfo::System;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::warn;
@@ -154,8 +163,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
loop {
tokio::select! {
_ = interval.tick() => {
let stats = collect_cluster_stats().await;
let metrics = collect_cluster_metrics(&stats);
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
let mut metrics = collect_cluster_metrics(&stats);
metrics.extend(collect_cluster_health_metrics(&cluster_health));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -192,8 +202,10 @@ pub fn init_metrics_runtime(token: CancellationToken) {
loop {
tokio::select! {
_ = interval.tick() => {
let stats = collect_disk_stats().await;
let metrics = collect_node_metrics(&stats);
let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_stats().await;
let mut metrics = collect_node_metrics(&disk_stats);
metrics.extend(collect_drive_detailed_metrics(&drive_stats));
metrics.extend(collect_drive_count_metrics(&drive_counts));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -212,7 +224,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
tokio::select! {
_ = interval.tick() => {
let stats = collect_bucket_replication_bandwidth_stats();
let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
let mut metrics = collect_bucket_replication_bandwidth_metrics(&stats);
let bucket_replication = collect_bucket_replication_detail_stats().await;
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
let replication = collect_replication_stats().await;
metrics.extend(collect_replication_metrics(&replication));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -297,6 +313,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let token_clone = token;
tokio::spawn(async move {
let labels = current_process_metric_labels();
let mut host_system = System::new_all();
let process_interval = resource_interval.min(system_interval);
let mut interval = tokio::time::interval(process_interval);
let now = Instant::now();
@@ -327,9 +344,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
if now >= next_system_run {
#[cfg(feature = "gpu")]
let mut metrics = collect_system_monitoring_metrics(&bundle, &labels);
let mut metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
#[cfg(not(feature = "gpu"))]
let metrics = collect_system_monitoring_metrics(&bundle, &labels);
let metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
#[cfg(feature = "gpu")]
if let Some(pid) = current_pid {
@@ -418,6 +435,7 @@ fn fallback_process_metric_labels(err: ProcessAttributeError) -> Vec<(&'static s
fn collect_system_monitoring_metrics(
bundle: &ProcessMetricBundle,
labels: &[(&'static str, Cow<'static, str>)],
host_system: &mut System,
) -> Vec<PrometheusMetric> {
let cpu_stats = ProcessCpuStats {
usage: bundle.resource.cpu_percent,
@@ -432,8 +450,11 @@ fn collect_system_monitoring_metrics(
written_bytes: bundle.disk_write_bytes,
};
let network_stats = collect_host_network_stats();
let (system_cpu_stats, system_memory_stats) = collect_system_cpu_and_memory_stats_with(host_system);
let mut metrics = Vec::new();
metrics.extend(collect_cpu_metrics(&system_cpu_stats));
metrics.extend(collect_memory_metrics(&system_memory_stats));
metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(labels)));
metrics.extend(collect_process_memory_metrics(&memory_stats, Some(labels)));
metrics.extend(collect_process_disk_metrics(&disk_stats, Some(labels)));