mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
escapecode
a80699b6dd
feat(targets): add an opt-in NATS JetStream publish path for the notify and audit targets The NATS notify and audit targets publish through NATS Core, which returns before the server has durably accepted the message. A broker restart or a connection drop between the publish and the flush loses the event, even though the send queue has already cleared it, and no acknowledgement gates that clear. An opt-in JetStream publish path clears a queued event only after the server returns a durable PublishAck, so delivery is at-least-once across a broker restart or a reconnect. It applies to both the notify and audit NATS targets, is off by default, and is byte-identical to the NATS Core path when disabled. The path includes durable store-and-forward, a stable dedup id sent as the Nats-Msg-Id header so a replayed event is collapsed by the stream duplicate window, pre-flight stream validation, and a bounded failed-events store for terminally-failed and retry-exhausted events. Three configuration keys per target select it: JETSTREAM_ENABLE, JETSTREAM_STREAM_NAME, and JETSTREAM_ACK_TIMEOUT_SECS, under the RUSTFS_NOTIFY_NATS_ and RUSTFS_AUDIT_NATS_ prefixes. The on-disk batch filename separator changes from colon to underscore so batch names are valid on Windows filesystems, with transparent read-back of files written under the previous separator. The migration affects the shared queue store for every target type and lands with this feature because the store gains its first Windows-exercised paths here. Co-authored-by: houseme <housemecn@gmail.com>
1916 lines
83 KiB
Rust
1916 lines
83 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Global metrics collector initialization.
|
|
//!
|
|
//! This module provides the entry point for initializing all metrics collectors.
|
|
//! The actual statistics collection functions are in `stats_collector.rs`.
|
|
//!
|
|
//! System monitoring collectors (migrated from `rustfs-obs::system`):
|
|
//! - Process CPU metrics
|
|
//! - Process memory metrics
|
|
//! - Process disk I/O metrics
|
|
//! - Host network I/O metrics
|
|
|
|
use crate::metrics::collectors::{
|
|
AuditTargetStats,
|
|
BucketReplicationBandwidthStats,
|
|
NotificationStats,
|
|
NotificationTargetStats,
|
|
// System monitoring collectors (migrated from rustfs-obs::system)
|
|
ProcessAttributeError,
|
|
ProcessCpuStats,
|
|
ProcessDiskStats,
|
|
ProcessMemoryStats,
|
|
collect_audit_metrics,
|
|
collect_bucket_metrics,
|
|
collect_bucket_replication_bandwidth_metrics,
|
|
collect_bucket_replication_metrics,
|
|
collect_bucket_usage_metrics,
|
|
collect_cluster_config_metrics,
|
|
collect_cluster_health_metrics,
|
|
collect_cluster_metrics,
|
|
collect_cluster_usage_metrics,
|
|
collect_compression_cluster_metrics,
|
|
collect_cpu_metrics,
|
|
collect_current_dial9_metrics,
|
|
collect_drive_count_metrics,
|
|
collect_drive_detailed_metrics,
|
|
collect_erasure_set_metrics,
|
|
collect_host_network_metrics,
|
|
collect_iam_metrics,
|
|
collect_ilm_metrics,
|
|
collect_memory_metrics,
|
|
collect_network_metrics,
|
|
collect_node_metrics,
|
|
collect_notification_metrics,
|
|
collect_notification_target_metrics,
|
|
collect_process_attributes,
|
|
collect_process_cpu_metrics,
|
|
collect_process_disk_metrics,
|
|
collect_process_memory_metrics,
|
|
collect_process_metrics,
|
|
collect_replication_metrics,
|
|
collect_resource_metrics,
|
|
collect_scanner_metrics,
|
|
};
|
|
use crate::metrics::config::{
|
|
DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
|
|
DEFAULT_CLUSTER_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL, DEFAULT_NOTIFICATION_METRICS_INTERVAL,
|
|
DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_AUDIT_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL,
|
|
ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
|
|
ENV_NODE_METRICS_INTERVAL, ENV_NOTIFICATION_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
|
|
};
|
|
use crate::metrics::obs_is_disk_compression_enabled;
|
|
use crate::metrics::report::{PrometheusMetric, report_metrics};
|
|
use crate::metrics::runtime_sources::bucket_monitor_available;
|
|
use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, AUDIT_TOTAL_MESSAGES_MD};
|
|
use crate::metrics::schema::bucket_replication::{
|
|
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
|
|
};
|
|
use crate::metrics::schema::cluster_usage::{
|
|
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
|
|
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
|
|
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
|
|
};
|
|
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
|
|
use crate::metrics::schema::notification_target::{
|
|
NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD,
|
|
TARGET_ID as NOTIFICATION_TARGET_ID_LABEL, TARGET_TYPE as NOTIFICATION_TARGET_TYPE_LABEL,
|
|
};
|
|
use crate::metrics::stats_collector::{
|
|
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
|
|
collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
|
|
collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
|
|
collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
|
|
collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_metric_stats,
|
|
collect_system_cpu_and_memory_stats_with,
|
|
};
|
|
use crate::telemetry::retire_metric_series;
|
|
use futures_util::FutureExt;
|
|
use rustfs_audit::audit_target_metrics;
|
|
use rustfs_io_metrics::ProcessSampler;
|
|
use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
|
|
use rustfs_utils::get_env_opt_u64;
|
|
use serde::Serialize;
|
|
use std::borrow::Cow;
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::panic::AssertUnwindSafe;
|
|
use std::sync::OnceLock;
|
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
use std::time::Duration;
|
|
use sysinfo::{Networks, System};
|
|
use tokio::time::{Instant, Interval, MissedTickBehavior};
|
|
use tokio_util::sync::CancellationToken;
|
|
use tracing::warn;
|
|
|
|
const LOG_COMPONENT_OBS: &str = "obs";
|
|
const LOG_SUBSYSTEM_METRICS_RUNTIME: &str = "metrics_runtime";
|
|
const EVENT_METRICS_RUNTIME_STATE: &str = "metrics_runtime_state";
|
|
|
|
/// Default interval for system monitoring metrics (15 seconds)
|
|
const DEFAULT_SYSTEM_METRICS_INTERVAL: Duration = Duration::from_secs(15);
|
|
/// Environment variable for system monitoring interval
|
|
const ENV_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_METRICS_SYSTEM_INTERVAL_SEC";
|
|
/// Legacy environment variable for system monitoring interval
|
|
const LEGACY_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_OBS_METRICS_SYSTEM_INTERVAL_MS";
|
|
const LEGACY_CLUSTER_INTERVAL: &str = "RUSTFS_METRICS_CLUSTER_INTERVAL";
|
|
const LEGACY_BUCKET_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_INTERVAL";
|
|
const LEGACY_NODE_INTERVAL: &str = "RUSTFS_METRICS_NODE_INTERVAL";
|
|
const LEGACY_REPLICATION_BANDWIDTH_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL";
|
|
const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL";
|
|
const LEGACY_AUDIT_INTERVAL: &str = "RUSTFS_METRICS_AUDIT_INTERVAL";
|
|
const LEGACY_NOTIFICATION_INTERVAL: &str = "RUSTFS_METRICS_NOTIFICATION_INTERVAL";
|
|
const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL";
|
|
const AUDIT_TARGET_ID_LABEL: &str = "target_id";
|
|
|
|
/// Default cycles to emit zero for removed replication bandwidth series before letting them expire.
|
|
const DEFAULT_REPL_BW_ZERO_TOMBSTONE_CYCLES: u8 = 3;
|
|
/// Env var that overrides the zero-emission tombstone cycles for removed replication bandwidth series.
|
|
const ENV_REPL_BW_ZERO_TOMBSTONE_CYCLES: &str = "RUSTFS_METRICS_REPL_BW_ZERO_TOMBSTONE_CYCLES";
|
|
const METRICS_RUNTIME_SERVICE_NAME: &str = "metrics_runtime";
|
|
const METRICS_RUNTIME_BASE_COLLECTOR_TASKS: u8 = 10;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
#[repr(u8)]
|
|
enum MetricsCollectorTaskId {
|
|
ClusterStats = 0,
|
|
SupplementaryClusterStats = 1,
|
|
BucketStats = 2,
|
|
NodeDiskStats = 3,
|
|
BucketReplicationBandwidth = 4,
|
|
AuditTargetStats = 5,
|
|
NotificationStats = 6,
|
|
BackgroundWorkflowStats = 7,
|
|
ProcessMetrics = 8,
|
|
InternodeNetworkStats = 9,
|
|
CompressionClusterStats = 10,
|
|
}
|
|
|
|
const BASE_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COLLECTOR_TASKS as usize] = [
|
|
MetricsCollectorTaskId::ClusterStats,
|
|
MetricsCollectorTaskId::SupplementaryClusterStats,
|
|
MetricsCollectorTaskId::BucketStats,
|
|
MetricsCollectorTaskId::NodeDiskStats,
|
|
MetricsCollectorTaskId::BucketReplicationBandwidth,
|
|
MetricsCollectorTaskId::AuditTargetStats,
|
|
MetricsCollectorTaskId::NotificationStats,
|
|
MetricsCollectorTaskId::BackgroundWorkflowStats,
|
|
MetricsCollectorTaskId::ProcessMetrics,
|
|
MetricsCollectorTaskId::InternodeNetworkStats,
|
|
];
|
|
|
|
const ALL_COLLECTOR_TASK_IDS: [MetricsCollectorTaskId; METRICS_RUNTIME_BASE_COLLECTOR_TASKS as usize + 1] = [
|
|
MetricsCollectorTaskId::ClusterStats,
|
|
MetricsCollectorTaskId::SupplementaryClusterStats,
|
|
MetricsCollectorTaskId::BucketStats,
|
|
MetricsCollectorTaskId::NodeDiskStats,
|
|
MetricsCollectorTaskId::BucketReplicationBandwidth,
|
|
MetricsCollectorTaskId::AuditTargetStats,
|
|
MetricsCollectorTaskId::NotificationStats,
|
|
MetricsCollectorTaskId::BackgroundWorkflowStats,
|
|
MetricsCollectorTaskId::ProcessMetrics,
|
|
MetricsCollectorTaskId::InternodeNetworkStats,
|
|
MetricsCollectorTaskId::CompressionClusterStats,
|
|
];
|
|
|
|
#[derive(Debug)]
|
|
struct MetricsRuntimeCollectorHealth {
|
|
last_success_unix_secs: [AtomicU64; ALL_COLLECTOR_TASK_IDS.len()],
|
|
collector_panics_total: [AtomicU64; ALL_COLLECTOR_TASK_IDS.len()],
|
|
failure_state: [AtomicBool; ALL_COLLECTOR_TASK_IDS.len()],
|
|
}
|
|
|
|
impl MetricsRuntimeCollectorHealth {
|
|
fn new() -> Self {
|
|
Self {
|
|
last_success_unix_secs: std::array::from_fn(|_| AtomicU64::new(0)),
|
|
collector_panics_total: std::array::from_fn(|_| AtomicU64::new(0)),
|
|
failure_state: std::array::from_fn(|_| AtomicBool::new(false)),
|
|
}
|
|
}
|
|
|
|
fn record_success(&self, collector_id: MetricsCollectorTaskId) {
|
|
let idx = collector_id as usize;
|
|
self.last_success_unix_secs[idx].store(unix_timestamp_secs_now(), Ordering::Relaxed);
|
|
self.failure_state[idx].store(false, Ordering::Relaxed);
|
|
}
|
|
|
|
fn record_panic(&self, collector_id: MetricsCollectorTaskId) {
|
|
let idx = collector_id as usize;
|
|
self.collector_panics_total[idx].fetch_add(1, Ordering::Relaxed);
|
|
self.failure_state[idx].store(true, Ordering::Relaxed);
|
|
}
|
|
|
|
fn snapshot(&self, active_collectors: &[MetricsCollectorTaskId]) -> MetricsRuntimeCollectorHealthSnapshot {
|
|
let now = unix_timestamp_secs_now();
|
|
let mut healthy_collectors = 0_u8;
|
|
let mut never_succeeded_collectors = 0_u8;
|
|
let mut collector_panics_total = 0_u64;
|
|
let mut oldest_success_age_secs = 0_u64;
|
|
|
|
for collector_id in active_collectors {
|
|
let idx = *collector_id as usize;
|
|
let last_success = self.last_success_unix_secs[idx].load(Ordering::Relaxed);
|
|
let has_failed = self.failure_state[idx].load(Ordering::Relaxed);
|
|
collector_panics_total =
|
|
collector_panics_total.saturating_add(self.collector_panics_total[idx].load(Ordering::Relaxed));
|
|
|
|
if last_success == 0 {
|
|
never_succeeded_collectors = never_succeeded_collectors.saturating_add(1);
|
|
} else {
|
|
oldest_success_age_secs = oldest_success_age_secs.max(now.saturating_sub(last_success));
|
|
}
|
|
|
|
if last_success > 0 && !has_failed {
|
|
healthy_collectors = healthy_collectors.saturating_add(1);
|
|
}
|
|
}
|
|
|
|
MetricsRuntimeCollectorHealthSnapshot {
|
|
healthy_collectors,
|
|
unhealthy_collectors: u8::try_from(active_collectors.len())
|
|
.unwrap_or(u8::MAX)
|
|
.saturating_sub(healthy_collectors),
|
|
never_succeeded_collectors,
|
|
collector_panics_total,
|
|
oldest_success_age_secs,
|
|
}
|
|
}
|
|
}
|
|
|
|
static METRICS_RUNTIME_COLLECTOR_HEALTH: OnceLock<MetricsRuntimeCollectorHealth> = OnceLock::new();
|
|
|
|
type ReplBwKey = (String, String); // (bucket, target_arn)
|
|
type BucketKey = String;
|
|
type BucketRangeKey = (String, String); // (bucket, range)
|
|
type AuditTargetKey = String;
|
|
type NotificationTargetKey = (String, String); // (target_id, target_type)
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
|
|
pub struct MetricsRuntimeCollectorHealthSnapshot {
|
|
pub healthy_collectors: u8,
|
|
pub unhealthy_collectors: u8,
|
|
pub never_succeeded_collectors: u8,
|
|
pub collector_panics_total: u64,
|
|
pub oldest_success_age_secs: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MetricsRuntimeServiceState {
|
|
Disabled,
|
|
Running,
|
|
Stopping,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MetricsRuntimeCancellationSource {
|
|
RuntimeToken,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MetricsRuntimeShutdownHandle {
|
|
RuntimeTokenOnly,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub struct MetricsRuntimeIntervalsSnapshot {
|
|
pub cluster_interval_secs: u64,
|
|
pub bucket_interval_secs: u64,
|
|
pub bucket_replication_bandwidth_interval_secs: u64,
|
|
pub node_interval_secs: u64,
|
|
pub resource_interval_secs: u64,
|
|
pub audit_interval_secs: u64,
|
|
pub notification_interval_secs: u64,
|
|
pub system_interval_secs: u64,
|
|
pub process_interval_secs: u64,
|
|
pub replication_bandwidth_zero_tombstone_cycles: u8,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub struct MetricsRuntimeStatusSnapshot {
|
|
pub service: &'static str,
|
|
pub state: MetricsRuntimeServiceState,
|
|
pub metrics_enabled: bool,
|
|
pub collector_tasks: u8,
|
|
pub collector_health: MetricsRuntimeCollectorHealthSnapshot,
|
|
pub intervals: MetricsRuntimeIntervalsSnapshot,
|
|
pub cancellation_source: MetricsRuntimeCancellationSource,
|
|
pub shutdown_handle: MetricsRuntimeShutdownHandle,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MetricsRuntimeDesiredState {
|
|
Disabled,
|
|
Enabled,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub struct MetricsRuntimeDesiredSnapshot {
|
|
pub state: MetricsRuntimeDesiredState,
|
|
pub collector_tasks: u8,
|
|
pub intervals: MetricsRuntimeIntervalsSnapshot,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub struct MetricsRuntimeControllerSnapshot {
|
|
pub desired: MetricsRuntimeDesiredSnapshot,
|
|
pub status: MetricsRuntimeStatusSnapshot,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MetricsRuntimeWorkerMutation {
|
|
None,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub struct MetricsRuntimeReconcilePlan {
|
|
pub service: &'static str,
|
|
pub desired: MetricsRuntimeDesiredSnapshot,
|
|
pub current_state: MetricsRuntimeServiceState,
|
|
pub worker_mutation: MetricsRuntimeWorkerMutation,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Default)]
|
|
pub struct MetricsRuntimeController;
|
|
|
|
impl MetricsRuntimeController {
|
|
pub fn snapshot(&self, token: &CancellationToken) -> MetricsRuntimeControllerSnapshot {
|
|
metrics_runtime_controller_snapshot(token)
|
|
}
|
|
|
|
pub fn reconcile(&self, token: &CancellationToken) -> MetricsRuntimeReconcilePlan {
|
|
let snapshot = self.snapshot(token);
|
|
self.reconcile_snapshot(snapshot)
|
|
}
|
|
|
|
pub fn reconcile_snapshot(&self, snapshot: MetricsRuntimeControllerSnapshot) -> MetricsRuntimeReconcilePlan {
|
|
MetricsRuntimeReconcilePlan {
|
|
service: METRICS_RUNTIME_SERVICE_NAME,
|
|
desired: snapshot.desired,
|
|
current_state: snapshot.status.state,
|
|
worker_mutation: MetricsRuntimeWorkerMutation::None,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
struct MetricsRuntimeConfig {
|
|
cluster_interval: Duration,
|
|
bucket_interval: Duration,
|
|
bucket_replication_bandwidth_interval: Duration,
|
|
node_interval: Duration,
|
|
resource_interval: Duration,
|
|
audit_interval: Duration,
|
|
notification_interval: Duration,
|
|
system_interval: Duration,
|
|
process_interval: Duration,
|
|
replication_bandwidth_zero_tombstone_cycles: u8,
|
|
}
|
|
|
|
fn parse_repl_bw_zero_tombstone_cycles() -> u8 {
|
|
get_env_opt_u64(ENV_REPL_BW_ZERO_TOMBSTONE_CYCLES)
|
|
.filter(|&v| v > 0)
|
|
.map(|v| u8::try_from(v).unwrap_or(u8::MAX))
|
|
.unwrap_or(DEFAULT_REPL_BW_ZERO_TOMBSTONE_CYCLES)
|
|
}
|
|
|
|
/// Parse metrics interval from environment variables with fallback to default.
|
|
///
|
|
/// Priority: primary_env > legacy_env > default_env > legacy_default > default_value
|
|
fn parse_metrics_interval(primary_env: &str, legacy_env: &str, default_interval: Duration) -> Duration {
|
|
get_env_opt_u64(primary_env)
|
|
.or_else(|| get_env_opt_u64(legacy_env))
|
|
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
|
|
.or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL))
|
|
.filter(|&v| v > 0)
|
|
.map(Duration::from_secs)
|
|
.unwrap_or(default_interval)
|
|
}
|
|
|
|
fn parse_system_metrics_interval() -> Duration {
|
|
get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL)
|
|
.or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| if ms == 0 { 0 } else { ms.div_ceil(1000) }))
|
|
.or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL))
|
|
.filter(|&v| v > 0)
|
|
.map(Duration::from_secs)
|
|
.unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL)
|
|
}
|
|
|
|
fn configured_metrics_runtime_config() -> MetricsRuntimeConfig {
|
|
let cluster_interval =
|
|
parse_metrics_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
|
|
let bucket_interval =
|
|
parse_metrics_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
|
|
let bucket_replication_bandwidth_interval = parse_metrics_interval(
|
|
ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
|
|
LEGACY_REPLICATION_BANDWIDTH_INTERVAL,
|
|
DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
|
|
);
|
|
let node_interval = parse_metrics_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
|
|
let resource_interval =
|
|
parse_metrics_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
|
|
let audit_interval =
|
|
parse_metrics_interval(ENV_AUDIT_METRICS_INTERVAL, LEGACY_AUDIT_INTERVAL, DEFAULT_AUDIT_METRICS_INTERVAL);
|
|
let notification_interval = parse_metrics_interval(
|
|
ENV_NOTIFICATION_METRICS_INTERVAL,
|
|
LEGACY_NOTIFICATION_INTERVAL,
|
|
DEFAULT_NOTIFICATION_METRICS_INTERVAL,
|
|
);
|
|
let system_interval = parse_system_metrics_interval();
|
|
let process_interval = resource_interval.min(system_interval);
|
|
|
|
MetricsRuntimeConfig {
|
|
cluster_interval,
|
|
bucket_interval,
|
|
bucket_replication_bandwidth_interval,
|
|
node_interval,
|
|
resource_interval,
|
|
audit_interval,
|
|
notification_interval,
|
|
system_interval,
|
|
process_interval,
|
|
replication_bandwidth_zero_tombstone_cycles: parse_repl_bw_zero_tombstone_cycles(),
|
|
}
|
|
}
|
|
|
|
fn metrics_runtime_intervals_snapshot(config: MetricsRuntimeConfig) -> MetricsRuntimeIntervalsSnapshot {
|
|
MetricsRuntimeIntervalsSnapshot {
|
|
cluster_interval_secs: config.cluster_interval.as_secs(),
|
|
bucket_interval_secs: config.bucket_interval.as_secs(),
|
|
bucket_replication_bandwidth_interval_secs: config.bucket_replication_bandwidth_interval.as_secs(),
|
|
node_interval_secs: config.node_interval.as_secs(),
|
|
resource_interval_secs: config.resource_interval.as_secs(),
|
|
audit_interval_secs: config.audit_interval.as_secs(),
|
|
notification_interval_secs: config.notification_interval.as_secs(),
|
|
system_interval_secs: config.system_interval.as_secs(),
|
|
process_interval_secs: config.process_interval.as_secs(),
|
|
replication_bandwidth_zero_tombstone_cycles: config.replication_bandwidth_zero_tombstone_cycles,
|
|
}
|
|
}
|
|
|
|
fn metrics_runtime_collector_health() -> &'static MetricsRuntimeCollectorHealth {
|
|
METRICS_RUNTIME_COLLECTOR_HEALTH.get_or_init(MetricsRuntimeCollectorHealth::new)
|
|
}
|
|
|
|
fn active_metrics_collector_task_ids(compression_enabled: bool) -> &'static [MetricsCollectorTaskId] {
|
|
if compression_enabled {
|
|
&ALL_COLLECTOR_TASK_IDS
|
|
} else {
|
|
&BASE_COLLECTOR_TASK_IDS
|
|
}
|
|
}
|
|
|
|
fn metrics_runtime_collector_tasks(compression_enabled: bool) -> u8 {
|
|
u8::try_from(active_metrics_collector_task_ids(compression_enabled).len()).unwrap_or(u8::MAX)
|
|
}
|
|
|
|
fn build_metrics_runtime_status_snapshot(
|
|
metrics_enabled: bool,
|
|
cancellation_requested: bool,
|
|
config: MetricsRuntimeConfig,
|
|
compression_enabled: bool,
|
|
) -> MetricsRuntimeStatusSnapshot {
|
|
let state = if !metrics_enabled {
|
|
MetricsRuntimeServiceState::Disabled
|
|
} else if cancellation_requested {
|
|
MetricsRuntimeServiceState::Stopping
|
|
} else {
|
|
MetricsRuntimeServiceState::Running
|
|
};
|
|
|
|
let collector_health = if metrics_enabled {
|
|
metrics_runtime_collector_health().snapshot(active_metrics_collector_task_ids(compression_enabled))
|
|
} else {
|
|
MetricsRuntimeCollectorHealthSnapshot::default()
|
|
};
|
|
|
|
MetricsRuntimeStatusSnapshot {
|
|
service: METRICS_RUNTIME_SERVICE_NAME,
|
|
state,
|
|
metrics_enabled,
|
|
collector_tasks: metrics_runtime_collector_tasks(compression_enabled),
|
|
collector_health,
|
|
intervals: metrics_runtime_intervals_snapshot(config),
|
|
cancellation_source: MetricsRuntimeCancellationSource::RuntimeToken,
|
|
shutdown_handle: MetricsRuntimeShutdownHandle::RuntimeTokenOnly,
|
|
}
|
|
}
|
|
|
|
fn build_metrics_runtime_desired_snapshot(
|
|
metrics_enabled: bool,
|
|
config: MetricsRuntimeConfig,
|
|
compression_enabled: bool,
|
|
) -> MetricsRuntimeDesiredSnapshot {
|
|
let state = if metrics_enabled {
|
|
MetricsRuntimeDesiredState::Enabled
|
|
} else {
|
|
MetricsRuntimeDesiredState::Disabled
|
|
};
|
|
|
|
MetricsRuntimeDesiredSnapshot {
|
|
state,
|
|
collector_tasks: metrics_runtime_collector_tasks(compression_enabled),
|
|
intervals: metrics_runtime_intervals_snapshot(config),
|
|
}
|
|
}
|
|
|
|
fn build_metrics_runtime_controller_snapshot(
|
|
metrics_enabled: bool,
|
|
cancellation_requested: bool,
|
|
config: MetricsRuntimeConfig,
|
|
compression_enabled: bool,
|
|
) -> MetricsRuntimeControllerSnapshot {
|
|
MetricsRuntimeControllerSnapshot {
|
|
desired: build_metrics_runtime_desired_snapshot(metrics_enabled, config, compression_enabled),
|
|
status: build_metrics_runtime_status_snapshot(metrics_enabled, cancellation_requested, config, compression_enabled),
|
|
}
|
|
}
|
|
|
|
pub fn metrics_runtime_status_snapshot(token: &CancellationToken) -> MetricsRuntimeStatusSnapshot {
|
|
build_metrics_runtime_status_snapshot(
|
|
crate::observability_metric_enabled(),
|
|
token.is_cancelled(),
|
|
configured_metrics_runtime_config(),
|
|
obs_is_disk_compression_enabled(),
|
|
)
|
|
}
|
|
|
|
pub fn metrics_runtime_controller_snapshot(token: &CancellationToken) -> MetricsRuntimeControllerSnapshot {
|
|
build_metrics_runtime_controller_snapshot(
|
|
crate::observability_metric_enabled(),
|
|
token.is_cancelled(),
|
|
configured_metrics_runtime_config(),
|
|
obs_is_disk_compression_enabled(),
|
|
)
|
|
}
|
|
|
|
fn unix_timestamp_secs_now() -> u64 {
|
|
std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs()
|
|
}
|
|
|
|
fn stagger_duration(period: Duration, numerator: u32, denominator: u32) -> Duration {
|
|
let staggered_nanos = period
|
|
.as_nanos()
|
|
.saturating_mul(u128::from(numerator))
|
|
.checked_div(u128::from(denominator))
|
|
.unwrap_or(0)
|
|
.min(u128::from(u64::MAX));
|
|
Duration::from_nanos(u64::try_from(staggered_nanos).unwrap_or(u64::MAX))
|
|
}
|
|
|
|
fn metrics_interval(period: Duration, initial_delay: Duration) -> Interval {
|
|
let mut interval = tokio::time::interval_at(Instant::now() + initial_delay, period);
|
|
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
|
interval
|
|
}
|
|
|
|
async fn run_metrics_collector_tick<F>(
|
|
health: &MetricsRuntimeCollectorHealth,
|
|
collector_id: MetricsCollectorTaskId,
|
|
collector: &'static str,
|
|
future: F,
|
|
) where
|
|
F: std::future::Future<Output = ()>,
|
|
{
|
|
match AssertUnwindSafe(future).catch_unwind().await {
|
|
Ok(()) => health.record_success(collector_id),
|
|
Err(payload) => {
|
|
health.record_panic(collector_id);
|
|
let panic_message = payload
|
|
.downcast_ref::<&'static str>()
|
|
.copied()
|
|
.or_else(|| payload.downcast_ref::<String>().map(String::as_str))
|
|
.unwrap_or("unknown panic payload");
|
|
warn!(
|
|
event = EVENT_METRICS_RUNTIME_STATE,
|
|
component = LOG_COMPONENT_OBS,
|
|
subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME,
|
|
collector,
|
|
result = "panic_caught",
|
|
panic_message,
|
|
"metrics runtime state changed"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn repl_bw_live_keys(stats: &[BucketReplicationBandwidthStats]) -> HashSet<ReplBwKey> {
|
|
stats.iter().map(|s| (s.bucket.clone(), s.target_arn.clone())).collect()
|
|
}
|
|
|
|
fn update_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(
|
|
has_seen_valid_snapshot: &mut bool,
|
|
prev_live_keys: &mut HashSet<T>,
|
|
zero_tombstones: &mut HashMap<T, u8>,
|
|
current_live_keys: HashSet<T>,
|
|
tombstone_cycles: u8,
|
|
) {
|
|
if *has_seen_valid_snapshot {
|
|
for removed in prev_live_keys.difference(¤t_live_keys) {
|
|
zero_tombstones.insert(removed.clone(), tombstone_cycles);
|
|
}
|
|
}
|
|
|
|
for key in ¤t_live_keys {
|
|
zero_tombstones.remove(key);
|
|
}
|
|
|
|
*prev_live_keys = current_live_keys;
|
|
*has_seen_valid_snapshot = true;
|
|
}
|
|
|
|
fn expire_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(zero_tombstones: &mut HashMap<T, u8>) -> Vec<T> {
|
|
let mut expired = Vec::new();
|
|
if !zero_tombstones.is_empty() {
|
|
zero_tombstones.retain(|key, remaining| {
|
|
if *remaining <= 1 {
|
|
expired.push(key.clone());
|
|
false
|
|
} else {
|
|
*remaining -= 1;
|
|
true
|
|
}
|
|
});
|
|
}
|
|
expired
|
|
}
|
|
|
|
fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
|
|
stats.iter().map(|stat| stat.name.clone()).collect()
|
|
}
|
|
|
|
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
|
|
if zero_tombstones.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 3);
|
|
for bucket in zero_tombstones.keys() {
|
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
|
|
zero_metrics
|
|
.push(PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, 0.0).with_label("bucket", bucket_label.clone()));
|
|
zero_metrics
|
|
.push(PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, 0.0).with_label("bucket", bucket_label.clone()));
|
|
zero_metrics.push(PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, 0.0).with_label("bucket", bucket_label));
|
|
}
|
|
|
|
zero_metrics
|
|
}
|
|
|
|
fn retire_bucket_metric_series(bucket: &str) -> usize {
|
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
|
|
let labels = [("bucket", bucket_label.clone())];
|
|
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
|
|
}
|
|
|
|
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
|
|
stats.iter().map(|stat| stat.bucket.clone()).collect()
|
|
}
|
|
|
|
fn bucket_usage_object_size_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketRangeKey> {
|
|
stats
|
|
.iter()
|
|
.flat_map(|stat| {
|
|
stat.object_size_distribution
|
|
.iter()
|
|
.map(move |(range, _)| (stat.bucket.clone(), range.clone()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn bucket_usage_version_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketRangeKey> {
|
|
stats
|
|
.iter()
|
|
.flat_map(|stat| {
|
|
stat.version_count_distribution
|
|
.iter()
|
|
.map(move |(range, _)| (stat.bucket.clone(), range.clone()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn collect_bucket_usage_zero_tombstone_metrics(
|
|
zero_bucket_tombstones: &HashMap<BucketKey, u8>,
|
|
zero_object_size_tombstones: &HashMap<BucketRangeKey, u8>,
|
|
zero_version_tombstones: &HashMap<BucketRangeKey, u8>,
|
|
) -> Vec<PrometheusMetric> {
|
|
let mut zero_metrics =
|
|
Vec::with_capacity(zero_bucket_tombstones.len() * 5 + zero_object_size_tombstones.len() + zero_version_tombstones.len());
|
|
|
|
for bucket in zero_bucket_tombstones.keys() {
|
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_TOTAL_BYTES_MD, 0.0)
|
|
.with_label(USAGE_BUCKET_LABEL, bucket_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECTS_TOTAL_MD, 0.0)
|
|
.with_label(USAGE_BUCKET_LABEL, bucket_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_VERSIONS_COUNT_MD, 0.0)
|
|
.with_label(USAGE_BUCKET_LABEL, bucket_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, 0.0)
|
|
.with_label(USAGE_BUCKET_LABEL, bucket_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, 0.0)
|
|
.with_label(USAGE_BUCKET_LABEL, bucket_label),
|
|
);
|
|
}
|
|
|
|
for (bucket, range) in zero_object_size_tombstones.keys() {
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, 0.0)
|
|
.with_label(USAGE_RANGE_LABEL, Cow::Owned(range.clone()))
|
|
.with_label(USAGE_BUCKET_LABEL, Cow::Owned(bucket.clone())),
|
|
);
|
|
}
|
|
|
|
for (bucket, range) in zero_version_tombstones.keys() {
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, 0.0)
|
|
.with_label(USAGE_RANGE_LABEL, Cow::Owned(range.clone()))
|
|
.with_label(USAGE_BUCKET_LABEL, Cow::Owned(bucket.clone())),
|
|
);
|
|
}
|
|
|
|
zero_metrics
|
|
}
|
|
|
|
fn retire_bucket_usage_metric_series(bucket: &str) -> usize {
|
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
|
|
let labels = [(USAGE_BUCKET_LABEL, bucket_label.clone())];
|
|
retire_metric_series(&USAGE_BUCKET_TOTAL_BYTES_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&USAGE_BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&USAGE_BUCKET_VERSIONS_COUNT_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD.get_full_metric_name(), &labels)
|
|
}
|
|
|
|
fn retire_bucket_usage_distribution_series(metric_name: String, bucket: &str, range: &str) -> usize {
|
|
let labels = [
|
|
(USAGE_RANGE_LABEL, Cow::Owned(range.to_string())),
|
|
(USAGE_BUCKET_LABEL, Cow::Owned(bucket.to_string())),
|
|
];
|
|
retire_metric_series(&metric_name, &labels)
|
|
}
|
|
|
|
fn audit_target_live_keys(stats: &[AuditTargetStats]) -> HashSet<AuditTargetKey> {
|
|
stats.iter().map(|stat| stat.target_id.clone()).collect()
|
|
}
|
|
|
|
fn collect_audit_zero_tombstone_metrics(zero_tombstones: &HashMap<AuditTargetKey, u8>) -> Vec<PrometheusMetric> {
|
|
if zero_tombstones.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 3);
|
|
for target_id in zero_tombstones.keys() {
|
|
let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone());
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, 0.0)
|
|
.with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, 0.0)
|
|
.with_label(AUDIT_TARGET_ID_LABEL, target_id_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, 0.0).with_label(AUDIT_TARGET_ID_LABEL, target_id_label),
|
|
);
|
|
}
|
|
|
|
zero_metrics
|
|
}
|
|
|
|
fn retire_audit_target_metric_series(target_id: &str) -> usize {
|
|
let labels = [(AUDIT_TARGET_ID_LABEL, Cow::Owned(target_id.to_string()))];
|
|
retire_metric_series(&AUDIT_FAILED_MESSAGES_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&AUDIT_TARGET_QUEUE_LENGTH_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&AUDIT_TOTAL_MESSAGES_MD.get_full_metric_name(), &labels)
|
|
}
|
|
|
|
fn notification_target_live_keys(stats: &[NotificationTargetStats]) -> HashSet<NotificationTargetKey> {
|
|
stats
|
|
.iter()
|
|
.map(|stat| (stat.target_id.clone(), stat.target_type.clone()))
|
|
.collect()
|
|
}
|
|
|
|
fn collect_notification_target_zero_tombstone_metrics(
|
|
zero_tombstones: &HashMap<NotificationTargetKey, u8>,
|
|
) -> Vec<PrometheusMetric> {
|
|
if zero_tombstones.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 3);
|
|
for (target_id, target_type) in zero_tombstones.keys() {
|
|
let target_id_label: Cow<'static, str> = Cow::Owned(target_id.clone());
|
|
let target_type_label: Cow<'static, str> = Cow::Owned(target_type.clone());
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_MESSAGES_MD, 0.0)
|
|
.with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone())
|
|
.with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD, 0.0)
|
|
.with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label.clone())
|
|
.with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label.clone()),
|
|
);
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, 0.0)
|
|
.with_label(NOTIFICATION_TARGET_ID_LABEL, target_id_label)
|
|
.with_label(NOTIFICATION_TARGET_TYPE_LABEL, target_type_label),
|
|
);
|
|
}
|
|
|
|
zero_metrics
|
|
}
|
|
|
|
fn retire_notification_target_metric_series(target_id: &str, target_type: &str) -> usize {
|
|
let labels = [
|
|
(NOTIFICATION_TARGET_ID_LABEL, Cow::Owned(target_id.to_string())),
|
|
(NOTIFICATION_TARGET_TYPE_LABEL, Cow::Owned(target_type.to_string())),
|
|
];
|
|
retire_metric_series(&NOTIFICATION_TARGET_FAILED_MESSAGES_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.get_full_metric_name(), &labels)
|
|
}
|
|
|
|
fn update_repl_bw_zero_tombstones(
|
|
monitor_available: bool,
|
|
has_seen_valid_snapshot: &mut bool,
|
|
prev_live_keys: &mut HashSet<ReplBwKey>,
|
|
zero_tombstones: &mut HashMap<ReplBwKey, u8>,
|
|
current_live_keys: HashSet<ReplBwKey>,
|
|
tombstone_cycles: u8,
|
|
) {
|
|
if !monitor_available {
|
|
return;
|
|
}
|
|
|
|
if *has_seen_valid_snapshot {
|
|
for removed in prev_live_keys.difference(¤t_live_keys) {
|
|
zero_tombstones.insert(removed.clone(), tombstone_cycles);
|
|
}
|
|
}
|
|
|
|
// Key becomes live again: stop zeroing immediately.
|
|
for key in ¤t_live_keys {
|
|
zero_tombstones.remove(key);
|
|
}
|
|
|
|
*prev_live_keys = current_live_keys;
|
|
*has_seen_valid_snapshot = true;
|
|
}
|
|
|
|
fn collect_repl_bw_zero_tombstone_metrics(zero_tombstones: &HashMap<ReplBwKey, u8>) -> Vec<PrometheusMetric> {
|
|
if zero_tombstones.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 2);
|
|
for (bucket, target_arn) in zero_tombstones.keys() {
|
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
|
|
let target_arn_label: Cow<'static, str> = Cow::Owned(target_arn.clone());
|
|
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, 0.0)
|
|
.with_label(BUCKET_L, bucket_label.clone())
|
|
.with_label(TARGET_ARN_L, target_arn_label.clone()),
|
|
);
|
|
|
|
zero_metrics.push(
|
|
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, 0.0)
|
|
.with_label(BUCKET_L, bucket_label)
|
|
.with_label(TARGET_ARN_L, target_arn_label),
|
|
);
|
|
}
|
|
|
|
zero_metrics
|
|
}
|
|
|
|
fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize {
|
|
let labels = [
|
|
(BUCKET_L, Cow::Owned(bucket.to_string())),
|
|
(TARGET_ARN_L, Cow::Owned(target_arn.to_string())),
|
|
];
|
|
retire_metric_series(&BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name(), &labels)
|
|
+ retire_metric_series(&BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name(), &labels)
|
|
}
|
|
|
|
fn expire_repl_bw_zero_tombstones(monitor_available: bool, zero_tombstones: &mut HashMap<ReplBwKey, u8>) -> Vec<ReplBwKey> {
|
|
if !monitor_available {
|
|
return Vec::new();
|
|
}
|
|
|
|
expire_series_zero_tombstones(zero_tombstones)
|
|
}
|
|
|
|
/// Initialize all metrics collectors.
|
|
///
|
|
/// This function spawns background tasks that periodically collect metrics
|
|
/// from various sources and report them to the metrics system.
|
|
///
|
|
/// # Arguments
|
|
/// * `token` - A `CancellationToken` that can be used to gracefully shut down
|
|
/// all metrics collection tasks.
|
|
///
|
|
/// # Environment Variables
|
|
/// The collection intervals can be configured via environment variables:
|
|
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`: Cluster metrics interval in seconds (default: 60)
|
|
/// - `RUSTFS_METRICS_BUCKET_INTERVAL_SEC`: Bucket metrics interval in seconds (default: 300)
|
|
/// - `RUSTFS_METRICS_NODE_INTERVAL_SEC`: Node/disk metrics interval in seconds (default: 60)
|
|
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC`: Bucket replication bandwidth interval in seconds (default: 30)
|
|
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL_SEC`: Resource metrics interval in seconds (default: 15)
|
|
/// - `RUSTFS_METRICS_DEFAULT_INTERVAL_SEC`: Optional global default interval in seconds.
|
|
///
|
|
/// Legacy interval names without `_SEC` are still accepted for backward compatibility:
|
|
/// - `RUSTFS_METRICS_CLUSTER_INTERVAL`
|
|
/// - `RUSTFS_METRICS_BUCKET_INTERVAL`
|
|
/// - `RUSTFS_METRICS_NODE_INTERVAL`
|
|
/// - `RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL`
|
|
/// - `RUSTFS_METRICS_RESOURCE_INTERVAL`
|
|
pub fn init_metrics_runtime(token: CancellationToken) {
|
|
let config = configured_metrics_runtime_config();
|
|
let health = metrics_runtime_collector_health();
|
|
let cluster_interval = config.cluster_interval;
|
|
let bucket_interval = config.bucket_interval;
|
|
let bucket_replication_bandwidth_interval = config.bucket_replication_bandwidth_interval;
|
|
let node_interval = config.node_interval;
|
|
let resource_interval = config.resource_interval;
|
|
let audit_interval = config.audit_interval;
|
|
let notification_interval = config.notification_interval;
|
|
let compression_enabled = obs_is_disk_compression_enabled();
|
|
|
|
// Spawn task for cluster metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
|
|
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
|
|
let mut metrics = collect_cluster_metrics(&stats);
|
|
metrics.extend(collect_cluster_health_metrics(&cluster_health));
|
|
report_metrics(&metrics);
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "cluster_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for supplementary cluster metrics that are defined in schema/collector
|
|
// but filled by later task-specific runtime sources.
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(cluster_interval, stagger_duration(cluster_interval, 1, 3));
|
|
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
|
let mut has_seen_bucket_usage_snapshot = false;
|
|
let mut prev_bucket_usage_keys: HashSet<BucketKey> = HashSet::new();
|
|
let mut bucket_usage_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
|
|
let mut prev_bucket_usage_object_size_keys: HashSet<BucketRangeKey> = HashSet::new();
|
|
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
|
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
|
|
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(
|
|
health,
|
|
MetricsCollectorTaskId::SupplementaryClusterStats,
|
|
"supplementary_cluster_stats",
|
|
async {
|
|
let mut metrics = Vec::new();
|
|
|
|
if let Some(stats) = collect_cluster_config_stats().await {
|
|
metrics.extend(collect_cluster_config_metrics(&stats));
|
|
}
|
|
|
|
let erasure_sets = collect_erasure_set_stats().await;
|
|
if !erasure_sets.is_empty() {
|
|
metrics.extend(collect_erasure_set_metrics(&erasure_sets));
|
|
}
|
|
|
|
if let Some(stats) = collect_iam_stats().await {
|
|
metrics.extend(collect_iam_metrics(&stats));
|
|
}
|
|
|
|
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
|
|
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_bucket_usage_snapshot,
|
|
&mut prev_bucket_usage_keys,
|
|
&mut bucket_usage_zero_tombstones,
|
|
bucket_usage_live_keys(&bucket_usage),
|
|
tombstone_cycles,
|
|
);
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_bucket_usage_snapshot,
|
|
&mut prev_bucket_usage_object_size_keys,
|
|
&mut bucket_usage_object_size_zero_tombstones,
|
|
bucket_usage_object_size_live_keys(&bucket_usage),
|
|
tombstone_cycles,
|
|
);
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_bucket_usage_snapshot,
|
|
&mut prev_bucket_usage_version_keys,
|
|
&mut bucket_usage_version_zero_tombstones,
|
|
bucket_usage_version_live_keys(&bucket_usage),
|
|
tombstone_cycles,
|
|
);
|
|
metrics.extend(collect_bucket_usage_metrics(&bucket_usage));
|
|
metrics.extend(collect_bucket_usage_zero_tombstone_metrics(
|
|
&bucket_usage_zero_tombstones,
|
|
&bucket_usage_object_size_zero_tombstones,
|
|
&bucket_usage_version_zero_tombstones,
|
|
));
|
|
for bucket in expire_series_zero_tombstones(&mut bucket_usage_zero_tombstones) {
|
|
let _ = retire_bucket_usage_metric_series(&bucket);
|
|
}
|
|
for (bucket, range) in expire_series_zero_tombstones(&mut bucket_usage_object_size_zero_tombstones) {
|
|
let _ = retire_bucket_usage_distribution_series(
|
|
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
|
|
&bucket,
|
|
&range,
|
|
);
|
|
}
|
|
for (bucket, range) in expire_series_zero_tombstones(&mut bucket_usage_version_zero_tombstones) {
|
|
let _ = retire_bucket_usage_distribution_series(
|
|
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
|
|
&bucket,
|
|
&range,
|
|
);
|
|
}
|
|
}
|
|
|
|
if !metrics.is_empty() {
|
|
report_metrics(&metrics);
|
|
}
|
|
},
|
|
).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "supplementary_cluster_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for bucket metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
|
|
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
|
let mut has_seen_bucket_snapshot = false;
|
|
let mut prev_bucket_keys: HashSet<BucketKey> = HashSet::new();
|
|
let mut bucket_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
|
|
let stats = collect_bucket_stats().await;
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_bucket_snapshot,
|
|
&mut prev_bucket_keys,
|
|
&mut bucket_zero_tombstones,
|
|
bucket_live_keys(&stats),
|
|
tombstone_cycles,
|
|
);
|
|
let mut metrics = collect_bucket_metrics(&stats);
|
|
metrics.extend(collect_bucket_zero_tombstone_metrics(&bucket_zero_tombstones));
|
|
report_metrics(&metrics);
|
|
for bucket in expire_series_zero_tombstones(&mut bucket_zero_tombstones) {
|
|
let _ = retire_bucket_metric_series(&bucket);
|
|
}
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "bucket_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for node/disk metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(node_interval, Duration::ZERO);
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::NodeDiskStats, "node_disk_stats", async {
|
|
let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_stats().await;
|
|
let mut metrics = collect_node_metrics(&disk_stats);
|
|
metrics.extend(collect_drive_detailed_metrics(&drive_stats));
|
|
metrics.extend(collect_drive_count_metrics(&drive_counts));
|
|
report_metrics(&metrics);
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "node_disk_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for bucket replication bandwidth metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(bucket_replication_bandwidth_interval, Duration::ZERO);
|
|
let repl_bw_zero_tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
|
let mut prev_live_keys: HashSet<ReplBwKey> = HashSet::new();
|
|
let mut zero_tombstones: HashMap<ReplBwKey, u8> = HashMap::new();
|
|
let mut has_seen_valid_snapshot = false;
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(
|
|
health,
|
|
MetricsCollectorTaskId::BucketReplicationBandwidth,
|
|
"bucket_replication_bandwidth",
|
|
async {
|
|
let monitor_available = bucket_monitor_available();
|
|
let stats = collect_bucket_replication_bandwidth_stats();
|
|
|
|
let current_live_keys = repl_bw_live_keys(&stats);
|
|
|
|
if !monitor_available {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "bucket_replication_bandwidth", result = "bucket_monitor_unavailable", "metrics runtime state changed");
|
|
}
|
|
update_repl_bw_zero_tombstones(
|
|
monitor_available,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
current_live_keys,
|
|
repl_bw_zero_tombstone_cycles,
|
|
);
|
|
let mut metrics = collect_bucket_replication_bandwidth_metrics(&stats);
|
|
|
|
// Phase-1 action: force zero for removed keys during tombstone cycles.
|
|
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
|
|
|
|
let bucket_replication = collect_bucket_replication_detail_stats().await;
|
|
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
|
|
let replication = collect_replication_stats().await;
|
|
metrics.extend(collect_replication_metrics(&replication));
|
|
report_metrics(&metrics);
|
|
|
|
// Phase-2: after N cycles, stop reporting -> series becomes absent after expiration.
|
|
for (bucket, target_arn) in expire_repl_bw_zero_tombstones(monitor_available, &mut zero_tombstones) {
|
|
let _ = retire_repl_bw_metric_series(&bucket, &target_arn);
|
|
}
|
|
},
|
|
).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "bucket_replication_bandwidth", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for audit target delivery metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(audit_interval, Duration::ZERO);
|
|
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
|
let mut has_seen_audit_snapshot = false;
|
|
let mut prev_audit_target_keys: HashSet<AuditTargetKey> = HashSet::new();
|
|
let mut audit_zero_tombstones: HashMap<AuditTargetKey, u8> = HashMap::new();
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::AuditTargetStats, "audit_target_stats", async {
|
|
let stats = audit_target_metrics().await
|
|
.into_iter()
|
|
.map(|snapshot| AuditTargetStats {
|
|
failed_messages: snapshot.failed_messages,
|
|
failed_store_length: snapshot.failed_store_length,
|
|
queue_length: snapshot.queue_length,
|
|
target_id: snapshot.target_id,
|
|
total_messages: snapshot.total_messages,
|
|
})
|
|
.collect::<Vec<_>>();
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_audit_snapshot,
|
|
&mut prev_audit_target_keys,
|
|
&mut audit_zero_tombstones,
|
|
audit_target_live_keys(&stats),
|
|
tombstone_cycles,
|
|
);
|
|
let mut metrics = collect_audit_metrics(&stats);
|
|
metrics.extend(collect_audit_zero_tombstone_metrics(&audit_zero_tombstones));
|
|
report_metrics(&metrics);
|
|
for target_id in expire_series_zero_tombstones(&mut audit_zero_tombstones) {
|
|
let _ = retire_audit_target_metric_series(&target_id);
|
|
}
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "audit_target_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for notification delivery metrics
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(notification_interval, Duration::ZERO);
|
|
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
|
let mut has_seen_notification_target_snapshot = false;
|
|
let mut prev_notification_target_keys: HashSet<NotificationTargetKey> = HashSet::new();
|
|
let mut notification_target_zero_tombstones: HashMap<NotificationTargetKey, u8> = HashMap::new();
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::NotificationStats, "notification_stats", async {
|
|
let snapshot = notification_metrics_snapshot();
|
|
let mut metrics = collect_notification_metrics(&NotificationStats {
|
|
current_send_in_progress: snapshot.current_send_in_progress,
|
|
events_errors_total: snapshot.events_errors_total,
|
|
events_sent_total: snapshot.events_sent_total,
|
|
events_skipped_total: snapshot.events_skipped_total,
|
|
});
|
|
|
|
let target_stats = notification_target_metrics().await
|
|
.into_iter()
|
|
.map(|snapshot| NotificationTargetStats {
|
|
failed_messages: snapshot.failed_messages,
|
|
failed_store_length: snapshot.failed_store_length,
|
|
queue_length: snapshot.queue_length,
|
|
target_id: snapshot.target_id,
|
|
target_type: snapshot.target_type,
|
|
total_messages: snapshot.total_messages,
|
|
})
|
|
.collect::<Vec<_>>();
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_notification_target_snapshot,
|
|
&mut prev_notification_target_keys,
|
|
&mut notification_target_zero_tombstones,
|
|
notification_target_live_keys(&target_stats),
|
|
tombstone_cycles,
|
|
);
|
|
metrics.extend(collect_notification_target_metrics(&target_stats));
|
|
metrics.extend(collect_notification_target_zero_tombstone_metrics(
|
|
¬ification_target_zero_tombstones,
|
|
));
|
|
report_metrics(&metrics);
|
|
for (target_id, target_type) in expire_series_zero_tombstones(&mut notification_target_zero_tombstones) {
|
|
let _ = retire_notification_target_metric_series(&target_id, &target_type);
|
|
}
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "notification_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for background workflow metrics such as ILM and scanner.
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(cluster_interval, stagger_duration(cluster_interval, 2, 3));
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(
|
|
health,
|
|
MetricsCollectorTaskId::BackgroundWorkflowStats,
|
|
"background_workflow_stats",
|
|
async {
|
|
let mut metrics = Vec::new();
|
|
|
|
if let Some(stats) = collect_ilm_metric_stats().await {
|
|
metrics.extend(collect_ilm_metrics(&stats));
|
|
}
|
|
|
|
if let Some(stats) = collect_scanner_metric_stats().await {
|
|
metrics.extend(collect_scanner_metrics(&stats));
|
|
}
|
|
|
|
if !metrics.is_empty() {
|
|
report_metrics(&metrics);
|
|
}
|
|
},
|
|
).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "background_workflow_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for system monitoring metrics (migrated from rustfs-obs::system)
|
|
let system_interval = config.system_interval;
|
|
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let labels = current_process_metric_labels();
|
|
let mut host_system = System::new_all();
|
|
let mut host_networks = Networks::new();
|
|
let mut process_sampler = ProcessSampler::new();
|
|
let process_interval = config.process_interval;
|
|
let mut interval = metrics_interval(process_interval, Duration::ZERO);
|
|
let now = Instant::now();
|
|
let mut next_resource_run = now;
|
|
let mut next_system_run = now;
|
|
|
|
host_system.refresh_cpu_all();
|
|
tokio::time::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await;
|
|
host_system.refresh_cpu_all();
|
|
|
|
#[cfg(feature = "gpu")]
|
|
let current_pid = match sysinfo::get_current_pid() {
|
|
Ok(pid) => Some(pid),
|
|
Err(e) => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "system_monitoring", result = "current_pid_unavailable", error = %e, "metrics runtime state changed");
|
|
None
|
|
}
|
|
};
|
|
|
|
#[cfg(feature = "gpu")]
|
|
let gpu_collector = current_pid.and_then(|pid| {
|
|
use crate::metrics::collectors::GpuCollector;
|
|
|
|
match GpuCollector::new(pid) {
|
|
Ok(collector) => Some(collector),
|
|
Err(e) => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "gpu_metrics", result = "collector_init_failed", error = %e, "metrics runtime state changed");
|
|
None
|
|
}
|
|
}
|
|
});
|
|
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::ProcessMetrics, "process_metrics", async {
|
|
let now = Instant::now();
|
|
let bundle = collect_process_metric_bundle_with(&mut process_sampler);
|
|
|
|
if now >= next_resource_run {
|
|
let mut metrics = collect_resource_metrics(&bundle.resource);
|
|
metrics.extend(collect_process_metrics(&bundle.process));
|
|
report_metrics(&metrics);
|
|
advance_deadline(&mut next_resource_run, resource_interval, now);
|
|
}
|
|
|
|
if now >= next_system_run {
|
|
#[cfg(feature = "gpu")]
|
|
let mut metrics =
|
|
collect_system_monitoring_metrics(&bundle, &labels, &mut host_system, &mut host_networks);
|
|
#[cfg(not(feature = "gpu"))]
|
|
let mut metrics =
|
|
collect_system_monitoring_metrics(&bundle, &labels, &mut host_system, &mut host_networks);
|
|
|
|
metrics.extend(collect_current_dial9_metrics());
|
|
|
|
#[cfg(feature = "gpu")]
|
|
if let Some(collector) = gpu_collector.as_ref() {
|
|
use crate::metrics::collectors::collect_gpu_metrics;
|
|
|
|
match collector.collect() {
|
|
Ok(gpu_stats) => {
|
|
metrics.extend(collect_gpu_metrics(&gpu_stats, &labels));
|
|
}
|
|
Err(e) => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "gpu_metrics", result = "collect_failed", error = %e, "metrics runtime state changed");
|
|
}
|
|
}
|
|
}
|
|
|
|
report_metrics(&metrics);
|
|
advance_deadline(&mut next_system_run, system_interval, now);
|
|
}
|
|
}).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "process_metrics", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Spawn task for compression metrics.
|
|
if compression_enabled {
|
|
let token_clone = token.clone();
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(cluster_interval, stagger_duration(cluster_interval, 3, 4));
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(
|
|
health,
|
|
MetricsCollectorTaskId::CompressionClusterStats,
|
|
"compression_cluster_stats",
|
|
async {
|
|
if let Some(stats) = collect_compression_cluster_stats().await {
|
|
let metrics = collect_compression_cluster_metrics(&stats);
|
|
if !metrics.is_empty() {
|
|
report_metrics(&metrics);
|
|
}
|
|
}
|
|
}
|
|
).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "compression_cluster_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// Spawn task for internode/system network metrics.
|
|
let token_clone = token;
|
|
tokio::spawn(async move {
|
|
let mut interval = metrics_interval(system_interval, Duration::ZERO);
|
|
loop {
|
|
tokio::select! {
|
|
_ = interval.tick() => {
|
|
run_metrics_collector_tick(
|
|
health,
|
|
MetricsCollectorTaskId::InternodeNetworkStats,
|
|
"internode_network_stats",
|
|
async {
|
|
if let Some(stats) = collect_internode_network_stats() {
|
|
let metrics = collect_network_metrics(&stats);
|
|
if !metrics.is_empty() {
|
|
report_metrics(&metrics);
|
|
}
|
|
}
|
|
},
|
|
).await;
|
|
}
|
|
_ = token_clone.cancelled() => {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "internode_network_stats", state = "cancelled", "metrics runtime state changed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
fn advance_deadline(deadline: &mut Instant, interval: Duration, now: Instant) {
|
|
if *deadline > now {
|
|
return;
|
|
}
|
|
|
|
let interval_nanos = interval.as_nanos();
|
|
if interval_nanos == 0 {
|
|
return;
|
|
}
|
|
|
|
let elapsed = now.duration_since(*deadline);
|
|
let missed_intervals = (elapsed.as_nanos() / interval_nanos) + 1;
|
|
let mut remaining = missed_intervals;
|
|
|
|
while remaining > 0 {
|
|
let chunk_u128 = remaining.min(u128::from(u32::MAX));
|
|
let chunk_u32 = chunk_u128 as u32;
|
|
|
|
if let Some(advance_by) = interval.checked_mul(chunk_u32) {
|
|
*deadline += advance_by;
|
|
remaining -= chunk_u128;
|
|
continue;
|
|
}
|
|
|
|
*deadline += interval;
|
|
remaining -= 1;
|
|
}
|
|
}
|
|
|
|
fn current_process_metric_labels() -> Vec<(&'static str, Cow<'static, str>)> {
|
|
match collect_process_attributes() {
|
|
Ok(attrs) => vec![
|
|
("process_pid", Cow::Owned(attrs.pid.to_string())),
|
|
("process_executable_name", Cow::Owned(attrs.executable_name)),
|
|
],
|
|
Err(err) => fallback_process_metric_labels(err),
|
|
}
|
|
}
|
|
|
|
fn fallback_process_metric_labels(err: ProcessAttributeError) -> Vec<(&'static str, Cow<'static, str>)> {
|
|
warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "process_metric_labels", result = "collect_failed", error = %err, "metrics runtime state changed");
|
|
vec![
|
|
("process_pid", Cow::Owned(std::process::id().to_string())),
|
|
("process_executable_name", Cow::Borrowed("unknown")),
|
|
]
|
|
}
|
|
|
|
fn collect_system_monitoring_metrics(
|
|
bundle: &ProcessMetricBundle,
|
|
labels: &[(&'static str, Cow<'static, str>)],
|
|
host_system: &mut System,
|
|
host_networks: &mut Networks,
|
|
) -> Vec<PrometheusMetric> {
|
|
let cpu_stats = ProcessCpuStats {
|
|
usage: bundle.resource.cpu_percent,
|
|
utilization: bundle.resource.cpu_percent,
|
|
};
|
|
let memory_stats = ProcessMemoryStats {
|
|
resident: bundle.process.resident_memory_bytes,
|
|
virtual_mem: bundle.process.virtual_memory_bytes,
|
|
};
|
|
let disk_stats = ProcessDiskStats {
|
|
read_bytes: bundle.disk_read_bytes,
|
|
written_bytes: bundle.disk_write_bytes,
|
|
};
|
|
let network_stats = collect_host_network_stats(host_networks);
|
|
let (system_cpu_stats, system_memory_stats) = collect_system_cpu_and_memory_stats_with(host_system);
|
|
|
|
let mut metrics = Vec::new();
|
|
metrics.extend(collect_cpu_metrics(&system_cpu_stats));
|
|
metrics.extend(collect_memory_metrics(&system_memory_stats));
|
|
metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(labels)));
|
|
metrics.extend(collect_process_memory_metrics(&memory_stats, Some(labels)));
|
|
metrics.extend(collect_process_disk_metrics(&disk_stats, Some(labels)));
|
|
// Interface counters are host-wide, so keep these metrics free of process labels.
|
|
metrics.extend(collect_host_network_metrics(&network_stats, None));
|
|
metrics
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::time::Duration;
|
|
use tokio::time::Instant;
|
|
|
|
fn fixed_metrics_runtime_config() -> MetricsRuntimeConfig {
|
|
MetricsRuntimeConfig {
|
|
cluster_interval: Duration::from_secs(60),
|
|
bucket_interval: Duration::from_secs(300),
|
|
bucket_replication_bandwidth_interval: Duration::from_secs(30),
|
|
node_interval: Duration::from_secs(60),
|
|
resource_interval: Duration::from_secs(15),
|
|
audit_interval: Duration::from_secs(45),
|
|
notification_interval: Duration::from_secs(20),
|
|
system_interval: Duration::from_secs(10),
|
|
process_interval: Duration::from_secs(10),
|
|
replication_bandwidth_zero_tombstone_cycles: 3,
|
|
}
|
|
}
|
|
|
|
fn reset_metrics_runtime_collector_health_for_test() {
|
|
let health = metrics_runtime_collector_health();
|
|
for entry in &health.last_success_unix_secs {
|
|
entry.store(0, Ordering::Relaxed);
|
|
}
|
|
for entry in &health.collector_panics_total {
|
|
entry.store(0, Ordering::Relaxed);
|
|
}
|
|
for entry in &health.failure_state {
|
|
entry.store(false, Ordering::Relaxed);
|
|
}
|
|
}
|
|
|
|
fn repl_bw_key(bucket: &str, target_arn: &str) -> ReplBwKey {
|
|
(bucket.to_string(), target_arn.to_string())
|
|
}
|
|
|
|
fn repl_bw_keys(keys: &[(&str, &str)]) -> HashSet<ReplBwKey> {
|
|
keys.iter()
|
|
.map(|(bucket, target_arn)| repl_bw_key(bucket, target_arn))
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_runtime_status_reports_disabled_state() {
|
|
let snapshot = build_metrics_runtime_status_snapshot(false, false, fixed_metrics_runtime_config(), false);
|
|
|
|
assert_eq!(snapshot.service, METRICS_RUNTIME_SERVICE_NAME);
|
|
assert_eq!(snapshot.state, MetricsRuntimeServiceState::Disabled);
|
|
assert!(!snapshot.metrics_enabled);
|
|
assert_eq!(snapshot.collector_tasks, METRICS_RUNTIME_BASE_COLLECTOR_TASKS);
|
|
assert_eq!(snapshot.collector_health, MetricsRuntimeCollectorHealthSnapshot::default());
|
|
assert_eq!(snapshot.intervals.cluster_interval_secs, 60);
|
|
assert_eq!(snapshot.intervals.bucket_interval_secs, 300);
|
|
assert_eq!(snapshot.intervals.process_interval_secs, 10);
|
|
assert_eq!(snapshot.intervals.replication_bandwidth_zero_tombstone_cycles, 3);
|
|
assert_eq!(snapshot.cancellation_source, MetricsRuntimeCancellationSource::RuntimeToken);
|
|
assert_eq!(snapshot.shutdown_handle, MetricsRuntimeShutdownHandle::RuntimeTokenOnly);
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_runtime_status_reports_running_and_stopping_states() {
|
|
reset_metrics_runtime_collector_health_for_test();
|
|
let running = build_metrics_runtime_status_snapshot(true, false, fixed_metrics_runtime_config(), false);
|
|
let stopping = build_metrics_runtime_status_snapshot(true, true, fixed_metrics_runtime_config(), false);
|
|
|
|
assert_eq!(running.state, MetricsRuntimeServiceState::Running);
|
|
assert_eq!(stopping.state, MetricsRuntimeServiceState::Stopping);
|
|
assert!(running.metrics_enabled);
|
|
assert!(stopping.metrics_enabled);
|
|
assert_eq!(running.collector_health.unhealthy_collectors, METRICS_RUNTIME_BASE_COLLECTOR_TASKS);
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_runtime_controller_reconcile_is_idempotent() {
|
|
let controller = MetricsRuntimeController;
|
|
let snapshot = build_metrics_runtime_controller_snapshot(true, false, fixed_metrics_runtime_config(), false);
|
|
|
|
let first = controller.reconcile_snapshot(snapshot);
|
|
let second = controller.reconcile_snapshot(snapshot);
|
|
|
|
assert_eq!(first, second);
|
|
assert_eq!(first.service, METRICS_RUNTIME_SERVICE_NAME);
|
|
assert_eq!(first.desired.state, MetricsRuntimeDesiredState::Enabled);
|
|
assert_eq!(first.current_state, MetricsRuntimeServiceState::Running);
|
|
assert_eq!(first.worker_mutation, MetricsRuntimeWorkerMutation::None);
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_runtime_controller_reports_disabled_without_worker_mutation() {
|
|
let controller = MetricsRuntimeController;
|
|
let snapshot = build_metrics_runtime_controller_snapshot(false, false, fixed_metrics_runtime_config(), false);
|
|
let plan = controller.reconcile_snapshot(snapshot);
|
|
|
|
assert_eq!(snapshot.desired.state, MetricsRuntimeDesiredState::Disabled);
|
|
assert_eq!(snapshot.status.state, MetricsRuntimeServiceState::Disabled);
|
|
assert_eq!(plan.current_state, MetricsRuntimeServiceState::Disabled);
|
|
assert_eq!(plan.worker_mutation, MetricsRuntimeWorkerMutation::None);
|
|
}
|
|
|
|
#[test]
|
|
fn advance_deadline_keeps_future_deadline_unchanged() {
|
|
let base = Instant::now();
|
|
let mut deadline = base + Duration::from_secs(10);
|
|
advance_deadline(&mut deadline, Duration::from_secs(5), base);
|
|
assert_eq!(deadline, base + Duration::from_secs(10));
|
|
}
|
|
|
|
#[test]
|
|
fn advance_deadline_moves_to_first_tick_after_now() {
|
|
let base = Instant::now();
|
|
let mut deadline = base;
|
|
advance_deadline(&mut deadline, Duration::from_secs(5), base + Duration::from_secs(12));
|
|
assert_eq!(deadline, base + Duration::from_secs(15));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn metrics_interval_uses_delay_missed_tick_behavior() {
|
|
let interval = metrics_interval(Duration::from_secs(5), Duration::from_secs(1));
|
|
assert_eq!(interval.missed_tick_behavior(), MissedTickBehavior::Delay);
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_runtime_collector_tasks_expand_when_compression_enabled() {
|
|
assert_eq!(metrics_runtime_collector_tasks(false), 10);
|
|
assert_eq!(metrics_runtime_collector_tasks(true), 11);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn supervised_tick_records_success_and_panic_health() {
|
|
let health = MetricsRuntimeCollectorHealth::new();
|
|
|
|
run_metrics_collector_tick(&health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {}).await;
|
|
let success = health.snapshot(&[MetricsCollectorTaskId::ClusterStats]);
|
|
assert_eq!(success.healthy_collectors, 1);
|
|
assert_eq!(success.unhealthy_collectors, 0);
|
|
assert_eq!(success.collector_panics_total, 0);
|
|
|
|
run_metrics_collector_tick(&health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
|
|
panic!("collector panic");
|
|
})
|
|
.await;
|
|
let failed = health.snapshot(&[MetricsCollectorTaskId::ClusterStats]);
|
|
assert_eq!(failed.healthy_collectors, 0);
|
|
assert_eq!(failed.unhealthy_collectors, 1);
|
|
assert_eq!(failed.collector_panics_total, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn repl_bw_tombstones_zero_removed_keys_then_expire() {
|
|
let mut has_seen_valid_snapshot = false;
|
|
let mut prev_live_keys = HashSet::new();
|
|
let mut zero_tombstones = HashMap::new();
|
|
let key = repl_bw_key("photos", "arn:rustfs:replication:target-a");
|
|
|
|
update_repl_bw_zero_tombstones(
|
|
true,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]),
|
|
2,
|
|
);
|
|
assert!(has_seen_valid_snapshot);
|
|
assert_eq!(prev_live_keys, repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]));
|
|
assert!(zero_tombstones.is_empty());
|
|
|
|
update_repl_bw_zero_tombstones(
|
|
true,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
HashSet::new(),
|
|
2,
|
|
);
|
|
assert_eq!(zero_tombstones.get(&key), Some(&2));
|
|
|
|
let metrics = collect_repl_bw_zero_tombstone_metrics(&zero_tombstones);
|
|
assert_eq!(metrics.len(), 2);
|
|
assert!(metrics.iter().all(|metric| metric.value == 0.0));
|
|
|
|
let names = metrics.iter().map(|metric| metric.name.to_string()).collect::<HashSet<_>>();
|
|
assert!(names.contains(&BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name()));
|
|
assert!(names.contains(&BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name()));
|
|
|
|
for metric in metrics {
|
|
let labels = metric
|
|
.labels
|
|
.into_iter()
|
|
.map(|(key, value)| (key, value.to_string()))
|
|
.collect::<HashMap<_, _>>();
|
|
assert_eq!(labels.get(BUCKET_L).map(String::as_str), Some("photos"));
|
|
assert_eq!(labels.get(TARGET_ARN_L).map(String::as_str), Some("arn:rustfs:replication:target-a"));
|
|
}
|
|
|
|
let expired = expire_repl_bw_zero_tombstones(true, &mut zero_tombstones);
|
|
assert!(expired.is_empty());
|
|
assert_eq!(zero_tombstones.get(&key), Some(&1));
|
|
|
|
let expired = expire_repl_bw_zero_tombstones(true, &mut zero_tombstones);
|
|
assert_eq!(expired, vec![key]);
|
|
assert!(zero_tombstones.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn repl_bw_tombstones_stop_zeroing_when_key_becomes_live_again() {
|
|
let mut has_seen_valid_snapshot = false;
|
|
let mut prev_live_keys = HashSet::new();
|
|
let mut zero_tombstones = HashMap::new();
|
|
let live_keys = repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]);
|
|
|
|
update_repl_bw_zero_tombstones(
|
|
true,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
live_keys.clone(),
|
|
3,
|
|
);
|
|
update_repl_bw_zero_tombstones(
|
|
true,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
HashSet::new(),
|
|
3,
|
|
);
|
|
assert_eq!(zero_tombstones.get(&repl_bw_key("photos", "arn:rustfs:replication:target-a")), Some(&3));
|
|
|
|
update_repl_bw_zero_tombstones(
|
|
true,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
live_keys.clone(),
|
|
3,
|
|
);
|
|
|
|
assert!(zero_tombstones.is_empty());
|
|
assert_eq!(prev_live_keys, live_keys);
|
|
}
|
|
|
|
#[test]
|
|
fn repl_bw_tombstones_do_not_advance_when_monitor_unavailable() {
|
|
let mut has_seen_valid_snapshot = true;
|
|
let mut prev_live_keys = repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]);
|
|
let mut zero_tombstones = HashMap::from([(repl_bw_key("videos", "arn:rustfs:replication:target-b"), 1)]);
|
|
|
|
update_repl_bw_zero_tombstones(
|
|
false,
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
HashSet::new(),
|
|
3,
|
|
);
|
|
|
|
assert!(has_seen_valid_snapshot);
|
|
assert_eq!(prev_live_keys, repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]));
|
|
assert_eq!(zero_tombstones.get(&repl_bw_key("videos", "arn:rustfs:replication:target-b")), Some(&1));
|
|
|
|
let expired = expire_repl_bw_zero_tombstones(false, &mut zero_tombstones);
|
|
assert!(expired.is_empty());
|
|
assert_eq!(zero_tombstones.get(&repl_bw_key("videos", "arn:rustfs:replication:target-b")), Some(&1));
|
|
}
|
|
|
|
#[test]
|
|
fn bucket_tombstones_zero_removed_buckets_then_expire() {
|
|
let mut has_seen_valid_snapshot = false;
|
|
let mut prev_live_keys = HashSet::new();
|
|
let mut zero_tombstones = HashMap::new();
|
|
let live_stats = vec![crate::metrics::collectors::BucketStats {
|
|
name: "tmp".to_string(),
|
|
size_bytes: 1024,
|
|
objects_count: 8,
|
|
quota_bytes: 2048,
|
|
}];
|
|
|
|
update_series_zero_tombstones(
|
|
&mut has_seen_valid_snapshot,
|
|
&mut prev_live_keys,
|
|
&mut zero_tombstones,
|
|
bucket_live_keys(&live_stats),
|
|
2,
|
|
);
|
|
assert!(zero_tombstones.is_empty());
|
|
|
|
update_series_zero_tombstones(&mut has_seen_valid_snapshot, &mut prev_live_keys, &mut zero_tombstones, HashSet::new(), 2);
|
|
assert_eq!(zero_tombstones.get("tmp"), Some(&2));
|
|
|
|
let metrics = collect_bucket_zero_tombstone_metrics(&zero_tombstones);
|
|
assert_eq!(metrics.len(), 3);
|
|
assert!(metrics.iter().all(|metric| metric.value == 0.0));
|
|
assert!(
|
|
metrics
|
|
.iter()
|
|
.all(|metric| { metric.labels.iter().any(|(key, value)| *key == "bucket" && value == "tmp") })
|
|
);
|
|
|
|
let expired = expire_series_zero_tombstones(&mut zero_tombstones);
|
|
assert!(expired.is_empty());
|
|
assert_eq!(zero_tombstones.get("tmp"), Some(&1));
|
|
|
|
let expired = expire_series_zero_tombstones(&mut zero_tombstones);
|
|
assert_eq!(expired, vec!["tmp".to_string()]);
|
|
assert!(zero_tombstones.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
|
|
temp_env::with_vars(
|
|
[
|
|
(ENV_SYSTEM_METRICS_INTERVAL, None::<&str>),
|
|
(LEGACY_SYSTEM_METRICS_INTERVAL, Some("500")),
|
|
(ENV_DEFAULT_METRICS_INTERVAL, None::<&str>),
|
|
],
|
|
|| {
|
|
assert_eq!(parse_system_metrics_interval(), Duration::from_secs(1));
|
|
},
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_system_metrics_interval_rounds_legacy_millis_up() {
|
|
temp_env::with_vars(
|
|
[
|
|
(ENV_SYSTEM_METRICS_INTERVAL, None::<&str>),
|
|
(LEGACY_SYSTEM_METRICS_INTERVAL, Some("1500")),
|
|
(ENV_DEFAULT_METRICS_INTERVAL, None::<&str>),
|
|
],
|
|
|| {
|
|
assert_eq!(parse_system_metrics_interval(), Duration::from_secs(2));
|
|
},
|
|
);
|
|
}
|
|
}
|