// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Observability and metrics for the audit system //! //! This module provides comprehensive observability features including: //! - Performance metrics (EPS, latency) //! - Target health monitoring //! - Configuration change tracking //! - Error rate monitoring //! - Queue depth monitoring use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tracing::info; const RUSTFS_AUDIT_METRICS_NAMESPACE: &str = "rustfs.audit."; const LOG_COMPONENT_AUDIT: &str = "audit"; const LOG_SUBSYSTEM_OBSERVABILITY: &str = "observability"; const EVENT_AUDIT_OBSERVABILITY_STATE: &str = "audit_observability_state"; const M_AUDIT_EVENTS_TOTAL: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "events.total"); const M_AUDIT_EVENTS_FAILED: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "events.failed"); const M_AUDIT_DISPATCH_NS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "dispatch.ns"); const M_AUDIT_EPS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "eps"); const M_AUDIT_TARGET_OPS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "target.ops"); const M_AUDIT_CONFIG_RELOADS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "config.reloads"); const M_AUDIT_SYSTEM_STARTS: &str = const_str::concat!(RUSTFS_AUDIT_METRICS_NAMESPACE, "system.starts"); const L_RESULT: &str = "result"; const L_STATUS: &str = "status"; const V_SUCCESS: &str = "success"; const V_FAILURE: &str = "failure"; /// One-time registration of indicator meta information /// This function ensures that metric descriptors are registered only once. pub fn init_observability_metrics() { static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new(); METRICS_DESC_INIT.get_or_init(|| { // Event/Time-consuming describe_counter!(M_AUDIT_EVENTS_TOTAL, "Total audit events (labeled by result)."); describe_counter!(M_AUDIT_EVENTS_FAILED, "Total failed audit events."); describe_histogram!(M_AUDIT_DISPATCH_NS, "Dispatch time per event (ns)."); describe_gauge!(M_AUDIT_EPS, "Events per second since last reset."); // Target operation/system event describe_counter!(M_AUDIT_TARGET_OPS, "Total target operations (labeled by status)."); describe_counter!(M_AUDIT_CONFIG_RELOADS, "Total configuration reloads."); describe_counter!(M_AUDIT_SYSTEM_STARTS, "Total system starts."); }); } /// Metrics collector for audit system observability #[derive(Debug)] pub struct AuditMetrics { // Performance metrics total_events_processed: AtomicU64, total_events_failed: AtomicU64, total_dispatch_time_ns: AtomicU64, // Target metrics target_success_count: AtomicU64, target_failure_count: AtomicU64, // System metrics config_reload_count: AtomicU64, system_start_count: AtomicU64, // Performance tracking last_reset_time: Arc>, } impl Default for AuditMetrics { fn default() -> Self { Self::new() } } impl AuditMetrics { /// Creates a new metrics collector pub fn new() -> Self { init_observability_metrics(); Self { total_events_processed: AtomicU64::new(0), total_events_failed: AtomicU64::new(0), total_dispatch_time_ns: AtomicU64::new(0), target_success_count: AtomicU64::new(0), target_failure_count: AtomicU64::new(0), config_reload_count: AtomicU64::new(0), system_start_count: AtomicU64::new(0), last_reset_time: Arc::new(RwLock::new(Instant::now())), } } // Suggestion: Call this auxiliary function in the existing "Successful Event Recording" method body to complete the instrumentation #[inline] fn emit_event_success_metrics(&self, dispatch_time: Duration) { // count + histogram counter!(M_AUDIT_EVENTS_TOTAL, L_RESULT => V_SUCCESS).increment(1); histogram!(M_AUDIT_DISPATCH_NS).record(dispatch_time.as_nanos() as f64); } // Suggestion: Call this auxiliary function in the existing "Failure Event Recording" method body to complete the instrumentation #[inline] fn emit_event_failure_metrics(&self, dispatch_time: Duration) { counter!(M_AUDIT_EVENTS_TOTAL, L_RESULT => V_FAILURE).increment(1); counter!(M_AUDIT_EVENTS_FAILED).increment(1); histogram!(M_AUDIT_DISPATCH_NS).record(dispatch_time.as_nanos() as f64); } /// Records a successful event dispatch pub fn record_event_success(&self, dispatch_time: Duration) { self.total_events_processed.fetch_add(1, Ordering::Relaxed); self.total_dispatch_time_ns .fetch_add(dispatch_time.as_nanos() as u64, Ordering::Relaxed); self.emit_event_success_metrics(dispatch_time); } /// Records a failed event dispatch pub fn record_event_failure(&self, dispatch_time: Duration) { self.total_events_failed.fetch_add(1, Ordering::Relaxed); self.total_dispatch_time_ns .fetch_add(dispatch_time.as_nanos() as u64, Ordering::Relaxed); self.emit_event_failure_metrics(dispatch_time); } /// Records a successful target operation pub fn record_target_success(&self) { self.target_success_count.fetch_add(1, Ordering::Relaxed); counter!(M_AUDIT_TARGET_OPS, L_STATUS => V_SUCCESS).increment(1); } /// Records a failed target operation pub fn record_target_failure(&self) { self.target_failure_count.fetch_add(1, Ordering::Relaxed); counter!(M_AUDIT_TARGET_OPS, L_STATUS => V_FAILURE).increment(1); } /// Records a configuration reload pub fn record_config_reload(&self) { self.config_reload_count.fetch_add(1, Ordering::Relaxed); counter!(M_AUDIT_CONFIG_RELOADS).increment(1); info!( event = EVENT_AUDIT_OBSERVABILITY_STATE, component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "config_reloaded", "Audit observability state updated" ); } /// Records a system start pub fn record_system_start(&self) { self.system_start_count.fetch_add(1, Ordering::Relaxed); counter!(M_AUDIT_SYSTEM_STARTS).increment(1); info!( event = EVENT_AUDIT_OBSERVABILITY_STATE, component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "system_started", "Audit observability state updated" ); } /// Gets the current events per second (EPS) pub async fn get_events_per_second(&self) -> f64 { let reset_time = *self.last_reset_time.read().await; let elapsed = reset_time.elapsed(); let total_events = self.total_events_processed.load(Ordering::Relaxed) + self.total_events_failed.load(Ordering::Relaxed); let eps = if elapsed.as_secs_f64() > 0.0 { total_events as f64 / elapsed.as_secs_f64() } else { 0.0 }; // EPS is reported in gauge gauge!(M_AUDIT_EPS).set(eps); eps } /// Gets the average dispatch latency in milliseconds pub fn get_average_latency_ms(&self) -> f64 { let total_events = self.total_events_processed.load(Ordering::Relaxed) + self.total_events_failed.load(Ordering::Relaxed); let total_time_ns = self.total_dispatch_time_ns.load(Ordering::Relaxed); if total_events > 0 { (total_time_ns as f64 / total_events as f64) / 1_000_000.0 // Convert ns to ms } else { 0.0 } } /// Gets the error rate as a percentage pub fn get_error_rate(&self) -> f64 { let total_events = self.total_events_processed.load(Ordering::Relaxed) + self.total_events_failed.load(Ordering::Relaxed); let failed_events = self.total_events_failed.load(Ordering::Relaxed); if total_events > 0 { (failed_events as f64 / total_events as f64) * 100.0 } else { 0.0 } } /// Gets target success rate as a percentage pub fn get_target_success_rate(&self) -> f64 { let total_ops = self.target_success_count.load(Ordering::Relaxed) + self.target_failure_count.load(Ordering::Relaxed); let success_ops = self.target_success_count.load(Ordering::Relaxed); if total_ops > 0 { (success_ops as f64 / total_ops as f64) * 100.0 } else { 100.0 // No operations = 100% success rate } } /// Resets all metrics and timing pub async fn reset(&self) { self.total_events_processed.store(0, Ordering::Relaxed); self.total_events_failed.store(0, Ordering::Relaxed); self.total_dispatch_time_ns.store(0, Ordering::Relaxed); self.target_success_count.store(0, Ordering::Relaxed); self.target_failure_count.store(0, Ordering::Relaxed); self.config_reload_count.store(0, Ordering::Relaxed); self.system_start_count.store(0, Ordering::Relaxed); let mut reset_time = self.last_reset_time.write().await; *reset_time = Instant::now(); // Reset EPS to zero after reset gauge!(M_AUDIT_EPS).set(0.0); info!( event = EVENT_AUDIT_OBSERVABILITY_STATE, component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "metrics_reset", "Audit observability state updated" ); } /// Generates a comprehensive metrics report pub async fn generate_report(&self) -> AuditMetricsReport { AuditMetricsReport { events_per_second: self.get_events_per_second().await, average_latency_ms: self.get_average_latency_ms(), error_rate_percent: self.get_error_rate(), target_success_rate_percent: self.get_target_success_rate(), total_events_processed: self.total_events_processed.load(Ordering::Relaxed), total_events_failed: self.total_events_failed.load(Ordering::Relaxed), config_reload_count: self.config_reload_count.load(Ordering::Relaxed), system_start_count: self.system_start_count.load(Ordering::Relaxed), } } /// Validates performance requirements pub async fn validate_performance_requirements(&self) -> PerformanceValidation { let eps = self.get_events_per_second().await; let avg_latency_ms = self.get_average_latency_ms(); let error_rate = self.get_error_rate(); let mut validation = PerformanceValidation { meets_eps_requirement: eps >= 3000.0, meets_latency_requirement: avg_latency_ms <= 30.0, meets_error_rate_requirement: error_rate <= 1.0, // Less than 1% error rate current_eps: eps, current_latency_ms: avg_latency_ms, current_error_rate: error_rate, recommendations: Vec::new(), }; // Generate recommendations if !validation.meets_eps_requirement { validation.recommendations.push(format!( "EPS ({eps:.0}) is below requirement (3000). Consider optimizing target dispatch or adding more target instances." )); } if !validation.meets_latency_requirement { validation.recommendations.push(format!( "Average latency ({avg_latency_ms:.2}ms) exceeds requirement (30ms). Consider optimizing target responses or increasing timeout values." )); } if !validation.meets_error_rate_requirement { validation.recommendations.push(format!( "Error rate ({error_rate:.2}%) exceeds recommendation (1%). Check target connectivity and configuration." )); } if validation.meets_eps_requirement && validation.meets_latency_requirement && validation.meets_error_rate_requirement { validation .recommendations .push("All performance requirements are met.".to_string()); } validation } } /// Comprehensive metrics report #[derive(Debug, Clone)] pub struct AuditMetricsReport { pub events_per_second: f64, pub average_latency_ms: f64, pub error_rate_percent: f64, pub target_success_rate_percent: f64, pub total_events_processed: u64, pub total_events_failed: u64, pub config_reload_count: u64, pub system_start_count: u64, } impl AuditMetricsReport { /// Formats the report as a human-readable string pub fn format(&self) -> String { format!( "Audit System Metrics Report:\n\ Events per Second: {:.2}\n\ Average Latency: {:.2}ms\n\ Error Rate: {:.2}%\n\ Target Success Rate: {:.2}%\n\ Total Events Processed: {}\n\ Total Events Failed: {}\n\ Configuration Reloads: {}\n\ System Starts: {}", self.events_per_second, self.average_latency_ms, self.error_rate_percent, self.target_success_rate_percent, self.total_events_processed, self.total_events_failed, self.config_reload_count, self.system_start_count ) } } /// Performance validation results #[derive(Debug, Clone)] pub struct PerformanceValidation { pub meets_eps_requirement: bool, pub meets_latency_requirement: bool, pub meets_error_rate_requirement: bool, pub current_eps: f64, pub current_latency_ms: f64, pub current_error_rate: f64, pub recommendations: Vec, } impl PerformanceValidation { /// Checks if all performance requirements are met pub fn all_requirements_met(&self) -> bool { self.meets_eps_requirement && self.meets_latency_requirement && self.meets_error_rate_requirement } /// Formats the validation as a human-readable string pub fn format(&self) -> String { let status = if self.all_requirements_met() { "✅ PASS" } else { "❌ FAIL" }; let mut result = format!( "Performance Requirements Validation: {}\n\ EPS Requirement (≥3000): {} ({:.2})\n\ Latency Requirement (≤30ms): {} ({:.2}ms)\n\ Error Rate Requirement (≤1%): {} ({:.2}%)\n\ \nRecommendations:", status, if self.meets_eps_requirement { "✅" } else { "❌" }, self.current_eps, if self.meets_latency_requirement { "✅" } else { "❌" }, self.current_latency_ms, if self.meets_error_rate_requirement { "✅" } else { "❌" }, self.current_error_rate ); for rec in &self.recommendations { result.push_str(&format!("\n• {rec}")); } result } } /// Global metrics instance static GLOBAL_METRICS: OnceLock> = OnceLock::new(); /// Get or initialize the global metrics instance pub fn global_metrics() -> Arc { GLOBAL_METRICS.get_or_init(|| Arc::new(AuditMetrics::new())).clone() } /// Record a successful audit event dispatch pub fn record_audit_success(dispatch_time: Duration) { global_metrics().record_event_success(dispatch_time); } /// Record a failed audit event dispatch pub fn record_audit_failure(dispatch_time: Duration) { global_metrics().record_event_failure(dispatch_time); } /// Record a successful target operation pub fn record_target_success() { global_metrics().record_target_success(); } /// Record a failed target operation pub fn record_target_failure() { global_metrics().record_target_failure(); } /// Record a configuration reload pub fn record_config_reload() { global_metrics().record_config_reload(); } /// Record a system start pub fn record_system_start() { global_metrics().record_system_start(); } /// Get the current metrics report pub async fn get_metrics_report() -> AuditMetricsReport { global_metrics().generate_report().await } /// Validate performance requirements pub async fn validate_performance() -> PerformanceValidation { global_metrics().validate_performance_requirements().await } /// Reset all metrics pub async fn reset_metrics() { global_metrics().reset().await; }