// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use std::{ collections::VecDeque, sync::{ Arc, atomic::{AtomicU64, Ordering}, }, time::{Duration, SystemTime}, }; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; use super::node_scanner::LoadLevel; use crate::error::Result; /// IO monitor config #[derive(Debug, Clone)] pub struct IOMonitorConfig { /// monitor interval pub monitor_interval: Duration, /// history data retention time pub history_retention: Duration, /// load evaluation window size pub load_window_size: usize, /// whether to enable actual system monitoring pub enable_system_monitoring: bool, /// disk path list (for monitoring specific disks) pub disk_paths: Vec, } impl Default for IOMonitorConfig { fn default() -> Self { Self { monitor_interval: Duration::from_secs(1), // 1 second monitor interval history_retention: Duration::from_secs(300), // keep 5 minutes history load_window_size: 30, // 30 sample points sliding window enable_system_monitoring: false, // default use simulated data disk_paths: Vec::new(), } } } /// IO monitor metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IOMetrics { /// timestamp pub timestamp: SystemTime, /// disk IOPS (read + write) pub iops: u64, /// read IOPS pub read_iops: u64, /// write IOPS pub write_iops: u64, /// disk queue depth pub queue_depth: u64, /// average latency (milliseconds) pub avg_latency: u64, /// read latency (milliseconds) pub read_latency: u64, /// write latency (milliseconds) pub write_latency: u64, /// CPU usage (0-100) pub cpu_usage: u8, /// memory usage (0-100) pub memory_usage: u8, /// disk usage (0-100) pub disk_utilization: u8, /// network IO (Mbps) pub network_io: u64, } impl Default for IOMetrics { fn default() -> Self { Self { timestamp: SystemTime::now(), iops: 0, read_iops: 0, write_iops: 0, queue_depth: 0, avg_latency: 0, read_latency: 0, write_latency: 0, cpu_usage: 0, memory_usage: 0, disk_utilization: 0, network_io: 0, } } } /// load level stats #[derive(Debug, Clone, Default)] pub struct LoadLevelStats { /// low load duration (seconds) pub low_load_duration: u64, /// medium load duration (seconds) pub medium_load_duration: u64, /// high load duration (seconds) pub high_load_duration: u64, /// critical load duration (seconds) pub critical_load_duration: u64, /// load transitions pub load_transitions: u64, } /// advanced IO monitor pub struct AdvancedIOMonitor { /// config config: Arc>, /// current metrics current_metrics: Arc>, /// history metrics (sliding window) history_metrics: Arc>>, /// current load level current_load_level: Arc>, /// load level history load_level_history: Arc>>, /// load level stats load_stats: Arc>, /// business IO metrics (updated by external) business_metrics: Arc, /// cancel token cancel_token: CancellationToken, } /// business IO metrics pub struct BusinessIOMetrics { /// business request latency (milliseconds) pub request_latency: AtomicU64, /// business request QPS pub request_qps: AtomicU64, /// business error rate (0-10000, 0.00%-100.00%) pub error_rate: AtomicU64, /// active connections pub active_connections: AtomicU64, /// last update time pub last_update: Arc>, } impl Default for BusinessIOMetrics { fn default() -> Self { Self { request_latency: AtomicU64::new(0), request_qps: AtomicU64::new(0), error_rate: AtomicU64::new(0), active_connections: AtomicU64::new(0), last_update: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)), } } } impl AdvancedIOMonitor { /// create new advanced IO monitor pub fn new(config: IOMonitorConfig) -> Self { Self { config: Arc::new(RwLock::new(config)), current_metrics: Arc::new(RwLock::new(IOMetrics::default())), history_metrics: Arc::new(RwLock::new(VecDeque::new())), current_load_level: Arc::new(RwLock::new(LoadLevel::Low)), load_level_history: Arc::new(RwLock::new(VecDeque::new())), load_stats: Arc::new(RwLock::new(LoadLevelStats::default())), business_metrics: Arc::new(BusinessIOMetrics::default()), cancel_token: CancellationToken::new(), } } /// start monitoring pub async fn start(&self) -> Result<()> { info!("start advanced IO monitor"); let monitor = self.clone_for_background(); tokio::spawn(async move { if let Err(e) = monitor.monitoring_loop().await { error!("IO monitoring loop failed: {}", e); } }); Ok(()) } /// stop monitoring pub async fn stop(&self) { info!("stop IO monitor"); self.cancel_token.cancel(); } /// monitoring loop async fn monitoring_loop(&self) -> Result<()> { let mut interval = { let config = self.config.read().await; tokio::time::interval(config.monitor_interval) }; let mut last_load_level = LoadLevel::Low; let mut load_level_start_time = SystemTime::now(); loop { tokio::select! { _ = self.cancel_token.cancelled() => { info!("IO monitoring loop cancelled"); break; } _ = interval.tick() => { // collect system metrics let metrics = self.collect_system_metrics().await; // update current metrics *self.current_metrics.write().await = metrics.clone(); // update history metrics self.update_metrics_history(metrics.clone()).await; // calculate load level let new_load_level = self.calculate_load_level(&metrics).await; // check if load level changed if new_load_level != last_load_level { self.handle_load_level_change(last_load_level, new_load_level, load_level_start_time).await; last_load_level = new_load_level; load_level_start_time = SystemTime::now(); } // update current load level *self.current_load_level.write().await = new_load_level; debug!("IO monitor updated: IOPS={}, queue depth={}, latency={}ms, load level={:?}", metrics.iops, metrics.queue_depth, metrics.avg_latency, new_load_level); } } } Ok(()) } /// collect system metrics async fn collect_system_metrics(&self) -> IOMetrics { let config = self.config.read().await; if config.enable_system_monitoring { // actual system monitoring implementation self.collect_real_system_metrics().await } else { // simulated data self.generate_simulated_metrics().await } } /// collect real system metrics (need to be implemented according to specific system) async fn collect_real_system_metrics(&self) -> IOMetrics { // TODO: implement actual system metrics collection // can use procfs, sysfs or other system API let metrics = IOMetrics { timestamp: SystemTime::now(), ..Default::default() }; // example: read /proc/diskstats if let Ok(diskstats) = tokio::fs::read_to_string("/proc/diskstats").await { // parse disk stats info // here need to implement specific parsing logic debug!("read disk stats info: {} bytes", diskstats.len()); } // example: read /proc/stat to get CPU info if let Ok(stat) = tokio::fs::read_to_string("/proc/stat").await { // parse CPU stats info debug!("read CPU stats info: {} bytes", stat.len()); } // example: read /proc/meminfo to get memory info if let Ok(meminfo) = tokio::fs::read_to_string("/proc/meminfo").await { // parse memory stats info debug!("read memory stats info: {} bytes", meminfo.len()); } metrics } /// generate simulated metrics (for testing and development) async fn generate_simulated_metrics(&self) -> IOMetrics { use rand::Rng; let mut rng = rand::rng(); // get business metrics impact let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed); let business_qps = self.business_metrics.request_qps.load(Ordering::Relaxed); // generate simulated system metrics based on business load let base_iops = 100 + (business_qps / 10); let base_latency = 5 + (business_latency / 10); IOMetrics { timestamp: SystemTime::now(), iops: base_iops + rng.random_range(0..50), read_iops: (base_iops * 6 / 10) + rng.random_range(0..20), write_iops: (base_iops * 4 / 10) + rng.random_range(0..20), queue_depth: rng.random_range(1..20), avg_latency: base_latency + rng.random_range(0..10), read_latency: base_latency + rng.random_range(0..5), write_latency: base_latency + rng.random_range(0..15), cpu_usage: rng.random_range(10..70), memory_usage: rng.random_range(30..80), disk_utilization: rng.random_range(20..90), network_io: rng.random_range(10..1000), } } /// update metrics history async fn update_metrics_history(&self, metrics: IOMetrics) { let mut history = self.history_metrics.write().await; let config = self.config.read().await; // add new metrics history.push_back(metrics); // clean expired data let retention_cutoff = SystemTime::now() - config.history_retention; while let Some(front) = history.front() { if front.timestamp < retention_cutoff { history.pop_front(); } else { break; } } // limit window size while history.len() > config.load_window_size { history.pop_front(); } } /// calculate load level async fn calculate_load_level(&self, metrics: &IOMetrics) -> LoadLevel { // multi-dimensional load evaluation algorithm let mut load_score = 0u32; // IOPS load evaluation (weight: 25%) let iops_score = match metrics.iops { 0..=200 => 0, 201..=500 => 15, 501..=1000 => 25, _ => 35, }; load_score += iops_score; // latency load evaluation (weight: 30%) let latency_score = match metrics.avg_latency { 0..=10 => 0, 11..=50 => 20, 51..=100 => 30, _ => 40, }; load_score += latency_score; // queue depth evaluation (weight: 20%) let queue_score = match metrics.queue_depth { 0..=5 => 0, 6..=15 => 10, 16..=30 => 20, _ => 25, }; load_score += queue_score; // CPU usage evaluation (weight: 15%) let cpu_score = match metrics.cpu_usage { 0..=30 => 0, 31..=60 => 8, 61..=80 => 12, _ => 15, }; load_score += cpu_score; // disk usage evaluation (weight: 10%) let disk_score = match metrics.disk_utilization { 0..=50 => 0, 51..=75 => 5, 76..=90 => 8, _ => 10, }; load_score += disk_score; // business metrics impact let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed); let business_error_rate = self.business_metrics.error_rate.load(Ordering::Relaxed); if business_latency > 100 { load_score += 20; // business latency too high } if business_error_rate > 100 { // > 1% load_score += 15; // business error rate too high } // history trend analysis let trend_score = self.calculate_trend_score().await; load_score += trend_score; // determine load level based on total score match load_score { 0..=30 => LoadLevel::Low, 31..=60 => LoadLevel::Medium, 61..=90 => LoadLevel::High, _ => LoadLevel::Critical, } } /// calculate trend score async fn calculate_trend_score(&self) -> u32 { let history = self.history_metrics.read().await; if history.len() < 5 { return 0; // data insufficient, cannot analyze trend } // analyze trend of last 5 samples let recent: Vec<_> = history.iter().rev().take(5).collect(); // check IOPS rising trend let mut iops_trend = 0; for i in 1..recent.len() { if recent[i - 1].iops > recent[i].iops { iops_trend += 1; } } // check latency rising trend let mut latency_trend = 0; for i in 1..recent.len() { if recent[i - 1].avg_latency > recent[i].avg_latency { latency_trend += 1; } } // if IOPS and latency are both rising, increase load score if iops_trend >= 3 && latency_trend >= 3 { 15 // obvious rising trend } else if iops_trend >= 2 || latency_trend >= 2 { 5 // slight rising trend } else { 0 // no obvious trend } } /// handle load level change async fn handle_load_level_change(&self, old_level: LoadLevel, new_level: LoadLevel, start_time: SystemTime) { let duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO); // update stats { let mut stats = self.load_stats.write().await; match old_level { LoadLevel::Low => stats.low_load_duration += duration.as_secs(), LoadLevel::Medium => stats.medium_load_duration += duration.as_secs(), LoadLevel::High => stats.high_load_duration += duration.as_secs(), LoadLevel::Critical => stats.critical_load_duration += duration.as_secs(), } stats.load_transitions += 1; } // update history { let mut history = self.load_level_history.write().await; history.push_back((SystemTime::now(), new_level)); // keep history record in reasonable range while history.len() > 100 { history.pop_front(); } } info!("load level changed: {:?} -> {:?}, duration: {:?}", old_level, new_level, duration); // if enter critical load state, record warning if new_level == LoadLevel::Critical { warn!("system entered critical load state, Scanner will pause running"); } } /// get current load level pub async fn get_business_load_level(&self) -> LoadLevel { *self.current_load_level.read().await } /// get current metrics pub async fn get_current_metrics(&self) -> IOMetrics { self.current_metrics.read().await.clone() } /// get history metrics pub async fn get_history_metrics(&self) -> Vec { self.history_metrics.read().await.iter().cloned().collect() } /// get load stats pub async fn get_load_stats(&self) -> LoadLevelStats { self.load_stats.read().await.clone() } /// update business IO metrics pub async fn update_business_metrics(&self, latency: u64, qps: u64, error_rate: u64, connections: u64) { self.business_metrics.request_latency.store(latency, Ordering::Relaxed); self.business_metrics.request_qps.store(qps, Ordering::Relaxed); self.business_metrics.error_rate.store(error_rate, Ordering::Relaxed); self.business_metrics.active_connections.store(connections, Ordering::Relaxed); *self.business_metrics.last_update.write().await = SystemTime::now(); debug!( "update business metrics: latency={}ms, QPS={}, error rate={}‰, connections={}", latency, qps, error_rate, connections ); } /// clone for background task fn clone_for_background(&self) -> Self { Self { config: self.config.clone(), current_metrics: self.current_metrics.clone(), history_metrics: self.history_metrics.clone(), current_load_level: self.current_load_level.clone(), load_level_history: self.load_level_history.clone(), load_stats: self.load_stats.clone(), business_metrics: self.business_metrics.clone(), cancel_token: self.cancel_token.clone(), } } /// reset stats pub async fn reset_stats(&self) { *self.load_stats.write().await = LoadLevelStats::default(); self.load_level_history.write().await.clear(); self.history_metrics.write().await.clear(); info!("IO monitor stats reset"); } /// get load level history pub async fn get_load_level_history(&self) -> Vec<(SystemTime, LoadLevel)> { self.load_level_history.read().await.iter().cloned().collect() } }