mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
Feature/scanner performance optimization (#498)
* Refactor: reimplement scanner Signed-off-by: RustFS Developer <dandan@rustfs.com> * comment lock Signed-off-by: junxiang Mu <1948535941@qq.com> * remove dirty file Signed-off-by: junxiang Mu <1948535941@qq.com> * Fix: fix rebase * fix(scanner): Improve error handling and logging Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: RustFS Developer <dandan@rustfs.com> Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: RustFS Developer <dandan@rustfs.com>
This commit is contained in:
guojidan
+14
-5
@@ -14,10 +14,8 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// RustFS AHM/Heal/Scanner 统一错误类型
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
// 通用
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
@@ -39,14 +37,26 @@ pub enum Error {
|
||||
#[error(transparent)]
|
||||
Anyhow(#[from] anyhow::Error),
|
||||
|
||||
// Scanner相关
|
||||
// Scanner
|
||||
#[error("Scanner error: {0}")]
|
||||
Scanner(String),
|
||||
|
||||
#[error("Metrics error: {0}")]
|
||||
Metrics(String),
|
||||
|
||||
// Heal相关
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IO(String),
|
||||
|
||||
#[error("Not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Invalid checkpoint: {0}")]
|
||||
InvalidCheckpoint(String),
|
||||
|
||||
// Heal
|
||||
#[error("Heal task not found: {task_id}")]
|
||||
TaskNotFound { task_id: String },
|
||||
|
||||
@@ -86,7 +96,6 @@ impl Error {
|
||||
}
|
||||
}
|
||||
|
||||
// 可选：实现与 std::io::Error 的互转
|
||||
impl From<Error> for std::io::Error {
|
||||
fn from(err: Error) -> Self {
|
||||
std::io::Error::other(err)
|
||||
|
||||
@@ -299,7 +299,7 @@ impl HealTask {
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(0, 4, 0, 0); // 开始heal，总共4个步骤
|
||||
progress.update_progress(0, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if object exists and get metadata
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::node_scanner::ScanProgress;
|
||||
use crate::{Error, error::Result};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct CheckpointData {
|
||||
pub version: u32,
|
||||
pub timestamp: SystemTime,
|
||||
pub progress: ScanProgress,
|
||||
pub node_id: String,
|
||||
pub checksum: u64,
|
||||
}
|
||||
|
||||
impl CheckpointData {
|
||||
pub fn new(progress: ScanProgress, node_id: String) -> Self {
|
||||
let mut checkpoint = Self {
|
||||
version: 1,
|
||||
timestamp: SystemTime::now(),
|
||||
progress,
|
||||
node_id,
|
||||
checksum: 0,
|
||||
};
|
||||
|
||||
checkpoint.checksum = checkpoint.calculate_checksum();
|
||||
checkpoint
|
||||
}
|
||||
|
||||
fn calculate_checksum(&self) -> u64 {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
self.version.hash(&mut hasher);
|
||||
self.node_id.hash(&mut hasher);
|
||||
self.progress.current_cycle.hash(&mut hasher);
|
||||
self.progress.current_disk_index.hash(&mut hasher);
|
||||
|
||||
if let Some(ref bucket) = self.progress.current_bucket {
|
||||
bucket.hash(&mut hasher);
|
||||
}
|
||||
|
||||
if let Some(ref key) = self.progress.last_scan_key {
|
||||
key.hash(&mut hasher);
|
||||
}
|
||||
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
pub fn verify_integrity(&self) -> bool {
|
||||
let calculated_checksum = self.calculate_checksum();
|
||||
self.checksum == calculated_checksum
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CheckpointManager {
|
||||
checkpoint_file: PathBuf,
|
||||
backup_file: PathBuf,
|
||||
temp_file: PathBuf,
|
||||
save_interval: Duration,
|
||||
last_save: RwLock<SystemTime>,
|
||||
node_id: String,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
||||
if !data_dir.exists() {
|
||||
if let Err(e) = std::fs::create_dir_all(data_dir) {
|
||||
error!("create data dir failed {:?}: {}", data_dir, e);
|
||||
}
|
||||
}
|
||||
|
||||
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{}.json", node_id));
|
||||
let backup_file = data_dir.join(format!("scanner_checkpoint_{}.backup", node_id));
|
||||
let temp_file = data_dir.join(format!("scanner_checkpoint_{}.tmp", node_id));
|
||||
|
||||
Self {
|
||||
checkpoint_file,
|
||||
backup_file,
|
||||
temp_file,
|
||||
save_interval: Duration::from_secs(30), // 30s
|
||||
last_save: RwLock::new(SystemTime::UNIX_EPOCH),
|
||||
node_id: node_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
|
||||
let now = SystemTime::now();
|
||||
let last_save = *self.last_save.read().await;
|
||||
|
||||
if now.duration_since(last_save).unwrap_or(Duration::ZERO) < self.save_interval {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let checkpoint_data = CheckpointData::new(progress.clone(), self.node_id.clone());
|
||||
|
||||
let json_data = serde_json::to_string_pretty(&checkpoint_data)
|
||||
.map_err(|e| Error::Serialization(format!("serialize checkpoint failed: {}", e)))?;
|
||||
|
||||
tokio::fs::write(&self.temp_file, json_data)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("write temp checkpoint file failed: {}", e)))?;
|
||||
|
||||
if self.checkpoint_file.exists() {
|
||||
tokio::fs::copy(&self.checkpoint_file, &self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("backup checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
tokio::fs::rename(&self.temp_file, &self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("replace checkpoint file failed: {}", e)))?;
|
||||
|
||||
*self.last_save.write().await = now;
|
||||
|
||||
debug!(
|
||||
"save checkpoint to {:?}, cycle: {}, disk index: {}",
|
||||
self.checkpoint_file, checkpoint_data.progress.current_cycle, checkpoint_data.progress.current_disk_index
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn load_checkpoint(&self) -> Result<Option<ScanProgress>> {
|
||||
// first try main checkpoint file
|
||||
match self.load_checkpoint_from_file(&self.checkpoint_file).await {
|
||||
Ok(checkpoint) => {
|
||||
info!(
|
||||
"restore scan progress from main checkpoint file: cycle={}, disk index={}, last scan key={:?}",
|
||||
checkpoint.current_cycle, checkpoint.current_disk_index, checkpoint.last_scan_key
|
||||
);
|
||||
Ok(Some(checkpoint))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("main checkpoint file is corrupted or not exists: {}", e);
|
||||
|
||||
// try backup file
|
||||
match self.load_checkpoint_from_file(&self.backup_file).await {
|
||||
Ok(checkpoint) => {
|
||||
warn!(
|
||||
"restore scan progress from backup file: cycle={}, disk index={}",
|
||||
checkpoint.current_cycle, checkpoint.current_disk_index
|
||||
);
|
||||
|
||||
// copy backup file to main checkpoint file
|
||||
if let Err(copy_err) = tokio::fs::copy(&self.backup_file, &self.checkpoint_file).await {
|
||||
warn!("restore main checkpoint file failed: {}", copy_err);
|
||||
}
|
||||
|
||||
Ok(Some(checkpoint))
|
||||
}
|
||||
Err(backup_e) => {
|
||||
warn!("backup file is corrupted or not exists: {}", backup_e);
|
||||
info!("cannot restore scan progress, will start fresh scan");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// load checkpoint from file
|
||||
async fn load_checkpoint_from_file(&self, file_path: &Path) -> Result<ScanProgress> {
|
||||
if !file_path.exists() {
|
||||
return Err(Error::NotFound(format!("checkpoint file not exists: {:?}", file_path)));
|
||||
}
|
||||
|
||||
// read file content
|
||||
let content = tokio::fs::read_to_string(file_path)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("read checkpoint file failed: {}", e)))?;
|
||||
|
||||
// deserialize
|
||||
let checkpoint_data: CheckpointData =
|
||||
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {}", e)))?;
|
||||
|
||||
// validate checkpoint data
|
||||
self.validate_checkpoint(&checkpoint_data)?;
|
||||
|
||||
Ok(checkpoint_data.progress)
|
||||
}
|
||||
|
||||
/// validate checkpoint data
|
||||
fn validate_checkpoint(&self, checkpoint: &CheckpointData) -> Result<()> {
|
||||
// validate data integrity
|
||||
if !checkpoint.verify_integrity() {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"checkpoint data verification failed, may be corrupted".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// validate node id match
|
||||
if checkpoint.node_id != self.node_id {
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"checkpoint node id not match: expected {}, actual {}",
|
||||
self.node_id, checkpoint.node_id
|
||||
)));
|
||||
}
|
||||
|
||||
let now = SystemTime::now();
|
||||
let checkpoint_age = now.duration_since(checkpoint.timestamp).unwrap_or(Duration::MAX);
|
||||
|
||||
// checkpoint is too old (more than 24 hours), may be data expired
|
||||
if checkpoint_age > Duration::from_secs(24 * 3600) {
|
||||
return Err(Error::InvalidCheckpoint(format!("checkpoint data is too old: {:?}", checkpoint_age)));
|
||||
}
|
||||
|
||||
// validate version compatibility
|
||||
if checkpoint.version > 1 {
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"unsupported checkpoint version: {}",
|
||||
checkpoint.version
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// clean checkpoint file
|
||||
///
|
||||
/// called when scanner stops or resets
|
||||
pub async fn cleanup_checkpoint(&self) -> Result<()> {
|
||||
// delete main file
|
||||
if self.checkpoint_file.exists() {
|
||||
tokio::fs::remove_file(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete main checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete backup file
|
||||
if self.backup_file.exists() {
|
||||
tokio::fs::remove_file(&self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete backup checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete temp file
|
||||
if self.temp_file.exists() {
|
||||
tokio::fs::remove_file(&self.temp_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete temp checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
info!("cleaned up all checkpoint files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// get checkpoint file info
|
||||
pub async fn get_checkpoint_info(&self) -> Result<Option<CheckpointInfo>> {
|
||||
if !self.checkpoint_file.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let metadata = tokio::fs::metadata(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("get checkpoint file metadata failed: {}", e)))?;
|
||||
|
||||
let content = tokio::fs::read_to_string(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("read checkpoint file failed: {}", e)))?;
|
||||
|
||||
let checkpoint_data: CheckpointData =
|
||||
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {}", e)))?;
|
||||
|
||||
Ok(Some(CheckpointInfo {
|
||||
file_size: metadata.len(),
|
||||
last_modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
|
||||
checkpoint_timestamp: checkpoint_data.timestamp,
|
||||
current_cycle: checkpoint_data.progress.current_cycle,
|
||||
current_disk_index: checkpoint_data.progress.current_disk_index,
|
||||
completed_disks_count: checkpoint_data.progress.completed_disks.len(),
|
||||
is_valid: checkpoint_data.verify_integrity(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// force save checkpoint (ignore time interval limit)
|
||||
pub async fn force_save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
|
||||
// temporarily reset last save time, force save
|
||||
*self.last_save.write().await = SystemTime::UNIX_EPOCH;
|
||||
self.save_checkpoint(progress).await
|
||||
}
|
||||
|
||||
/// set save interval
|
||||
pub async fn set_save_interval(&mut self, interval: Duration) {
|
||||
self.save_interval = interval;
|
||||
info!("checkpoint save interval set to: {:?}", interval);
|
||||
}
|
||||
}
|
||||
|
||||
/// checkpoint info
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CheckpointInfo {
|
||||
/// file size
|
||||
pub file_size: u64,
|
||||
/// file last modified time
|
||||
pub last_modified: SystemTime,
|
||||
/// checkpoint creation time
|
||||
pub checkpoint_timestamp: SystemTime,
|
||||
/// current scan cycle
|
||||
pub current_cycle: u64,
|
||||
/// current disk index
|
||||
pub current_disk_index: usize,
|
||||
/// completed disks count
|
||||
pub completed_disks_count: usize,
|
||||
/// checkpoint is valid
|
||||
pub is_valid: bool,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,557 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::node_scanner::LoadLevel;
|
||||
use crate::error::Result;
|
||||
|
||||
/// IO monitor config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IOMonitorConfig {
|
||||
/// monitor interval
|
||||
pub monitor_interval: Duration,
|
||||
/// history data retention time
|
||||
pub history_retention: Duration,
|
||||
/// load evaluation window size
|
||||
pub load_window_size: usize,
|
||||
/// whether to enable actual system monitoring
|
||||
pub enable_system_monitoring: bool,
|
||||
/// disk path list (for monitoring specific disks)
|
||||
pub disk_paths: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for IOMonitorConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
monitor_interval: Duration::from_secs(1), // 1 second monitor interval
|
||||
history_retention: Duration::from_secs(300), // keep 5 minutes history
|
||||
load_window_size: 30, // 30 sample points sliding window
|
||||
enable_system_monitoring: false, // default use simulated data
|
||||
disk_paths: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// IO monitor metrics
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct IOMetrics {
|
||||
/// timestamp
|
||||
pub timestamp: SystemTime,
|
||||
/// disk IOPS (read + write)
|
||||
pub iops: u64,
|
||||
/// read IOPS
|
||||
pub read_iops: u64,
|
||||
/// write IOPS
|
||||
pub write_iops: u64,
|
||||
/// disk queue depth
|
||||
pub queue_depth: u64,
|
||||
/// average latency (milliseconds)
|
||||
pub avg_latency: u64,
|
||||
/// read latency (milliseconds)
|
||||
pub read_latency: u64,
|
||||
/// write latency (milliseconds)
|
||||
pub write_latency: u64,
|
||||
/// CPU usage (0-100)
|
||||
pub cpu_usage: u8,
|
||||
/// memory usage (0-100)
|
||||
pub memory_usage: u8,
|
||||
/// disk usage (0-100)
|
||||
pub disk_utilization: u8,
|
||||
/// network IO (Mbps)
|
||||
pub network_io: u64,
|
||||
}
|
||||
|
||||
impl Default for IOMetrics {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
timestamp: SystemTime::now(),
|
||||
iops: 0,
|
||||
read_iops: 0,
|
||||
write_iops: 0,
|
||||
queue_depth: 0,
|
||||
avg_latency: 0,
|
||||
read_latency: 0,
|
||||
write_latency: 0,
|
||||
cpu_usage: 0,
|
||||
memory_usage: 0,
|
||||
disk_utilization: 0,
|
||||
network_io: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// load level stats
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct LoadLevelStats {
|
||||
/// low load duration (seconds)
|
||||
pub low_load_duration: u64,
|
||||
/// medium load duration (seconds)
|
||||
pub medium_load_duration: u64,
|
||||
/// high load duration (seconds)
|
||||
pub high_load_duration: u64,
|
||||
/// critical load duration (seconds)
|
||||
pub critical_load_duration: u64,
|
||||
/// load transitions
|
||||
pub load_transitions: u64,
|
||||
}
|
||||
|
||||
/// advanced IO monitor
|
||||
pub struct AdvancedIOMonitor {
|
||||
/// config
|
||||
config: Arc<RwLock<IOMonitorConfig>>,
|
||||
/// current metrics
|
||||
current_metrics: Arc<RwLock<IOMetrics>>,
|
||||
/// history metrics (sliding window)
|
||||
history_metrics: Arc<RwLock<VecDeque<IOMetrics>>>,
|
||||
/// current load level
|
||||
current_load_level: Arc<RwLock<LoadLevel>>,
|
||||
/// load level history
|
||||
load_level_history: Arc<RwLock<VecDeque<(SystemTime, LoadLevel)>>>,
|
||||
/// load level stats
|
||||
load_stats: Arc<RwLock<LoadLevelStats>>,
|
||||
/// business IO metrics (updated by external)
|
||||
business_metrics: Arc<BusinessIOMetrics>,
|
||||
/// cancel token
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
/// business IO metrics
|
||||
pub struct BusinessIOMetrics {
|
||||
/// business request latency (milliseconds)
|
||||
pub request_latency: AtomicU64,
|
||||
/// business request QPS
|
||||
pub request_qps: AtomicU64,
|
||||
/// business error rate (0-10000, 0.00%-100.00%)
|
||||
pub error_rate: AtomicU64,
|
||||
/// active connections
|
||||
pub active_connections: AtomicU64,
|
||||
/// last update time
|
||||
pub last_update: Arc<RwLock<SystemTime>>,
|
||||
}
|
||||
|
||||
impl Default for BusinessIOMetrics {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request_latency: AtomicU64::new(0),
|
||||
request_qps: AtomicU64::new(0),
|
||||
error_rate: AtomicU64::new(0),
|
||||
active_connections: AtomicU64::new(0),
|
||||
last_update: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdvancedIOMonitor {
|
||||
/// create new advanced IO monitor
|
||||
pub fn new(config: IOMonitorConfig) -> Self {
|
||||
Self {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
current_metrics: Arc::new(RwLock::new(IOMetrics::default())),
|
||||
history_metrics: Arc::new(RwLock::new(VecDeque::new())),
|
||||
current_load_level: Arc::new(RwLock::new(LoadLevel::Low)),
|
||||
load_level_history: Arc::new(RwLock::new(VecDeque::new())),
|
||||
load_stats: Arc::new(RwLock::new(LoadLevelStats::default())),
|
||||
business_metrics: Arc::new(BusinessIOMetrics::default()),
|
||||
cancel_token: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// start monitoring
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
info!("start advanced IO monitor");
|
||||
|
||||
let monitor = self.clone_for_background();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = monitor.monitoring_loop().await {
|
||||
error!("IO monitoring loop failed: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// stop monitoring
|
||||
pub async fn stop(&self) {
|
||||
info!("stop IO monitor");
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
|
||||
/// monitoring loop
|
||||
async fn monitoring_loop(&self) -> Result<()> {
|
||||
let mut interval = {
|
||||
let config = self.config.read().await;
|
||||
tokio::time::interval(config.monitor_interval)
|
||||
};
|
||||
|
||||
let mut last_load_level = LoadLevel::Low;
|
||||
let mut load_level_start_time = SystemTime::now();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = self.cancel_token.cancelled() => {
|
||||
info!("IO monitoring loop cancelled");
|
||||
break;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
// collect system metrics
|
||||
let metrics = self.collect_system_metrics().await;
|
||||
|
||||
// update current metrics
|
||||
*self.current_metrics.write().await = metrics.clone();
|
||||
|
||||
// update history metrics
|
||||
self.update_metrics_history(metrics.clone()).await;
|
||||
|
||||
// calculate load level
|
||||
let new_load_level = self.calculate_load_level(&metrics).await;
|
||||
|
||||
// check if load level changed
|
||||
if new_load_level != last_load_level {
|
||||
self.handle_load_level_change(last_load_level, new_load_level, load_level_start_time).await;
|
||||
last_load_level = new_load_level;
|
||||
load_level_start_time = SystemTime::now();
|
||||
}
|
||||
|
||||
// update current load level
|
||||
*self.current_load_level.write().await = new_load_level;
|
||||
|
||||
debug!("IO monitor updated: IOPS={}, queue depth={}, latency={}ms, load level={:?}",
|
||||
metrics.iops, metrics.queue_depth, metrics.avg_latency, new_load_level);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// collect system metrics
|
||||
async fn collect_system_metrics(&self) -> IOMetrics {
|
||||
let config = self.config.read().await;
|
||||
|
||||
if config.enable_system_monitoring {
|
||||
// actual system monitoring implementation
|
||||
self.collect_real_system_metrics().await
|
||||
} else {
|
||||
// simulated data
|
||||
self.generate_simulated_metrics().await
|
||||
}
|
||||
}
|
||||
|
||||
/// collect real system metrics (need to be implemented according to specific system)
|
||||
async fn collect_real_system_metrics(&self) -> IOMetrics {
|
||||
// TODO: implement actual system metrics collection
|
||||
// can use procfs, sysfs or other system API
|
||||
|
||||
let metrics = IOMetrics {
|
||||
timestamp: SystemTime::now(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// example: read /proc/diskstats
|
||||
if let Ok(diskstats) = tokio::fs::read_to_string("/proc/diskstats").await {
|
||||
// parse disk stats info
|
||||
// here need to implement specific parsing logic
|
||||
debug!("read disk stats info: {} bytes", diskstats.len());
|
||||
}
|
||||
|
||||
// example: read /proc/stat to get CPU info
|
||||
if let Ok(stat) = tokio::fs::read_to_string("/proc/stat").await {
|
||||
// parse CPU stats info
|
||||
debug!("read CPU stats info: {} bytes", stat.len());
|
||||
}
|
||||
|
||||
// example: read /proc/meminfo to get memory info
|
||||
if let Ok(meminfo) = tokio::fs::read_to_string("/proc/meminfo").await {
|
||||
// parse memory stats info
|
||||
debug!("read memory stats info: {} bytes", meminfo.len());
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
/// generate simulated metrics (for testing and development)
|
||||
async fn generate_simulated_metrics(&self) -> IOMetrics {
|
||||
use rand::Rng;
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// get business metrics impact
|
||||
let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed);
|
||||
let business_qps = self.business_metrics.request_qps.load(Ordering::Relaxed);
|
||||
|
||||
// generate simulated system metrics based on business load
|
||||
let base_iops = 100 + (business_qps / 10);
|
||||
let base_latency = 5 + (business_latency / 10);
|
||||
|
||||
IOMetrics {
|
||||
timestamp: SystemTime::now(),
|
||||
iops: base_iops + rng.random_range(0..50),
|
||||
read_iops: (base_iops * 6 / 10) + rng.random_range(0..20),
|
||||
write_iops: (base_iops * 4 / 10) + rng.random_range(0..20),
|
||||
queue_depth: rng.random_range(1..20),
|
||||
avg_latency: base_latency + rng.random_range(0..10),
|
||||
read_latency: base_latency + rng.random_range(0..5),
|
||||
write_latency: base_latency + rng.random_range(0..15),
|
||||
cpu_usage: rng.random_range(10..70),
|
||||
memory_usage: rng.random_range(30..80),
|
||||
disk_utilization: rng.random_range(20..90),
|
||||
network_io: rng.random_range(10..1000),
|
||||
}
|
||||
}
|
||||
|
||||
/// update metrics history
|
||||
async fn update_metrics_history(&self, metrics: IOMetrics) {
|
||||
let mut history = self.history_metrics.write().await;
|
||||
let config = self.config.read().await;
|
||||
|
||||
// add new metrics
|
||||
history.push_back(metrics);
|
||||
|
||||
// clean expired data
|
||||
let retention_cutoff = SystemTime::now() - config.history_retention;
|
||||
while let Some(front) = history.front() {
|
||||
if front.timestamp < retention_cutoff {
|
||||
history.pop_front();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// limit window size
|
||||
while history.len() > config.load_window_size {
|
||||
history.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
/// calculate load level
|
||||
async fn calculate_load_level(&self, metrics: &IOMetrics) -> LoadLevel {
|
||||
// multi-dimensional load evaluation algorithm
|
||||
let mut load_score = 0u32;
|
||||
|
||||
// IOPS load evaluation (weight: 25%)
|
||||
let iops_score = match metrics.iops {
|
||||
0..=200 => 0,
|
||||
201..=500 => 15,
|
||||
501..=1000 => 25,
|
||||
_ => 35,
|
||||
};
|
||||
load_score += iops_score;
|
||||
|
||||
// latency load evaluation (weight: 30%)
|
||||
let latency_score = match metrics.avg_latency {
|
||||
0..=10 => 0,
|
||||
11..=50 => 20,
|
||||
51..=100 => 30,
|
||||
_ => 40,
|
||||
};
|
||||
load_score += latency_score;
|
||||
|
||||
// queue depth evaluation (weight: 20%)
|
||||
let queue_score = match metrics.queue_depth {
|
||||
0..=5 => 0,
|
||||
6..=15 => 10,
|
||||
16..=30 => 20,
|
||||
_ => 25,
|
||||
};
|
||||
load_score += queue_score;
|
||||
|
||||
// CPU usage evaluation (weight: 15%)
|
||||
let cpu_score = match metrics.cpu_usage {
|
||||
0..=30 => 0,
|
||||
31..=60 => 8,
|
||||
61..=80 => 12,
|
||||
_ => 15,
|
||||
};
|
||||
load_score += cpu_score;
|
||||
|
||||
// disk usage evaluation (weight: 10%)
|
||||
let disk_score = match metrics.disk_utilization {
|
||||
0..=50 => 0,
|
||||
51..=75 => 5,
|
||||
76..=90 => 8,
|
||||
_ => 10,
|
||||
};
|
||||
load_score += disk_score;
|
||||
|
||||
// business metrics impact
|
||||
let business_latency = self.business_metrics.request_latency.load(Ordering::Relaxed);
|
||||
let business_error_rate = self.business_metrics.error_rate.load(Ordering::Relaxed);
|
||||
|
||||
if business_latency > 100 {
|
||||
load_score += 20; // business latency too high
|
||||
}
|
||||
if business_error_rate > 100 {
|
||||
// > 1%
|
||||
load_score += 15; // business error rate too high
|
||||
}
|
||||
|
||||
// history trend analysis
|
||||
let trend_score = self.calculate_trend_score().await;
|
||||
load_score += trend_score;
|
||||
|
||||
// determine load level based on total score
|
||||
match load_score {
|
||||
0..=30 => LoadLevel::Low,
|
||||
31..=60 => LoadLevel::Medium,
|
||||
61..=90 => LoadLevel::High,
|
||||
_ => LoadLevel::Critical,
|
||||
}
|
||||
}
|
||||
|
||||
/// calculate trend score
|
||||
async fn calculate_trend_score(&self) -> u32 {
|
||||
let history = self.history_metrics.read().await;
|
||||
|
||||
if history.len() < 5 {
|
||||
return 0; // data insufficient, cannot analyze trend
|
||||
}
|
||||
|
||||
// analyze trend of last 5 samples
|
||||
let recent: Vec<_> = history.iter().rev().take(5).collect();
|
||||
|
||||
// check IOPS rising trend
|
||||
let mut iops_trend = 0;
|
||||
for i in 1..recent.len() {
|
||||
if recent[i - 1].iops > recent[i].iops {
|
||||
iops_trend += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// check latency rising trend
|
||||
let mut latency_trend = 0;
|
||||
for i in 1..recent.len() {
|
||||
if recent[i - 1].avg_latency > recent[i].avg_latency {
|
||||
latency_trend += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// if IOPS and latency are both rising, increase load score
|
||||
if iops_trend >= 3 && latency_trend >= 3 {
|
||||
15 // obvious rising trend
|
||||
} else if iops_trend >= 2 || latency_trend >= 2 {
|
||||
5 // slight rising trend
|
||||
} else {
|
||||
0 // no obvious trend
|
||||
}
|
||||
}
|
||||
|
||||
/// handle load level change
|
||||
async fn handle_load_level_change(&self, old_level: LoadLevel, new_level: LoadLevel, start_time: SystemTime) {
|
||||
let duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
|
||||
|
||||
// update stats
|
||||
{
|
||||
let mut stats = self.load_stats.write().await;
|
||||
match old_level {
|
||||
LoadLevel::Low => stats.low_load_duration += duration.as_secs(),
|
||||
LoadLevel::Medium => stats.medium_load_duration += duration.as_secs(),
|
||||
LoadLevel::High => stats.high_load_duration += duration.as_secs(),
|
||||
LoadLevel::Critical => stats.critical_load_duration += duration.as_secs(),
|
||||
}
|
||||
stats.load_transitions += 1;
|
||||
}
|
||||
|
||||
// update history
|
||||
{
|
||||
let mut history = self.load_level_history.write().await;
|
||||
history.push_back((SystemTime::now(), new_level));
|
||||
|
||||
// keep history record in reasonable range
|
||||
while history.len() > 100 {
|
||||
history.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
info!("load level changed: {:?} -> {:?}, duration: {:?}", old_level, new_level, duration);
|
||||
|
||||
// if enter critical load state, record warning
|
||||
if new_level == LoadLevel::Critical {
|
||||
warn!("system entered critical load state, Scanner will pause running");
|
||||
}
|
||||
}
|
||||
|
||||
/// get current load level
|
||||
pub async fn get_business_load_level(&self) -> LoadLevel {
|
||||
*self.current_load_level.read().await
|
||||
}
|
||||
|
||||
/// get current metrics
|
||||
pub async fn get_current_metrics(&self) -> IOMetrics {
|
||||
self.current_metrics.read().await.clone()
|
||||
}
|
||||
|
||||
/// get history metrics
|
||||
pub async fn get_history_metrics(&self) -> Vec<IOMetrics> {
|
||||
self.history_metrics.read().await.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// get load stats
|
||||
pub async fn get_load_stats(&self) -> LoadLevelStats {
|
||||
self.load_stats.read().await.clone()
|
||||
}
|
||||
|
||||
/// update business IO metrics
|
||||
pub async fn update_business_metrics(&self, latency: u64, qps: u64, error_rate: u64, connections: u64) {
|
||||
self.business_metrics.request_latency.store(latency, Ordering::Relaxed);
|
||||
self.business_metrics.request_qps.store(qps, Ordering::Relaxed);
|
||||
self.business_metrics.error_rate.store(error_rate, Ordering::Relaxed);
|
||||
self.business_metrics.active_connections.store(connections, Ordering::Relaxed);
|
||||
|
||||
*self.business_metrics.last_update.write().await = SystemTime::now();
|
||||
|
||||
debug!(
|
||||
"update business metrics: latency={}ms, QPS={}, error rate={}‰, connections={}",
|
||||
latency, qps, error_rate, connections
|
||||
);
|
||||
}
|
||||
|
||||
/// clone for background task
|
||||
fn clone_for_background(&self) -> Self {
|
||||
Self {
|
||||
config: self.config.clone(),
|
||||
current_metrics: self.current_metrics.clone(),
|
||||
history_metrics: self.history_metrics.clone(),
|
||||
current_load_level: self.current_load_level.clone(),
|
||||
load_level_history: self.load_level_history.clone(),
|
||||
load_stats: self.load_stats.clone(),
|
||||
business_metrics: self.business_metrics.clone(),
|
||||
cancel_token: self.cancel_token.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// reset stats
|
||||
pub async fn reset_stats(&self) {
|
||||
*self.load_stats.write().await = LoadLevelStats::default();
|
||||
self.load_level_history.write().await.clear();
|
||||
self.history_metrics.write().await.clear();
|
||||
info!("IO monitor stats reset");
|
||||
}
|
||||
|
||||
/// get load level history
|
||||
pub async fn get_load_level_history(&self) -> Vec<(SystemTime, LoadLevel)> {
|
||||
self.load_level_history.read().await.iter().cloned().collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU8, AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::node_scanner::LoadLevel;
|
||||
|
||||
/// IO throttler config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IOThrottlerConfig {
|
||||
/// max IOPS limit
|
||||
pub max_iops: u64,
|
||||
/// business priority baseline (percentage)
|
||||
pub base_business_priority: u8,
|
||||
/// scanner minimum delay (milliseconds)
|
||||
pub min_scan_delay: u64,
|
||||
/// scanner maximum delay (milliseconds)
|
||||
pub max_scan_delay: u64,
|
||||
/// whether enable dynamic adjustment
|
||||
pub enable_dynamic_adjustment: bool,
|
||||
/// adjustment response time (seconds)
|
||||
pub adjustment_response_time: u64,
|
||||
}
|
||||
|
||||
impl Default for IOThrottlerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_iops: 1000, // default max 1000 IOPS
|
||||
base_business_priority: 95, // business priority 95%
|
||||
min_scan_delay: 5000, // minimum 5s delay
|
||||
max_scan_delay: 60000, // maximum 60s delay
|
||||
enable_dynamic_adjustment: true,
|
||||
adjustment_response_time: 5, // 5 seconds response time
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// resource allocation strategy
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ResourceAllocationStrategy {
|
||||
/// business priority strategy
|
||||
BusinessFirst,
|
||||
/// balanced strategy
|
||||
Balanced,
|
||||
/// maintenance priority strategy (only used in special cases)
|
||||
MaintenanceFirst,
|
||||
}
|
||||
|
||||
/// throttle decision
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThrottleDecision {
|
||||
/// whether should pause scanning
|
||||
pub should_pause: bool,
|
||||
/// suggested scanning delay
|
||||
pub suggested_delay: Duration,
|
||||
/// resource allocation suggestion
|
||||
pub resource_allocation: ResourceAllocation,
|
||||
/// decision reason
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// resource allocation
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResourceAllocation {
|
||||
/// business IO allocation percentage (0-100)
|
||||
pub business_percentage: u8,
|
||||
/// scanner IO allocation percentage (0-100)
|
||||
pub scanner_percentage: u8,
|
||||
/// allocation strategy
|
||||
pub strategy: ResourceAllocationStrategy,
|
||||
}
|
||||
|
||||
/// enhanced IO throttler
|
||||
///
|
||||
/// dynamically adjust the resource usage of the scanner based on real-time system load and business demand,
|
||||
/// ensure business IO gets priority protection.
|
||||
pub struct AdvancedIOThrottler {
|
||||
/// config
|
||||
config: Arc<RwLock<IOThrottlerConfig>>,
|
||||
/// current IOPS usage (reserved field)
|
||||
#[allow(dead_code)]
|
||||
current_iops: Arc<AtomicU64>,
|
||||
/// business priority weight (0-100)
|
||||
business_priority: Arc<AtomicU8>,
|
||||
/// scanning operation delay (milliseconds)
|
||||
scan_delay: Arc<AtomicU64>,
|
||||
/// resource allocation strategy
|
||||
allocation_strategy: Arc<RwLock<ResourceAllocationStrategy>>,
|
||||
/// throttle history record
|
||||
throttle_history: Arc<RwLock<Vec<ThrottleRecord>>>,
|
||||
/// last adjustment time (reserved field)
|
||||
#[allow(dead_code)]
|
||||
last_adjustment: Arc<RwLock<SystemTime>>,
|
||||
}
|
||||
|
||||
/// throttle record
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThrottleRecord {
|
||||
/// timestamp
|
||||
pub timestamp: SystemTime,
|
||||
/// load level
|
||||
pub load_level: LoadLevel,
|
||||
/// decision
|
||||
pub decision: ThrottleDecision,
|
||||
/// system metrics snapshot
|
||||
pub metrics_snapshot: MetricsSnapshot,
|
||||
}
|
||||
|
||||
/// metrics snapshot
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetricsSnapshot {
|
||||
/// IOPS
|
||||
pub iops: u64,
|
||||
/// latency
|
||||
pub latency: u64,
|
||||
/// CPU usage
|
||||
pub cpu_usage: u8,
|
||||
/// memory usage
|
||||
pub memory_usage: u8,
|
||||
}
|
||||
|
||||
impl AdvancedIOThrottler {
|
||||
/// create new advanced IO throttler
|
||||
pub fn new(config: IOThrottlerConfig) -> Self {
|
||||
Self {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
current_iops: Arc::new(AtomicU64::new(0)),
|
||||
business_priority: Arc::new(AtomicU8::new(95)),
|
||||
scan_delay: Arc::new(AtomicU64::new(5000)),
|
||||
allocation_strategy: Arc::new(RwLock::new(ResourceAllocationStrategy::BusinessFirst)),
|
||||
throttle_history: Arc::new(RwLock::new(Vec::new())),
|
||||
last_adjustment: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
|
||||
}
|
||||
}
|
||||
|
||||
/// adjust scanning delay based on load level
|
||||
pub async fn adjust_for_load_level(&self, load_level: LoadLevel) -> Duration {
|
||||
let config = self.config.read().await;
|
||||
|
||||
let delay_ms = match load_level {
|
||||
LoadLevel::Low => {
|
||||
// low load: use minimum delay
|
||||
self.scan_delay.store(config.min_scan_delay, Ordering::Relaxed);
|
||||
self.business_priority
|
||||
.store(config.base_business_priority.saturating_sub(5), Ordering::Relaxed);
|
||||
config.min_scan_delay
|
||||
}
|
||||
LoadLevel::Medium => {
|
||||
// medium load: increase delay moderately
|
||||
let delay = config.min_scan_delay * 5; // 500ms
|
||||
self.scan_delay.store(delay, Ordering::Relaxed);
|
||||
self.business_priority.store(config.base_business_priority, Ordering::Relaxed);
|
||||
delay
|
||||
}
|
||||
LoadLevel::High => {
|
||||
// high load: increase delay significantly
|
||||
let delay = config.min_scan_delay * 10; // 50s
|
||||
self.scan_delay.store(delay, Ordering::Relaxed);
|
||||
self.business_priority
|
||||
.store(config.base_business_priority.saturating_add(3), Ordering::Relaxed);
|
||||
delay
|
||||
}
|
||||
LoadLevel::Critical => {
|
||||
// critical load: maximum delay or pause
|
||||
let delay = config.max_scan_delay; // 60s
|
||||
self.scan_delay.store(delay, Ordering::Relaxed);
|
||||
self.business_priority.store(99, Ordering::Relaxed);
|
||||
delay
|
||||
}
|
||||
};
|
||||
|
||||
let duration = Duration::from_millis(delay_ms);
|
||||
|
||||
debug!("Adjust scanning delay based on load level {:?}: {:?}", load_level, duration);
|
||||
|
||||
duration
|
||||
}
|
||||
|
||||
/// create throttle decision
|
||||
pub async fn make_throttle_decision(&self, load_level: LoadLevel, metrics: Option<MetricsSnapshot>) -> ThrottleDecision {
|
||||
let _config = self.config.read().await;
|
||||
|
||||
let should_pause = matches!(load_level, LoadLevel::Critical);
|
||||
|
||||
let suggested_delay = self.adjust_for_load_level(load_level).await;
|
||||
|
||||
let resource_allocation = self.calculate_resource_allocation(load_level).await;
|
||||
|
||||
let reason = match load_level {
|
||||
LoadLevel::Low => "system load is low, scanner can run normally".to_string(),
|
||||
LoadLevel::Medium => "system load is moderate, scanner is running at reduced speed".to_string(),
|
||||
LoadLevel::High => "system load is high, scanner is running at significantly reduced speed".to_string(),
|
||||
LoadLevel::Critical => "system load is too high, scanner is paused".to_string(),
|
||||
};
|
||||
|
||||
let decision = ThrottleDecision {
|
||||
should_pause,
|
||||
suggested_delay,
|
||||
resource_allocation,
|
||||
reason,
|
||||
};
|
||||
|
||||
// record decision history
|
||||
if let Some(snapshot) = metrics {
|
||||
self.record_throttle_decision(load_level, decision.clone(), snapshot).await;
|
||||
}
|
||||
|
||||
decision
|
||||
}
|
||||
|
||||
/// calculate resource allocation
|
||||
async fn calculate_resource_allocation(&self, load_level: LoadLevel) -> ResourceAllocation {
|
||||
let strategy = *self.allocation_strategy.read().await;
|
||||
|
||||
let (business_pct, scanner_pct) = match (strategy, load_level) {
|
||||
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Low) => (90, 10),
|
||||
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Medium) => (95, 5),
|
||||
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::High) => (98, 2),
|
||||
(ResourceAllocationStrategy::BusinessFirst, LoadLevel::Critical) => (99, 1),
|
||||
|
||||
(ResourceAllocationStrategy::Balanced, LoadLevel::Low) => (80, 20),
|
||||
(ResourceAllocationStrategy::Balanced, LoadLevel::Medium) => (85, 15),
|
||||
(ResourceAllocationStrategy::Balanced, LoadLevel::High) => (90, 10),
|
||||
(ResourceAllocationStrategy::Balanced, LoadLevel::Critical) => (95, 5),
|
||||
|
||||
(ResourceAllocationStrategy::MaintenanceFirst, _) => (70, 30), // special maintenance mode
|
||||
};
|
||||
|
||||
ResourceAllocation {
|
||||
business_percentage: business_pct,
|
||||
scanner_percentage: scanner_pct,
|
||||
strategy,
|
||||
}
|
||||
}
|
||||
|
||||
/// check whether should pause scanning
|
||||
pub async fn should_pause_scanning(&self, load_level: LoadLevel) -> bool {
|
||||
match load_level {
|
||||
LoadLevel::Critical => {
|
||||
warn!("System load reached critical level, pausing scanner");
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// record throttle decision
|
||||
async fn record_throttle_decision(&self, load_level: LoadLevel, decision: ThrottleDecision, metrics: MetricsSnapshot) {
|
||||
let record = ThrottleRecord {
|
||||
timestamp: SystemTime::now(),
|
||||
load_level,
|
||||
decision,
|
||||
metrics_snapshot: metrics,
|
||||
};
|
||||
|
||||
let mut history = self.throttle_history.write().await;
|
||||
history.push(record);
|
||||
|
||||
// keep history record in reasonable range (last 1000 records)
|
||||
while history.len() > 1000 {
|
||||
history.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// set resource allocation strategy
|
||||
pub async fn set_allocation_strategy(&self, strategy: ResourceAllocationStrategy) {
|
||||
*self.allocation_strategy.write().await = strategy;
|
||||
info!("Set resource allocation strategy: {:?}", strategy);
|
||||
}
|
||||
|
||||
/// get current resource allocation
|
||||
pub async fn get_current_allocation(&self) -> ResourceAllocation {
|
||||
let current_load = LoadLevel::Low; // need to get from external
|
||||
self.calculate_resource_allocation(current_load).await
|
||||
}
|
||||
|
||||
/// get throttle history
|
||||
pub async fn get_throttle_history(&self) -> Vec<ThrottleRecord> {
|
||||
self.throttle_history.read().await.clone()
|
||||
}
|
||||
|
||||
/// get throttle stats
|
||||
pub async fn get_throttle_stats(&self) -> ThrottleStats {
|
||||
let history = self.throttle_history.read().await;
|
||||
|
||||
let total_decisions = history.len();
|
||||
let pause_decisions = history.iter().filter(|r| r.decision.should_pause).count();
|
||||
|
||||
let mut delay_sum = Duration::ZERO;
|
||||
for record in history.iter() {
|
||||
delay_sum += record.decision.suggested_delay;
|
||||
}
|
||||
|
||||
let avg_delay = if total_decisions > 0 {
|
||||
delay_sum / total_decisions as u32
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
|
||||
// count by load level
|
||||
let low_count = history.iter().filter(|r| r.load_level == LoadLevel::Low).count();
|
||||
let medium_count = history.iter().filter(|r| r.load_level == LoadLevel::Medium).count();
|
||||
let high_count = history.iter().filter(|r| r.load_level == LoadLevel::High).count();
|
||||
let critical_count = history.iter().filter(|r| r.load_level == LoadLevel::Critical).count();
|
||||
|
||||
ThrottleStats {
|
||||
total_decisions,
|
||||
pause_decisions,
|
||||
average_delay: avg_delay,
|
||||
load_level_distribution: LoadLevelDistribution {
|
||||
low_count,
|
||||
medium_count,
|
||||
high_count,
|
||||
critical_count,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// reset throttle history
|
||||
pub async fn reset_history(&self) {
|
||||
self.throttle_history.write().await.clear();
|
||||
info!("Reset throttle history");
|
||||
}
|
||||
|
||||
/// update config
|
||||
pub async fn update_config(&self, new_config: IOThrottlerConfig) {
|
||||
*self.config.write().await = new_config;
|
||||
info!("Updated IO throttler configuration");
|
||||
}
|
||||
|
||||
/// get current scanning delay
|
||||
pub fn get_current_scan_delay(&self) -> Duration {
|
||||
let delay_ms = self.scan_delay.load(Ordering::Relaxed);
|
||||
Duration::from_millis(delay_ms)
|
||||
}
|
||||
|
||||
/// get current business priority
|
||||
pub fn get_current_business_priority(&self) -> u8 {
|
||||
self.business_priority.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// simulate business load pressure test
|
||||
pub async fn simulate_business_pressure(&self, duration: Duration) -> SimulationResult {
|
||||
info!("Start simulating business load pressure test, duration: {:?}", duration);
|
||||
|
||||
let start_time = SystemTime::now();
|
||||
let mut simulation_records = Vec::new();
|
||||
|
||||
// simulate different load level changes
|
||||
let load_levels = [
|
||||
LoadLevel::Low,
|
||||
LoadLevel::Medium,
|
||||
LoadLevel::High,
|
||||
LoadLevel::Critical,
|
||||
LoadLevel::High,
|
||||
LoadLevel::Medium,
|
||||
LoadLevel::Low,
|
||||
];
|
||||
|
||||
let step_duration = duration / load_levels.len() as u32;
|
||||
|
||||
for (i, &load_level) in load_levels.iter().enumerate() {
|
||||
let _step_start = SystemTime::now();
|
||||
|
||||
// simulate metrics for this load level
|
||||
let metrics = MetricsSnapshot {
|
||||
iops: match load_level {
|
||||
LoadLevel::Low => 200,
|
||||
LoadLevel::Medium => 500,
|
||||
LoadLevel::High => 800,
|
||||
LoadLevel::Critical => 1200,
|
||||
},
|
||||
latency: match load_level {
|
||||
LoadLevel::Low => 10,
|
||||
LoadLevel::Medium => 25,
|
||||
LoadLevel::High => 60,
|
||||
LoadLevel::Critical => 150,
|
||||
},
|
||||
cpu_usage: match load_level {
|
||||
LoadLevel::Low => 30,
|
||||
LoadLevel::Medium => 50,
|
||||
LoadLevel::High => 75,
|
||||
LoadLevel::Critical => 95,
|
||||
},
|
||||
memory_usage: match load_level {
|
||||
LoadLevel::Low => 40,
|
||||
LoadLevel::Medium => 60,
|
||||
LoadLevel::High => 80,
|
||||
LoadLevel::Critical => 90,
|
||||
},
|
||||
};
|
||||
|
||||
let decision = self.make_throttle_decision(load_level, Some(metrics.clone())).await;
|
||||
|
||||
simulation_records.push(SimulationRecord {
|
||||
step: i + 1,
|
||||
load_level,
|
||||
metrics,
|
||||
decision: decision.clone(),
|
||||
step_duration,
|
||||
});
|
||||
|
||||
info!(
|
||||
"simulate step {}: load={:?}, delay={:?}, pause={}",
|
||||
i + 1,
|
||||
load_level,
|
||||
decision.suggested_delay,
|
||||
decision.should_pause
|
||||
);
|
||||
|
||||
// wait for step duration
|
||||
tokio::time::sleep(step_duration).await;
|
||||
}
|
||||
|
||||
let total_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
|
||||
|
||||
SimulationResult {
|
||||
total_duration,
|
||||
simulation_records,
|
||||
final_stats: self.get_throttle_stats().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// throttle stats
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThrottleStats {
|
||||
/// total decisions
|
||||
pub total_decisions: usize,
|
||||
/// pause decisions
|
||||
pub pause_decisions: usize,
|
||||
/// average delay
|
||||
pub average_delay: Duration,
|
||||
/// load level distribution
|
||||
pub load_level_distribution: LoadLevelDistribution,
|
||||
}
|
||||
|
||||
/// load level distribution
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LoadLevelDistribution {
|
||||
/// low load count
|
||||
pub low_count: usize,
|
||||
/// medium load count
|
||||
pub medium_count: usize,
|
||||
/// high load count
|
||||
pub high_count: usize,
|
||||
/// critical load count
|
||||
pub critical_count: usize,
|
||||
}
|
||||
|
||||
/// simulation result
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SimulationResult {
|
||||
/// total duration
|
||||
pub total_duration: Duration,
|
||||
/// simulation records
|
||||
pub simulation_records: Vec<SimulationRecord>,
|
||||
/// final stats
|
||||
pub final_stats: ThrottleStats,
|
||||
}
|
||||
|
||||
/// simulation record
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SimulationRecord {
|
||||
/// step number
|
||||
pub step: usize,
|
||||
/// load level
|
||||
pub load_level: LoadLevel,
|
||||
/// metrics snapshot
|
||||
pub metrics: MetricsSnapshot,
|
||||
/// throttle decision
|
||||
pub decision: ThrottleDecision,
|
||||
/// step duration
|
||||
pub step_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for AdvancedIOThrottler {
|
||||
fn default() -> Self {
|
||||
Self::new(IOThrottlerConfig::default())
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::error::Result;
|
||||
use rustfs_common::data_usage::SizeSummary;
|
||||
@@ -33,6 +32,7 @@ use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
|
||||
use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use s3s::dto::BucketLifecycleConfiguration as LifecycleConfig;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::info;
|
||||
|
||||
static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100);
|
||||
@@ -187,9 +187,12 @@ impl ScannerItem {
|
||||
async fn apply_lifecycle(&mut self, oi: &ObjectInfo) -> (IlmAction, i64) {
|
||||
let size = oi.size;
|
||||
if self.lifecycle.is_none() {
|
||||
info!("apply_lifecycle: No lifecycle config for object: {}", oi.name);
|
||||
return (IlmAction::NoneAction, size);
|
||||
}
|
||||
|
||||
info!("apply_lifecycle: Lifecycle config exists for object: {}", oi.name);
|
||||
|
||||
let (olcfg, rcfg) = if self.bucket != ".minio.sys" {
|
||||
(
|
||||
get_object_lock_config(&self.bucket).await.ok(),
|
||||
@@ -199,36 +202,61 @@ impl ScannerItem {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
info!("apply_lifecycle: Evaluating lifecycle for object: {}", oi.name);
|
||||
|
||||
let lifecycle = match self.lifecycle.as_ref() {
|
||||
Some(lc) => lc,
|
||||
None => {
|
||||
info!("No lifecycle configuration found for object: {}", oi.name);
|
||||
return (IlmAction::NoneAction, 0);
|
||||
}
|
||||
};
|
||||
|
||||
let lc_evt = eval_action_from_lifecycle(
|
||||
self.lifecycle.as_ref().unwrap(),
|
||||
lifecycle,
|
||||
olcfg
|
||||
.as_ref()
|
||||
.and_then(|(c, _)| c.rule.as_ref().and_then(|r| r.default_retention.clone())),
|
||||
rcfg.clone(),
|
||||
oi,
|
||||
oi, // Pass oi directly
|
||||
)
|
||||
.await;
|
||||
|
||||
info!("lifecycle: {} Initial scan: {}", oi.name, lc_evt.action);
|
||||
info!("lifecycle: {} Initial scan: {} (action: {:?})", oi.name, lc_evt.action, lc_evt.action);
|
||||
|
||||
let mut new_size = size;
|
||||
match lc_evt.action {
|
||||
IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
info!("apply_lifecycle: Object {} marked for version deletion, new_size=0", oi.name);
|
||||
new_size = 0;
|
||||
}
|
||||
IlmAction::DeleteAction => {
|
||||
info!("apply_lifecycle: Object {} marked for deletion", oi.name);
|
||||
if let Some(vcfg) = &self.versioning {
|
||||
if !vcfg.is_enabled() {
|
||||
info!("apply_lifecycle: Versioning disabled, setting new_size=0");
|
||||
new_size = 0;
|
||||
}
|
||||
} else {
|
||||
info!("apply_lifecycle: No versioning config, setting new_size=0");
|
||||
new_size = 0;
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
IlmAction::NoneAction => {
|
||||
info!("apply_lifecycle: No action for object {}", oi.name);
|
||||
}
|
||||
_ => {
|
||||
info!("apply_lifecycle: Other action {:?} for object {}", lc_evt.action, oi.name);
|
||||
}
|
||||
}
|
||||
|
||||
if lc_evt.action != IlmAction::NoneAction {
|
||||
info!("apply_lifecycle: Applying lifecycle action {:?} for object {}", lc_evt.action, oi.name);
|
||||
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
|
||||
} else {
|
||||
info!("apply_lifecycle: Skipping lifecycle action for object {} as no action is needed", oi.name);
|
||||
}
|
||||
|
||||
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
|
||||
(lc_evt.action, new_size)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
|
||||
use super::node_scanner::{BucketStats, DiskStats, LocalScanStats};
|
||||
use crate::{Error, error::Result};
|
||||
|
||||
/// local stats manager
|
||||
pub struct LocalStatsManager {
|
||||
/// node id
|
||||
node_id: String,
|
||||
/// stats file path
|
||||
stats_file: PathBuf,
|
||||
/// backup file path
|
||||
backup_file: PathBuf,
|
||||
/// temp file path
|
||||
temp_file: PathBuf,
|
||||
/// local stats data
|
||||
stats: Arc<RwLock<LocalScanStats>>,
|
||||
/// save interval
|
||||
save_interval: Duration,
|
||||
/// last save time
|
||||
last_save: Arc<RwLock<SystemTime>>,
|
||||
/// stats counters
|
||||
counters: Arc<StatsCounters>,
|
||||
}
|
||||
|
||||
/// stats counters
|
||||
pub struct StatsCounters {
|
||||
/// total scanned objects
|
||||
pub total_objects_scanned: AtomicU64,
|
||||
/// total healthy objects
|
||||
pub total_healthy_objects: AtomicU64,
|
||||
/// total corrupted objects
|
||||
pub total_corrupted_objects: AtomicU64,
|
||||
/// total scanned bytes
|
||||
pub total_bytes_scanned: AtomicU64,
|
||||
/// total scan errors
|
||||
pub total_scan_errors: AtomicU64,
|
||||
/// total heal triggered
|
||||
pub total_heal_triggered: AtomicU64,
|
||||
}
|
||||
|
||||
impl Default for StatsCounters {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
total_objects_scanned: AtomicU64::new(0),
|
||||
total_healthy_objects: AtomicU64::new(0),
|
||||
total_corrupted_objects: AtomicU64::new(0),
|
||||
total_bytes_scanned: AtomicU64::new(0),
|
||||
total_scan_errors: AtomicU64::new(0),
|
||||
total_heal_triggered: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// scan result entry
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ScanResultEntry {
|
||||
/// object path
|
||||
pub object_path: String,
|
||||
/// bucket name
|
||||
pub bucket_name: String,
|
||||
/// object size
|
||||
pub object_size: u64,
|
||||
/// is healthy
|
||||
pub is_healthy: bool,
|
||||
/// error message (if any)
|
||||
pub error_message: Option<String>,
|
||||
/// scan time
|
||||
pub scan_time: SystemTime,
|
||||
/// disk id
|
||||
pub disk_id: String,
|
||||
}
|
||||
|
||||
/// batch scan result
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BatchScanResult {
|
||||
/// disk id
|
||||
pub disk_id: String,
|
||||
/// scan result entries
|
||||
pub entries: Vec<ScanResultEntry>,
|
||||
/// scan start time
|
||||
pub scan_start: SystemTime,
|
||||
/// scan end time
|
||||
pub scan_end: SystemTime,
|
||||
/// scan duration
|
||||
pub scan_duration: Duration,
|
||||
}
|
||||
|
||||
impl LocalStatsManager {
|
||||
/// create new local stats manager
|
||||
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
||||
// ensure data directory exists
|
||||
if !data_dir.exists() {
|
||||
if let Err(e) = std::fs::create_dir_all(data_dir) {
|
||||
error!("create stats data directory failed {:?}: {}", data_dir, e);
|
||||
}
|
||||
}
|
||||
|
||||
let stats_file = data_dir.join(format!("scanner_stats_{}.json", node_id));
|
||||
let backup_file = data_dir.join(format!("scanner_stats_{}.backup", node_id));
|
||||
let temp_file = data_dir.join(format!("scanner_stats_{}.tmp", node_id));
|
||||
|
||||
Self {
|
||||
node_id: node_id.to_string(),
|
||||
stats_file,
|
||||
backup_file,
|
||||
temp_file,
|
||||
stats: Arc::new(RwLock::new(LocalScanStats::default())),
|
||||
save_interval: Duration::from_secs(60), // 60 seconds save once
|
||||
last_save: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
|
||||
counters: Arc::new(StatsCounters::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// load local stats data
|
||||
pub async fn load_stats(&self) -> Result<()> {
|
||||
if !self.stats_file.exists() {
|
||||
info!("stats data file not exists, will create new stats data");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match self.load_stats_from_file(&self.stats_file).await {
|
||||
Ok(stats) => {
|
||||
*self.stats.write().await = stats;
|
||||
info!("success load local stats data");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("load main stats file failed: {}, try backup file", e);
|
||||
|
||||
match self.load_stats_from_file(&self.backup_file).await {
|
||||
Ok(stats) => {
|
||||
*self.stats.write().await = stats;
|
||||
warn!("restore stats data from backup file");
|
||||
Ok(())
|
||||
}
|
||||
Err(backup_e) => {
|
||||
warn!("backup file also cannot load: {}, will use default stats data", backup_e);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// load stats data from file
|
||||
async fn load_stats_from_file(&self, file_path: &Path) -> Result<LocalScanStats> {
|
||||
let content = tokio::fs::read_to_string(file_path)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("read stats file failed: {}", e)))?;
|
||||
|
||||
let stats: LocalScanStats =
|
||||
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize stats data failed: {}", e)))?;
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// save stats data to disk
|
||||
pub async fn save_stats(&self) -> Result<()> {
|
||||
let now = SystemTime::now();
|
||||
let last_save = *self.last_save.read().await;
|
||||
|
||||
// frequency control
|
||||
if now.duration_since(last_save).unwrap_or(Duration::ZERO) < self.save_interval {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let stats = self.stats.read().await.clone();
|
||||
|
||||
// serialize
|
||||
let json_data = serde_json::to_string_pretty(&stats)
|
||||
.map_err(|e| Error::Serialization(format!("serialize stats data failed: {}", e)))?;
|
||||
|
||||
// atomic write
|
||||
tokio::fs::write(&self.temp_file, json_data)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("write temp stats file failed: {}", e)))?;
|
||||
|
||||
// backup existing file
|
||||
if self.stats_file.exists() {
|
||||
tokio::fs::copy(&self.stats_file, &self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("backup stats file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// atomic replace
|
||||
tokio::fs::rename(&self.temp_file, &self.stats_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("replace stats file failed: {}", e)))?;
|
||||
|
||||
*self.last_save.write().await = now;
|
||||
|
||||
debug!("save local stats data to {:?}", self.stats_file);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// force save stats data
|
||||
pub async fn force_save_stats(&self) -> Result<()> {
|
||||
*self.last_save.write().await = SystemTime::UNIX_EPOCH;
|
||||
self.save_stats().await
|
||||
}
|
||||
|
||||
/// update disk scan result
|
||||
pub async fn update_disk_scan_result(&self, result: &BatchScanResult) -> Result<()> {
|
||||
let mut stats = self.stats.write().await;
|
||||
|
||||
// update disk stats
|
||||
let disk_stat = stats.disks_stats.entry(result.disk_id.clone()).or_insert_with(|| DiskStats {
|
||||
disk_id: result.disk_id.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let healthy_count = result.entries.iter().filter(|e| e.is_healthy).count() as u64;
|
||||
let error_count = result.entries.iter().filter(|e| !e.is_healthy).count() as u64;
|
||||
|
||||
disk_stat.objects_scanned += result.entries.len() as u64;
|
||||
disk_stat.errors_count += error_count;
|
||||
disk_stat.last_scan_time = result.scan_end;
|
||||
disk_stat.scan_duration = result.scan_duration;
|
||||
disk_stat.scan_completed = true;
|
||||
|
||||
// update overall stats
|
||||
stats.objects_scanned += result.entries.len() as u64;
|
||||
stats.healthy_objects += healthy_count;
|
||||
stats.corrupted_objects += error_count;
|
||||
stats.last_update = SystemTime::now();
|
||||
|
||||
// update bucket stats
|
||||
for entry in &result.entries {
|
||||
let _bucket_stat = stats
|
||||
.buckets_stats
|
||||
.entry(entry.bucket_name.clone())
|
||||
.or_insert_with(BucketStats::default);
|
||||
|
||||
// TODO: update BucketStats
|
||||
}
|
||||
|
||||
// update atomic counters
|
||||
self.counters
|
||||
.total_objects_scanned
|
||||
.fetch_add(result.entries.len() as u64, Ordering::Relaxed);
|
||||
self.counters
|
||||
.total_healthy_objects
|
||||
.fetch_add(healthy_count, Ordering::Relaxed);
|
||||
self.counters
|
||||
.total_corrupted_objects
|
||||
.fetch_add(error_count, Ordering::Relaxed);
|
||||
|
||||
let total_bytes: u64 = result.entries.iter().map(|e| e.object_size).sum();
|
||||
self.counters.total_bytes_scanned.fetch_add(total_bytes, Ordering::Relaxed);
|
||||
|
||||
if error_count > 0 {
|
||||
self.counters.total_scan_errors.fetch_add(error_count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
drop(stats);
|
||||
|
||||
debug!(
|
||||
"update disk {} scan result: objects {}, healthy {}, error {}",
|
||||
result.disk_id,
|
||||
result.entries.len(),
|
||||
healthy_count,
|
||||
error_count
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// record single object scan result
|
||||
pub async fn record_object_scan(&self, entry: ScanResultEntry) -> Result<()> {
|
||||
let result = BatchScanResult {
|
||||
disk_id: entry.disk_id.clone(),
|
||||
entries: vec![entry],
|
||||
scan_start: SystemTime::now(),
|
||||
scan_end: SystemTime::now(),
|
||||
scan_duration: Duration::from_millis(0),
|
||||
};
|
||||
|
||||
self.update_disk_scan_result(&result).await
|
||||
}
|
||||
|
||||
/// get local stats data copy
|
||||
pub async fn get_stats(&self) -> LocalScanStats {
|
||||
self.stats.read().await.clone()
|
||||
}
|
||||
|
||||
/// get real-time counters
|
||||
pub fn get_counters(&self) -> Arc<StatsCounters> {
|
||||
self.counters.clone()
|
||||
}
|
||||
|
||||
/// reset stats data
|
||||
pub async fn reset_stats(&self) -> Result<()> {
|
||||
{
|
||||
let mut stats = self.stats.write().await;
|
||||
*stats = LocalScanStats::default();
|
||||
}
|
||||
|
||||
// reset counters
|
||||
self.counters.total_objects_scanned.store(0, Ordering::Relaxed);
|
||||
self.counters.total_healthy_objects.store(0, Ordering::Relaxed);
|
||||
self.counters.total_corrupted_objects.store(0, Ordering::Relaxed);
|
||||
self.counters.total_bytes_scanned.store(0, Ordering::Relaxed);
|
||||
self.counters.total_scan_errors.store(0, Ordering::Relaxed);
|
||||
self.counters.total_heal_triggered.store(0, Ordering::Relaxed);
|
||||
|
||||
info!("reset local stats data");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// get stats summary
|
||||
pub async fn get_stats_summary(&self) -> StatsSummary {
|
||||
let stats = self.stats.read().await;
|
||||
|
||||
StatsSummary {
|
||||
node_id: self.node_id.clone(),
|
||||
total_objects_scanned: self.counters.total_objects_scanned.load(Ordering::Relaxed),
|
||||
total_healthy_objects: self.counters.total_healthy_objects.load(Ordering::Relaxed),
|
||||
total_corrupted_objects: self.counters.total_corrupted_objects.load(Ordering::Relaxed),
|
||||
total_bytes_scanned: self.counters.total_bytes_scanned.load(Ordering::Relaxed),
|
||||
total_scan_errors: self.counters.total_scan_errors.load(Ordering::Relaxed),
|
||||
total_heal_triggered: self.counters.total_heal_triggered.load(Ordering::Relaxed),
|
||||
total_disks: stats.disks_stats.len(),
|
||||
total_buckets: stats.buckets_stats.len(),
|
||||
last_update: stats.last_update,
|
||||
scan_progress: stats.scan_progress.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// record heal triggered
|
||||
pub async fn record_heal_triggered(&self, object_path: &str, error_message: &str) {
|
||||
self.counters.total_heal_triggered.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
info!("record heal triggered: object={}, error={}", object_path, error_message);
|
||||
}
|
||||
|
||||
/// update data usage stats
|
||||
pub async fn update_data_usage(&self, data_usage: DataUsageInfo) {
|
||||
let mut stats = self.stats.write().await;
|
||||
stats.data_usage = data_usage;
|
||||
stats.last_update = SystemTime::now();
|
||||
|
||||
debug!("update data usage stats");
|
||||
}
|
||||
|
||||
/// cleanup stats files
|
||||
pub async fn cleanup_stats_files(&self) -> Result<()> {
|
||||
// delete main file
|
||||
if self.stats_file.exists() {
|
||||
tokio::fs::remove_file(&self.stats_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete stats file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete backup file
|
||||
if self.backup_file.exists() {
|
||||
tokio::fs::remove_file(&self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete backup stats file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete temp file
|
||||
if self.temp_file.exists() {
|
||||
tokio::fs::remove_file(&self.temp_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete temp stats file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
info!("cleanup all stats files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// set save interval
|
||||
pub fn set_save_interval(&mut self, interval: Duration) {
|
||||
self.save_interval = interval;
|
||||
info!("set stats data save interval to {:?}", interval);
|
||||
}
|
||||
}
|
||||
|
||||
/// stats summary
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StatsSummary {
|
||||
/// node id
|
||||
pub node_id: String,
|
||||
/// total scanned objects
|
||||
pub total_objects_scanned: u64,
|
||||
/// total healthy objects
|
||||
pub total_healthy_objects: u64,
|
||||
/// total corrupted objects
|
||||
pub total_corrupted_objects: u64,
|
||||
/// total scanned bytes
|
||||
pub total_bytes_scanned: u64,
|
||||
/// total scan errors
|
||||
pub total_scan_errors: u64,
|
||||
/// total heal triggered
|
||||
pub total_heal_triggered: u64,
|
||||
/// total disks
|
||||
pub total_disks: usize,
|
||||
/// total buckets
|
||||
pub total_buckets: usize,
|
||||
/// last update time
|
||||
pub last_update: SystemTime,
|
||||
/// scan progress
|
||||
pub scan_progress: super::node_scanner::ScanProgress,
|
||||
}
|
||||
@@ -12,10 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod checkpoint;
|
||||
pub mod data_scanner;
|
||||
pub mod histogram;
|
||||
pub mod io_monitor;
|
||||
pub mod io_throttler;
|
||||
pub mod lifecycle;
|
||||
pub mod local_stats;
|
||||
pub mod metrics;
|
||||
pub mod node_scanner;
|
||||
pub mod stats_aggregator;
|
||||
|
||||
pub use data_scanner::Scanner;
|
||||
pub use checkpoint::{CheckpointData, CheckpointInfo, CheckpointManager};
|
||||
pub use data_scanner::{ScanMode, Scanner, ScannerConfig, ScannerState};
|
||||
pub use io_monitor::{AdvancedIOMonitor, IOMetrics, IOMonitorConfig};
|
||||
pub use io_throttler::{AdvancedIOThrottler, IOThrottlerConfig, ResourceAllocation, ThrottleDecision};
|
||||
pub use local_stats::{BatchScanResult, LocalStatsManager, ScanResultEntry, StatsSummary};
|
||||
pub use metrics::ScannerMetrics;
|
||||
pub use node_scanner::{IOMonitor, IOThrottler, LoadLevel, LocalScanStats, NodeScanner, NodeScannerConfig};
|
||||
pub use stats_aggregator::{AggregatedStats, DecentralizedStatsAggregator, NodeClient, NodeInfo};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,572 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
|
||||
use super::{
|
||||
local_stats::StatsSummary,
|
||||
node_scanner::{BucketStats, LoadLevel, ScanProgress},
|
||||
};
|
||||
use crate::{Error, error::Result};
|
||||
|
||||
/// node client config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeClientConfig {
|
||||
/// connect timeout
|
||||
pub connect_timeout: Duration,
|
||||
/// request timeout
|
||||
pub request_timeout: Duration,
|
||||
/// retry times
|
||||
pub max_retries: u32,
|
||||
/// retry interval
|
||||
pub retry_interval: Duration,
|
||||
}
|
||||
|
||||
impl Default for NodeClientConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
connect_timeout: Duration::from_secs(5),
|
||||
request_timeout: Duration::from_secs(10),
|
||||
max_retries: 3,
|
||||
retry_interval: Duration::from_secs(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// node info
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NodeInfo {
|
||||
/// node id
|
||||
pub node_id: String,
|
||||
/// node address
|
||||
pub address: String,
|
||||
/// node port
|
||||
pub port: u16,
|
||||
/// is online
|
||||
pub is_online: bool,
|
||||
/// last heartbeat time
|
||||
pub last_heartbeat: SystemTime,
|
||||
/// node version
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
/// aggregated stats
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AggregatedStats {
|
||||
/// aggregation timestamp
|
||||
pub aggregation_timestamp: SystemTime,
|
||||
/// number of nodes participating in aggregation
|
||||
pub node_count: usize,
|
||||
/// number of online nodes
|
||||
pub online_node_count: usize,
|
||||
/// total scanned objects
|
||||
pub total_objects_scanned: u64,
|
||||
/// total healthy objects
|
||||
pub total_healthy_objects: u64,
|
||||
/// total corrupted objects
|
||||
pub total_corrupted_objects: u64,
|
||||
/// total scanned bytes
|
||||
pub total_bytes_scanned: u64,
|
||||
/// total scan errors
|
||||
pub total_scan_errors: u64,
|
||||
/// total heal triggered
|
||||
pub total_heal_triggered: u64,
|
||||
/// total disks
|
||||
pub total_disks: usize,
|
||||
/// total buckets
|
||||
pub total_buckets: usize,
|
||||
/// aggregated data usage
|
||||
pub aggregated_data_usage: DataUsageInfo,
|
||||
/// node summaries
|
||||
pub node_summaries: HashMap<String, StatsSummary>,
|
||||
/// aggregated bucket stats
|
||||
pub aggregated_bucket_stats: HashMap<String, BucketStats>,
|
||||
/// aggregated scan progress
|
||||
pub scan_progress_summary: ScanProgressSummary,
|
||||
/// load level distribution
|
||||
pub load_level_distribution: HashMap<LoadLevel, usize>,
|
||||
}
|
||||
|
||||
impl Default for AggregatedStats {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
aggregation_timestamp: SystemTime::now(),
|
||||
node_count: 0,
|
||||
online_node_count: 0,
|
||||
total_objects_scanned: 0,
|
||||
total_healthy_objects: 0,
|
||||
total_corrupted_objects: 0,
|
||||
total_bytes_scanned: 0,
|
||||
total_scan_errors: 0,
|
||||
total_heal_triggered: 0,
|
||||
total_disks: 0,
|
||||
total_buckets: 0,
|
||||
aggregated_data_usage: DataUsageInfo::default(),
|
||||
node_summaries: HashMap::new(),
|
||||
aggregated_bucket_stats: HashMap::new(),
|
||||
scan_progress_summary: ScanProgressSummary::default(),
|
||||
load_level_distribution: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// scan progress summary
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ScanProgressSummary {
|
||||
/// average current cycle
|
||||
pub average_current_cycle: f64,
|
||||
/// total completed disks
|
||||
pub total_completed_disks: usize,
|
||||
/// total completed buckets
|
||||
pub total_completed_buckets: usize,
|
||||
/// latest scan start time
|
||||
pub earliest_scan_start: Option<SystemTime>,
|
||||
/// estimated completion time
|
||||
pub estimated_completion: Option<SystemTime>,
|
||||
/// node progress
|
||||
pub node_progress: HashMap<String, ScanProgress>,
|
||||
}
|
||||
|
||||
/// node client
|
||||
///
|
||||
/// responsible for communicating with other nodes, getting stats data
|
||||
pub struct NodeClient {
|
||||
/// node info
|
||||
node_info: NodeInfo,
|
||||
/// config
|
||||
config: NodeClientConfig,
|
||||
/// HTTP client
|
||||
http_client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl NodeClient {
|
||||
/// create new node client
|
||||
pub fn new(node_info: NodeInfo, config: NodeClientConfig) -> Self {
|
||||
let http_client = reqwest::Client::builder()
|
||||
.timeout(config.request_timeout)
|
||||
.connect_timeout(config.connect_timeout)
|
||||
.build()
|
||||
.expect("Failed to create HTTP client");
|
||||
|
||||
Self {
|
||||
node_info,
|
||||
config,
|
||||
http_client,
|
||||
}
|
||||
}
|
||||
|
||||
/// get node stats summary
|
||||
pub async fn get_stats_summary(&self) -> Result<StatsSummary> {
|
||||
let url = format!("http://{}:{}/internal/scanner/stats", self.node_info.address, self.node_info.port);
|
||||
|
||||
for attempt in 1..=self.config.max_retries {
|
||||
match self.try_get_stats_summary(&url).await {
|
||||
Ok(summary) => return Ok(summary),
|
||||
Err(e) => {
|
||||
warn!("try to get node {} stats failed: {}", self.node_info.node_id, e);
|
||||
|
||||
if attempt < self.config.max_retries {
|
||||
tokio::time::sleep(self.config.retry_interval).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::Other(format!("cannot get stats data from node {}", self.node_info.node_id)))
|
||||
}
|
||||
|
||||
/// try to get stats summary
|
||||
async fn try_get_stats_summary(&self, url: &str) -> Result<StatsSummary> {
|
||||
let response = self
|
||||
.http_client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::Other(format!("HTTP request failed: {}", e)))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::Other(format!("HTTP status error: {}", response.status())));
|
||||
}
|
||||
|
||||
let summary = response
|
||||
.json::<StatsSummary>()
|
||||
.await
|
||||
.map_err(|e| Error::Serialization(format!("deserialize stats data failed: {}", e)))?;
|
||||
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// check node health status
|
||||
pub async fn check_health(&self) -> bool {
|
||||
let url = format!("http://{}:{}/internal/health", self.node_info.address, self.node_info.port);
|
||||
|
||||
match self.http_client.get(&url).send().await {
|
||||
Ok(response) => response.status().is_success(),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// get node info
|
||||
pub fn get_node_info(&self) -> &NodeInfo {
|
||||
&self.node_info
|
||||
}
|
||||
|
||||
/// update node online status
|
||||
pub fn update_online_status(&mut self, is_online: bool) {
|
||||
self.node_info.is_online = is_online;
|
||||
if is_online {
|
||||
self.node_info.last_heartbeat = SystemTime::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// decentralized stats aggregator config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DecentralizedStatsAggregatorConfig {
|
||||
/// aggregation interval
|
||||
pub aggregation_interval: Duration,
|
||||
/// cache ttl
|
||||
pub cache_ttl: Duration,
|
||||
/// node timeout
|
||||
pub node_timeout: Duration,
|
||||
/// max concurrent aggregations
|
||||
pub max_concurrent_aggregations: usize,
|
||||
}
|
||||
|
||||
impl Default for DecentralizedStatsAggregatorConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
aggregation_interval: Duration::from_secs(30), // 30 seconds to aggregate
|
||||
cache_ttl: Duration::from_secs(3), // 3 seconds to cache
|
||||
node_timeout: Duration::from_secs(5), // 5 seconds to node timeout
|
||||
max_concurrent_aggregations: 10, // max 10 nodes to aggregate concurrently
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// decentralized stats aggregator
|
||||
///
|
||||
/// real-time aggregate stats data from all nodes, provide global view
|
||||
pub struct DecentralizedStatsAggregator {
|
||||
/// config
|
||||
config: Arc<RwLock<DecentralizedStatsAggregatorConfig>>,
|
||||
/// node clients
|
||||
node_clients: Arc<RwLock<HashMap<String, Arc<NodeClient>>>>,
|
||||
/// cached aggregated stats
|
||||
cached_stats: Arc<RwLock<Option<AggregatedStats>>>,
|
||||
/// cache timestamp
|
||||
cache_timestamp: Arc<RwLock<SystemTime>>,
|
||||
/// local node stats summary
|
||||
local_stats_summary: Arc<RwLock<Option<StatsSummary>>>,
|
||||
}
|
||||
|
||||
impl DecentralizedStatsAggregator {
|
||||
/// create new decentralized stats aggregator
|
||||
pub fn new(config: DecentralizedStatsAggregatorConfig) -> Self {
|
||||
Self {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
node_clients: Arc::new(RwLock::new(HashMap::new())),
|
||||
cached_stats: Arc::new(RwLock::new(None)),
|
||||
cache_timestamp: Arc::new(RwLock::new(SystemTime::UNIX_EPOCH)),
|
||||
local_stats_summary: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// add node client
|
||||
pub async fn add_node(&self, node_info: NodeInfo) {
|
||||
let client_config = NodeClientConfig::default();
|
||||
let client = Arc::new(NodeClient::new(node_info.clone(), client_config));
|
||||
|
||||
self.node_clients.write().await.insert(node_info.node_id.clone(), client);
|
||||
|
||||
info!("add node to aggregator: {}", node_info.node_id);
|
||||
}
|
||||
|
||||
/// remove node client
|
||||
pub async fn remove_node(&self, node_id: &str) {
|
||||
self.node_clients.write().await.remove(node_id);
|
||||
info!("remove node from aggregator: {}", node_id);
|
||||
}
|
||||
|
||||
/// set local node stats summary
|
||||
pub async fn set_local_stats(&self, stats: StatsSummary) {
|
||||
*self.local_stats_summary.write().await = Some(stats);
|
||||
}
|
||||
|
||||
/// get aggregated stats data (with cache)
|
||||
pub async fn get_aggregated_stats(&self) -> Result<AggregatedStats> {
|
||||
let config = self.config.read().await;
|
||||
let cache_ttl = config.cache_ttl;
|
||||
drop(config);
|
||||
|
||||
// check cache validity
|
||||
let cache_timestamp = *self.cache_timestamp.read().await;
|
||||
let now = SystemTime::now();
|
||||
|
||||
debug!(
|
||||
"cache check: cache_timestamp={:?}, now={:?}, cache_ttl={:?}",
|
||||
cache_timestamp, now, cache_ttl
|
||||
);
|
||||
|
||||
// Check cache validity if timestamp is not initial value (UNIX_EPOCH)
|
||||
if cache_timestamp != SystemTime::UNIX_EPOCH {
|
||||
if let Ok(elapsed) = now.duration_since(cache_timestamp) {
|
||||
if elapsed < cache_ttl {
|
||||
if let Some(cached) = self.cached_stats.read().await.as_ref() {
|
||||
debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed);
|
||||
return Ok(cached.clone());
|
||||
}
|
||||
} else {
|
||||
debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cache expired, re-aggregate
|
||||
info!("cache expired, start re-aggregating stats data");
|
||||
let aggregation_timestamp = now;
|
||||
let aggregated = self.aggregate_stats_from_all_nodes(aggregation_timestamp).await?;
|
||||
|
||||
// update cache
|
||||
*self.cached_stats.write().await = Some(aggregated.clone());
|
||||
*self.cache_timestamp.write().await = aggregation_timestamp;
|
||||
|
||||
Ok(aggregated)
|
||||
}
|
||||
|
||||
/// force refresh aggregated stats (ignore cache)
|
||||
pub async fn force_refresh_aggregated_stats(&self) -> Result<AggregatedStats> {
|
||||
let now = SystemTime::now();
|
||||
let aggregated = self.aggregate_stats_from_all_nodes(now).await?;
|
||||
|
||||
// update cache
|
||||
*self.cached_stats.write().await = Some(aggregated.clone());
|
||||
*self.cache_timestamp.write().await = now;
|
||||
|
||||
Ok(aggregated)
|
||||
}
|
||||
|
||||
/// aggregate stats data from all nodes
|
||||
async fn aggregate_stats_from_all_nodes(&self, aggregation_timestamp: SystemTime) -> Result<AggregatedStats> {
|
||||
let node_clients = self.node_clients.read().await;
|
||||
let config = self.config.read().await;
|
||||
|
||||
// concurrent get stats data from all nodes
|
||||
let mut tasks = Vec::new();
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_aggregations));
|
||||
|
||||
// add local node stats
|
||||
let mut node_summaries = HashMap::new();
|
||||
if let Some(local_stats) = self.local_stats_summary.read().await.as_ref() {
|
||||
node_summaries.insert(local_stats.node_id.clone(), local_stats.clone());
|
||||
}
|
||||
|
||||
// get remote node stats
|
||||
for (node_id, client) in node_clients.iter() {
|
||||
let client = client.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
let node_id = node_id.clone();
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let _permit = match semaphore.acquire().await {
|
||||
Ok(permit) => permit,
|
||||
Err(e) => {
|
||||
warn!("Failed to acquire semaphore for node {}: {}", node_id, e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match client.get_stats_summary().await {
|
||||
Ok(summary) => {
|
||||
debug!("successfully get node {} stats data", node_id);
|
||||
Some((node_id, summary))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("get node {} stats data failed: {}", node_id, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tasks.push(task);
|
||||
}
|
||||
|
||||
// wait for all tasks to complete
|
||||
for task in tasks {
|
||||
if let Ok(Some((node_id, summary))) = task.await {
|
||||
node_summaries.insert(node_id, summary);
|
||||
}
|
||||
}
|
||||
|
||||
drop(node_clients);
|
||||
drop(config);
|
||||
|
||||
// aggregate stats data
|
||||
let aggregated = self.aggregate_node_summaries(node_summaries, aggregation_timestamp).await;
|
||||
|
||||
info!(
|
||||
"aggregate stats completed: {} nodes, {} online",
|
||||
aggregated.node_count, aggregated.online_node_count
|
||||
);
|
||||
|
||||
Ok(aggregated)
|
||||
}
|
||||
|
||||
/// aggregate node summaries
|
||||
async fn aggregate_node_summaries(
|
||||
&self,
|
||||
node_summaries: HashMap<String, StatsSummary>,
|
||||
aggregation_timestamp: SystemTime,
|
||||
) -> AggregatedStats {
|
||||
let mut aggregated = AggregatedStats {
|
||||
aggregation_timestamp,
|
||||
node_count: node_summaries.len(),
|
||||
online_node_count: node_summaries.len(), // assume all nodes with data are online
|
||||
node_summaries: node_summaries.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// aggregate numeric stats
|
||||
for (node_id, summary) in &node_summaries {
|
||||
aggregated.total_objects_scanned += summary.total_objects_scanned;
|
||||
aggregated.total_healthy_objects += summary.total_healthy_objects;
|
||||
aggregated.total_corrupted_objects += summary.total_corrupted_objects;
|
||||
aggregated.total_bytes_scanned += summary.total_bytes_scanned;
|
||||
aggregated.total_scan_errors += summary.total_scan_errors;
|
||||
aggregated.total_heal_triggered += summary.total_heal_triggered;
|
||||
aggregated.total_disks += summary.total_disks;
|
||||
aggregated.total_buckets += summary.total_buckets;
|
||||
|
||||
// aggregate scan progress
|
||||
aggregated
|
||||
.scan_progress_summary
|
||||
.node_progress
|
||||
.insert(node_id.clone(), summary.scan_progress.clone());
|
||||
|
||||
aggregated.scan_progress_summary.total_completed_disks += summary.scan_progress.completed_disks.len();
|
||||
aggregated.scan_progress_summary.total_completed_buckets += summary.scan_progress.completed_buckets.len();
|
||||
}
|
||||
|
||||
// calculate average scan cycle
|
||||
if !node_summaries.is_empty() {
|
||||
let total_cycles: u64 = node_summaries.values().map(|s| s.scan_progress.current_cycle).sum();
|
||||
aggregated.scan_progress_summary.average_current_cycle = total_cycles as f64 / node_summaries.len() as f64;
|
||||
}
|
||||
|
||||
// find earliest scan start time
|
||||
aggregated.scan_progress_summary.earliest_scan_start =
|
||||
node_summaries.values().map(|s| s.scan_progress.scan_start_time).min();
|
||||
|
||||
// TODO: aggregate bucket stats and data usage
|
||||
// here we need to implement it based on the specific BucketStats and DataUsageInfo structure
|
||||
|
||||
aggregated
|
||||
}
|
||||
|
||||
/// get nodes health status
|
||||
pub async fn get_nodes_health(&self) -> HashMap<String, bool> {
|
||||
let node_clients = self.node_clients.read().await;
|
||||
let mut health_status = HashMap::new();
|
||||
|
||||
// concurrent check all nodes health status
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (node_id, client) in node_clients.iter() {
|
||||
let client = client.clone();
|
||||
let node_id = node_id.clone();
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let is_healthy = client.check_health().await;
|
||||
(node_id, is_healthy)
|
||||
});
|
||||
|
||||
tasks.push(task);
|
||||
}
|
||||
|
||||
// collect results
|
||||
for task in tasks {
|
||||
if let Ok((node_id, is_healthy)) = task.await {
|
||||
health_status.insert(node_id, is_healthy);
|
||||
}
|
||||
}
|
||||
|
||||
health_status
|
||||
}
|
||||
|
||||
/// get online nodes list
|
||||
pub async fn get_online_nodes(&self) -> Vec<String> {
|
||||
let health_status = self.get_nodes_health().await;
|
||||
|
||||
health_status
|
||||
.into_iter()
|
||||
.filter_map(|(node_id, is_healthy)| if is_healthy { Some(node_id) } else { None })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// clear cache
|
||||
pub async fn clear_cache(&self) {
|
||||
*self.cached_stats.write().await = None;
|
||||
*self.cache_timestamp.write().await = SystemTime::UNIX_EPOCH;
|
||||
info!("clear aggregated stats cache");
|
||||
}
|
||||
|
||||
/// get cache status
|
||||
pub async fn get_cache_status(&self) -> CacheStatus {
|
||||
let cached_stats = self.cached_stats.read().await;
|
||||
let cache_timestamp = *self.cache_timestamp.read().await;
|
||||
let config = self.config.read().await;
|
||||
|
||||
let is_valid = if let Ok(elapsed) = SystemTime::now().duration_since(cache_timestamp) {
|
||||
elapsed < config.cache_ttl
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
CacheStatus {
|
||||
has_cached_data: cached_stats.is_some(),
|
||||
cache_timestamp,
|
||||
is_valid,
|
||||
ttl: config.cache_ttl,
|
||||
}
|
||||
}
|
||||
|
||||
/// update config
|
||||
pub async fn update_config(&self, new_config: DecentralizedStatsAggregatorConfig) {
|
||||
*self.config.write().await = new_config;
|
||||
info!("update aggregator config");
|
||||
}
|
||||
}
|
||||
|
||||
/// cache status
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CacheStatus {
|
||||
/// has cached data
|
||||
pub has_cached_data: bool,
|
||||
/// cache timestamp
|
||||
pub cache_timestamp: SystemTime,
|
||||
/// cache is valid
|
||||
pub is_valid: bool,
|
||||
/// cache ttl
|
||||
pub ttl: Duration,
|
||||
}
|
||||
Reference in New Issue
Block a user