diff --git a/Cargo.lock b/Cargo.lock index 0613b2f8d..3098046bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7930,8 +7930,10 @@ dependencies = [ "anyhow", "async-trait", "bytes", + "chrono", "futures", "lazy_static", + "once_cell", "rmp-serde", "rustfs-common", "rustfs-ecstore", @@ -7941,14 +7943,18 @@ dependencies = [ "rustfs-utils", "serde", "serde_json", + "serial_test", + "tempfile", "thiserror 2.0.12", "time", "tokio", "tokio-test", "tokio-util", "tracing", + "tracing-subscriber", "url", "uuid", + "walkdir", ] [[package]] @@ -7965,8 +7971,18 @@ dependencies = [ name = "rustfs-common" version = "0.0.5" dependencies = [ + "async-trait", + "chrono", + "lazy_static", + "path-clean", + "rmp-serde", + "rustfs-filemeta", + "rustfs-madmin", + "s3s", + "serde", "tokio", "tonic", + "uuid", ] [[package]] diff --git a/crates/ahm/Cargo.toml b/crates/ahm/Cargo.toml index fc4eae4bd..8446f5802 100644 --- a/crates/ahm/Cargo.toml +++ b/crates/ahm/Cargo.toml @@ -34,8 +34,14 @@ url = { workspace = true } rustfs-lock = { workspace = true } lazy_static = { workspace = true } +chrono = { workspace = true } [dev-dependencies] rmp-serde = { workspace = true } tokio-test = { workspace = true } serde_json = { workspace = true } +serial_test = "3.2.0" +once_cell = { workspace = true } +tracing-subscriber = { workspace = true } +walkdir = "2.5.0" +tempfile = { workspace = true } diff --git a/crates/ahm/src/error.rs b/crates/ahm/src/error.rs index 894638970..aca503839 100644 --- a/crates/ahm/src/error.rs +++ b/crates/ahm/src/error.rs @@ -14,30 +14,79 @@ use thiserror::Error; +/// RustFS AHM/Heal/Scanner 统一错误类型 #[derive(Debug, Error)] pub enum Error { + // 通用 #[error("I/O error: {0}")] Io(#[from] std::io::Error), #[error("Storage error: {0}")] Storage(#[from] rustfs_ecstore::error::Error), + #[error("Disk error: {0}")] + Disk(#[from] rustfs_ecstore::disk::error::DiskError), + #[error("Configuration error: {0}")] Config(String), + #[error("Heal configuration error: {message}")] + ConfigurationError { message: String }, + + #[error("Other error: {0}")] + Other(String), + + #[error(transparent)] + Anyhow(#[from] anyhow::Error), + + // Scanner相关 #[error("Scanner error: {0}")] Scanner(String), #[error("Metrics error: {0}")] Metrics(String), - #[error(transparent)] - Other(#[from] anyhow::Error), + // Heal相关 + #[error("Heal task not found: {task_id}")] + TaskNotFound { task_id: String }, + + #[error("Heal task already exists: {task_id}")] + TaskAlreadyExists { task_id: String }, + + #[error("Heal manager is not running")] + ManagerNotRunning, + + #[error("Heal task execution failed: {message}")] + TaskExecutionFailed { message: String }, + + #[error("Invalid heal type: {heal_type}")] + InvalidHealType { heal_type: String }, + + #[error("Heal task cancelled")] + TaskCancelled, + + #[error("Heal task timeout")] + TaskTimeout, + + #[error("Heal event processing failed: {message}")] + EventProcessingFailed { message: String }, + + #[error("Heal progress tracking failed: {message}")] + ProgressTrackingFailed { message: String }, } pub type Result = std::result::Result; -// Implement conversion from ahm::Error to std::io::Error for use in main.rs +impl Error { + pub fn other(error: E) -> Self + where + E: Into>, + { + Error::Other(error.into().to_string()) + } +} + +// 可选：实现与 std::io::Error 的互转 impl From for std::io::Error { fn from(err: Error) -> Self { std::io::Error::other(err) diff --git a/crates/ahm/src/heal/channel.rs b/crates/ahm/src/heal/channel.rs new file mode 100644 index 000000000..ecfeae783 --- /dev/null +++ b/crates/ahm/src/heal/channel.rs @@ -0,0 +1,233 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::Result; +use crate::heal::{ + manager::HealManager, + task::{HealOptions, HealPriority, HealRequest, HealType}, +}; + +use rustfs_common::heal_channel::{ + HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse, HealScanMode, +}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tracing::{error, info}; + +/// Heal channel processor +pub struct HealChannelProcessor { + /// Heal manager + heal_manager: Arc, + /// Response sender + response_sender: mpsc::UnboundedSender, + /// Response receiver + response_receiver: mpsc::UnboundedReceiver, +} + +impl HealChannelProcessor { + /// Create new HealChannelProcessor + pub fn new(heal_manager: Arc) -> Self { + let (response_tx, response_rx) = mpsc::unbounded_channel(); + Self { + heal_manager, + response_sender: response_tx, + response_receiver: response_rx, + } + } + + /// Start processing heal channel requests + pub async fn start(&mut self, mut receiver: HealChannelReceiver) -> Result<()> { + info!("Starting heal channel processor"); + + loop { + tokio::select! { + command = receiver.recv() => { + match command { + Some(command) => { + if let Err(e) = self.process_command(command).await { + error!("Failed to process heal command: {}", e); + } + } + None => { + info!("Heal channel receiver closed, stopping processor"); + break; + } + } + } + response = self.response_receiver.recv() => { + if let Some(response) = response { + // Handle response if needed + info!("Received heal response for request: {}", response.request_id); + } + } + } + } + + info!("Heal channel processor stopped"); + Ok(()) + } + + /// Process heal command + async fn process_command(&self, command: HealChannelCommand) -> Result<()> { + match command { + HealChannelCommand::Start(request) => self.process_start_request(request).await, + HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await, + HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await, + } + } + + /// Process start request + async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> { + info!("Processing heal start request: {} for bucket: {}", request.id, request.bucket); + + // Convert channel request to heal request + let heal_request = self.convert_to_heal_request(request.clone())?; + + // Submit to heal manager + match self.heal_manager.submit_heal_request(heal_request).await { + Ok(task_id) => { + info!("Successfully submitted heal request: {} as task: {}", request.id, task_id); + + // Send success response + let response = HealChannelResponse { + request_id: request.id, + success: true, + data: Some(format!("Task ID: {task_id}").into_bytes()), + error: None, + }; + + if let Err(e) = self.response_sender.send(response) { + error!("Failed to send heal response: {}", e); + } + } + Err(e) => { + error!("Failed to submit heal request: {} - {}", request.id, e); + + // Send error response + let response = HealChannelResponse { + request_id: request.id, + success: false, + data: None, + error: Some(e.to_string()), + }; + + if let Err(e) = self.response_sender.send(response) { + error!("Failed to send heal error response: {}", e); + } + } + } + + Ok(()) + } + + /// Process query request + async fn process_query_request(&self, heal_path: String, client_token: String) -> Result<()> { + info!("Processing heal query request for path: {}", heal_path); + + // TODO: Implement query logic based on heal_path and client_token + // For now, return a placeholder response + let response = HealChannelResponse { + request_id: client_token, + success: true, + data: Some(format!("Query result for path: {heal_path}").into_bytes()), + error: None, + }; + + if let Err(e) = self.response_sender.send(response) { + error!("Failed to send query response: {}", e); + } + + Ok(()) + } + + /// Process cancel request + async fn process_cancel_request(&self, heal_path: String) -> Result<()> { + info!("Processing heal cancel request for path: {}", heal_path); + + // TODO: Implement cancel logic based on heal_path + // For now, return a placeholder response + let response = HealChannelResponse { + request_id: heal_path.clone(), + success: true, + data: Some(format!("Cancel request for path: {heal_path}").into_bytes()), + error: None, + }; + + if let Err(e) = self.response_sender.send(response) { + error!("Failed to send cancel response: {}", e); + } + + Ok(()) + } + + /// Convert channel request to heal request + fn convert_to_heal_request(&self, request: HealChannelRequest) -> Result { + let heal_type = if let Some(disk_id) = &request.disk { + HealType::ErasureSet { + buckets: vec![], + set_disk_id: disk_id.clone(), + } + } else if let Some(prefix) = &request.object_prefix { + if !prefix.is_empty() { + HealType::Object { + bucket: request.bucket.clone(), + object: prefix.clone(), + version_id: None, + } + } else { + HealType::Bucket { + bucket: request.bucket.clone(), + } + } + } else { + HealType::Bucket { + bucket: request.bucket.clone(), + } + }; + + let priority = match request.priority { + HealChannelPriority::Low => HealPriority::Low, + HealChannelPriority::Normal => HealPriority::Normal, + HealChannelPriority::High => HealPriority::High, + HealChannelPriority::Critical => HealPriority::Urgent, + }; + + // Build HealOptions with all available fields + let mut options = HealOptions { + scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal), + remove_corrupted: request.remove_corrupted.unwrap_or(false), + recreate_missing: request.recreate_missing.unwrap_or(true), + update_parity: request.update_parity.unwrap_or(true), + recursive: request.recursive.unwrap_or(false), + dry_run: request.dry_run.unwrap_or(false), + timeout: request.timeout_seconds.map(std::time::Duration::from_secs), + pool_index: request.pool_index, + set_index: request.set_index, + }; + + // Apply force_start overrides + if request.force_start { + options.remove_corrupted = true; + options.recreate_missing = true; + options.update_parity = true; + } + + Ok(HealRequest::new(heal_type, options, priority)) + } + + /// Get response sender for external use + pub fn get_response_sender(&self) -> mpsc::UnboundedSender { + self.response_sender.clone() + } +} diff --git a/crates/ahm/src/heal/erasure_healer.rs b/crates/ahm/src/heal/erasure_healer.rs new file mode 100644 index 000000000..f60d4afba --- /dev/null +++ b/crates/ahm/src/heal/erasure_healer.rs @@ -0,0 +1,456 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use crate::heal::{ + progress::HealProgress, + resume::{CheckpointManager, ResumeManager, ResumeUtils}, + storage::HealStorageAPI, +}; +use futures::future::join_all; +use rustfs_common::heal_channel::{HealOpts, HealScanMode}; +use rustfs_ecstore::disk::DiskStore; +use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{error, info, warn}; + +/// Erasure Set Healer +pub struct ErasureSetHealer { + storage: Arc, + progress: Arc>, + cancel_token: tokio_util::sync::CancellationToken, + disk: DiskStore, +} + +impl ErasureSetHealer { + pub fn new( + storage: Arc, + progress: Arc>, + cancel_token: tokio_util::sync::CancellationToken, + disk: DiskStore, + ) -> Self { + Self { + storage, + progress, + cancel_token, + disk, + } + } + + /// execute erasure set heal with resume + pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> { + info!("Starting erasure set heal for {} buckets on set disk {}", buckets.len(), set_disk_id); + + // 1. generate or get task id + let task_id = self.get_or_create_task_id(set_disk_id).await?; + + // 2. initialize or resume resume state + let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, buckets).await?; + + // 3. execute heal with resume + let result = self + .execute_heal_with_resume(buckets, &resume_manager, &checkpoint_manager) + .await; + + // 4. cleanup resume state + if result.is_ok() { + if let Err(e) = resume_manager.cleanup().await { + warn!("Failed to cleanup resume state: {}", e); + } + if let Err(e) = checkpoint_manager.cleanup().await { + warn!("Failed to cleanup checkpoint: {}", e); + } + } + + result + } + + /// get or create task id + async fn get_or_create_task_id(&self, _set_disk_id: &str) -> Result { + // check if there are resumable tasks + let resumable_tasks = ResumeUtils::get_resumable_tasks(&self.disk).await?; + + for task_id in resumable_tasks { + if ResumeUtils::can_resume_task(&self.disk, &task_id).await { + info!("Found resumable task: {}", task_id); + return Ok(task_id); + } + } + + // create new task id + let task_id = ResumeUtils::generate_task_id(); + info!("Created new heal task: {}", task_id); + Ok(task_id) + } + + /// initialize or resume resume state + async fn initialize_resume_state(&self, task_id: &str, buckets: &[String]) -> Result<(ResumeManager, CheckpointManager)> { + // check if resume state exists + if ResumeManager::has_resume_state(&self.disk, task_id).await { + info!("Loading existing resume state for task: {}", task_id); + + let resume_manager = ResumeManager::load_from_disk(self.disk.clone(), task_id).await?; + let checkpoint_manager = if CheckpointManager::has_checkpoint(&self.disk, task_id).await { + CheckpointManager::load_from_disk(self.disk.clone(), task_id).await? + } else { + CheckpointManager::new(self.disk.clone(), task_id.to_string()).await? + }; + + Ok((resume_manager, checkpoint_manager)) + } else { + info!("Creating new resume state for task: {}", task_id); + + let resume_manager = + ResumeManager::new(self.disk.clone(), task_id.to_string(), "erasure_set".to_string(), buckets.to_vec()).await?; + + let checkpoint_manager = CheckpointManager::new(self.disk.clone(), task_id.to_string()).await?; + + Ok((resume_manager, checkpoint_manager)) + } + } + + /// execute heal with resume + async fn execute_heal_with_resume( + &self, + buckets: &[String], + resume_manager: &ResumeManager, + checkpoint_manager: &CheckpointManager, + ) -> Result<()> { + // 1. get current state + let state = resume_manager.get_state().await; + let checkpoint = checkpoint_manager.get_checkpoint().await; + + info!( + "Resuming from bucket {} object {}", + checkpoint.current_bucket_index, checkpoint.current_object_index + ); + + // 2. initialize progress + self.initialize_progress(buckets, &state).await; + + // 3. continue from checkpoint + let current_bucket_index = checkpoint.current_bucket_index; + let mut current_object_index = checkpoint.current_object_index; + + let mut processed_objects = state.processed_objects; + let mut successful_objects = state.successful_objects; + let mut failed_objects = state.failed_objects; + let mut skipped_objects = state.skipped_objects; + + // 4. process remaining buckets + for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) { + // check if completed + if state.completed_buckets.contains(bucket) { + continue; + } + + // update current bucket + resume_manager.set_current_item(Some(bucket.clone()), None).await?; + + // process objects in bucket + let bucket_result = self + .heal_bucket_with_resume( + bucket, + &mut current_object_index, + &mut processed_objects, + &mut successful_objects, + &mut failed_objects, + &mut skipped_objects, + resume_manager, + checkpoint_manager, + ) + .await; + + // update checkpoint position + checkpoint_manager.update_position(bucket_idx, current_object_index).await?; + + // update progress + resume_manager + .update_progress(processed_objects, successful_objects, failed_objects, skipped_objects) + .await?; + + // check cancel status + if self.cancel_token.is_cancelled() { + info!("Heal task cancelled"); + return Err(Error::TaskCancelled); + } + + // process bucket result + match bucket_result { + Ok(_) => { + resume_manager.complete_bucket(bucket).await?; + info!("Completed heal for bucket: {}", bucket); + } + Err(e) => { + error!("Failed to heal bucket {}: {}", bucket, e); + // continue to next bucket, do not interrupt the whole process + } + } + + // reset object index + current_object_index = 0; + } + + // 5. mark task completed + resume_manager.mark_completed().await?; + + info!("Erasure set heal completed successfully"); + Ok(()) + } + + /// heal single bucket with resume + #[allow(clippy::too_many_arguments)] + async fn heal_bucket_with_resume( + &self, + bucket: &str, + current_object_index: &mut usize, + processed_objects: &mut u64, + successful_objects: &mut u64, + failed_objects: &mut u64, + _skipped_objects: &mut u64, + resume_manager: &ResumeManager, + checkpoint_manager: &CheckpointManager, + ) -> Result<()> { + info!("Starting heal for bucket: {} from object index {}", bucket, current_object_index); + + // 1. get bucket info + let _bucket_info = match self.storage.get_bucket_info(bucket).await? { + Some(info) => info, + None => { + warn!("Bucket {} not found, skipping", bucket); + return Ok(()); + } + }; + + // 2. get objects to heal + let objects = self.storage.list_objects_for_heal(bucket, "").await?; + + // 3. continue from checkpoint + for (obj_idx, object) in objects.iter().enumerate().skip(*current_object_index) { + // check if already processed + if checkpoint_manager.get_checkpoint().await.processed_objects.contains(object) { + continue; + } + + // update current object + resume_manager + .set_current_item(Some(bucket.to_string()), Some(object.clone())) + .await?; + + // heal object + let heal_opts = HealOpts { + scan_mode: HealScanMode::Normal, + remove: true, + recreate: true, + ..Default::default() + }; + + match self.storage.heal_object(bucket, object, None, &heal_opts).await { + Ok((_result, None)) => { + *successful_objects += 1; + checkpoint_manager.add_processed_object(object.clone()).await?; + info!("Successfully healed object {}/{}", bucket, object); + } + Ok((_, Some(err))) => { + *failed_objects += 1; + checkpoint_manager.add_failed_object(object.clone()).await?; + warn!("Failed to heal object {}/{}: {}", bucket, object, err); + } + Err(err) => { + *failed_objects += 1; + checkpoint_manager.add_failed_object(object.clone()).await?; + warn!("Error healing object {}/{}: {}", bucket, object, err); + } + } + + *processed_objects += 1; + *current_object_index = obj_idx + 1; + + // check cancel status + if self.cancel_token.is_cancelled() { + info!("Heal task cancelled during object processing"); + return Err(Error::TaskCancelled); + } + + // save checkpoint periodically + if obj_idx % 100 == 0 { + checkpoint_manager.update_position(0, *current_object_index).await?; + } + } + + Ok(()) + } + + /// initialize progress tracking + async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) { + let mut progress = self.progress.write().await; + progress.objects_scanned = state.total_objects; + progress.objects_healed = state.successful_objects; + progress.objects_failed = state.failed_objects; + progress.bytes_processed = 0; // set to 0 for now, can be extended later + progress.set_current_object(state.current_object.clone()); + } + + /// heal all buckets concurrently + #[allow(dead_code)] + async fn heal_buckets_concurrently(&self, buckets: &[String]) -> Vec> { + // use semaphore to control concurrency, avoid too many concurrent healings + let semaphore = Arc::new(tokio::sync::Semaphore::new(4)); // max 4 concurrent healings + + let heal_futures = buckets.iter().map(|bucket| { + let bucket = bucket.clone(); + let storage = self.storage.clone(); + let progress = self.progress.clone(); + let semaphore = semaphore.clone(); + let cancel_token = self.cancel_token.clone(); + + async move { + let _permit = semaphore.acquire().await.unwrap(); + + if cancel_token.is_cancelled() { + return Err(Error::TaskCancelled); + } + + Self::heal_single_bucket(&storage, &bucket, &progress).await + } + }); + + // use join_all to process concurrently + join_all(heal_futures).await + } + + /// heal single bucket + #[allow(dead_code)] + async fn heal_single_bucket( + storage: &Arc, + bucket: &str, + progress: &Arc>, + ) -> Result<()> { + info!("Starting heal for bucket: {}", bucket); + + // 1. get bucket info + let _bucket_info = match storage.get_bucket_info(bucket).await? { + Some(info) => info, + None => { + warn!("Bucket {} not found, skipping", bucket); + return Ok(()); + } + }; + + // 2. get objects to heal + let objects = storage.list_objects_for_heal(bucket, "").await?; + + // 3. update progress + { + let mut p = progress.write().await; + p.objects_scanned += objects.len() as u64; + } + + // 4. heal objects concurrently + let heal_opts = HealOpts { + scan_mode: HealScanMode::Normal, + remove: true, // remove corrupted data + recreate: true, // recreate missing data + ..Default::default() + }; + + let object_results = Self::heal_objects_concurrently(storage, bucket, &objects, &heal_opts, progress).await; + + // 5. count results + let (success_count, failure_count) = object_results + .into_iter() + .fold((0, 0), |(success, failure), result| match result { + Ok(_) => (success + 1, failure), + Err(_) => (success, failure + 1), + }); + + // 6. update progress + { + let mut p = progress.write().await; + p.objects_healed += success_count; + p.objects_failed += failure_count; + p.set_current_object(Some(format!("completed bucket: {bucket}"))); + } + + info!( + "Completed heal for bucket {}: {} success, {} failures", + bucket, success_count, failure_count + ); + + Ok(()) + } + + /// heal objects concurrently + #[allow(dead_code)] + async fn heal_objects_concurrently( + storage: &Arc, + bucket: &str, + objects: &[String], + heal_opts: &HealOpts, + _progress: &Arc>, + ) -> Vec> { + // use semaphore to control object healing concurrency + let semaphore = Arc::new(tokio::sync::Semaphore::new(8)); // max 8 concurrent object healings + + let heal_futures = objects.iter().map(|object| { + let object = object.clone(); + let bucket = bucket.to_string(); + let storage = storage.clone(); + let heal_opts = *heal_opts; + let semaphore = semaphore.clone(); + + async move { + let _permit = semaphore.acquire().await.unwrap(); + + match storage.heal_object(&bucket, &object, None, &heal_opts).await { + Ok((_result, None)) => { + info!("Successfully healed object {}/{}", bucket, object); + Ok(()) + } + Ok((_, Some(err))) => { + warn!("Failed to heal object {}/{}: {}", bucket, object, err); + Err(Error::other(err)) + } + Err(err) => { + warn!("Error healing object {}/{}: {}", bucket, object, err); + Err(err) + } + } + } + }); + + join_all(heal_futures).await + } + + /// process results + #[allow(dead_code)] + async fn process_results(&self, results: Vec>) -> Result<()> { + let (success_count, failure_count): (usize, usize) = + results.into_iter().fold((0, 0), |(success, failure), result| match result { + Ok(_) => (success + 1, failure), + Err(_) => (success, failure + 1), + }); + + let total = success_count + failure_count; + + info!("Erasure set heal completed: {}/{} buckets successful", success_count, total); + + if failure_count > 0 { + warn!("{} buckets failed to heal", failure_count); + return Err(Error::other(format!("{failure_count} buckets failed to heal"))); + } + + Ok(()) + } +} diff --git a/crates/ahm/src/heal/event.rs b/crates/ahm/src/heal/event.rs new file mode 100644 index 000000000..4e773d1df --- /dev/null +++ b/crates/ahm/src/heal/event.rs @@ -0,0 +1,359 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; +use rustfs_ecstore::disk::endpoint::Endpoint; +use serde::{Deserialize, Serialize}; +use std::time::SystemTime; + +/// Corruption type +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum CorruptionType { + /// Data corruption + DataCorruption, + /// Metadata corruption + MetadataCorruption, + /// Partial corruption + PartialCorruption, + /// Complete corruption + CompleteCorruption, +} + +/// Severity level +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Severity { + /// Low severity + Low = 0, + /// Medium severity + Medium = 1, + /// High severity + High = 2, + /// Critical severity + Critical = 3, +} + +/// Heal event +#[derive(Debug, Clone)] +pub enum HealEvent { + /// Object corruption event + ObjectCorruption { + bucket: String, + object: String, + version_id: Option, + corruption_type: CorruptionType, + severity: Severity, + }, + /// Object missing event + ObjectMissing { + bucket: String, + object: String, + version_id: Option, + expected_locations: Vec, + available_locations: Vec, + }, + /// Metadata corruption event + MetadataCorruption { + bucket: String, + object: String, + corruption_type: CorruptionType, + }, + /// Disk status change event + DiskStatusChange { + endpoint: Endpoint, + old_status: String, + new_status: String, + }, + /// EC decode failure event + ECDecodeFailure { + bucket: String, + object: String, + version_id: Option, + missing_shards: Vec, + available_shards: Vec, + }, + /// Checksum mismatch event + ChecksumMismatch { + bucket: String, + object: String, + version_id: Option, + expected_checksum: String, + actual_checksum: String, + }, + /// Bucket metadata corruption event + BucketMetadataCorruption { + bucket: String, + corruption_type: CorruptionType, + }, + /// MRF metadata corruption event + MRFMetadataCorruption { + meta_path: String, + corruption_type: CorruptionType, + }, +} + +impl HealEvent { + /// Convert HealEvent to HealRequest + pub fn to_heal_request(&self) -> HealRequest { + match self { + HealEvent::ObjectCorruption { + bucket, + object, + version_id, + severity, + .. + } => HealRequest::new( + HealType::Object { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + HealOptions::default(), + Self::severity_to_priority(severity), + ), + HealEvent::ObjectMissing { + bucket, + object, + version_id, + .. + } => HealRequest::new( + HealType::Object { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + HealOptions::default(), + HealPriority::High, + ), + HealEvent::MetadataCorruption { bucket, object, .. } => HealRequest::new( + HealType::Metadata { + bucket: bucket.clone(), + object: object.clone(), + }, + HealOptions::default(), + HealPriority::High, + ), + HealEvent::DiskStatusChange { endpoint, .. } => { + // Convert disk status change to erasure set heal + // Note: This requires access to storage to get bucket list, which is not available here + // The actual bucket list will need to be provided by the caller or retrieved differently + HealRequest::new( + HealType::ErasureSet { + buckets: vec![], // Empty bucket list - caller should populate this + set_disk_id: format!("{}_{}", endpoint.pool_idx, endpoint.set_idx), + }, + HealOptions::default(), + HealPriority::High, + ) + } + HealEvent::ECDecodeFailure { + bucket, + object, + version_id, + .. + } => HealRequest::new( + HealType::ECDecode { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + HealOptions::default(), + HealPriority::Urgent, + ), + HealEvent::ChecksumMismatch { + bucket, + object, + version_id, + .. + } => HealRequest::new( + HealType::Object { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + HealOptions::default(), + HealPriority::High, + ), + HealEvent::BucketMetadataCorruption { bucket, .. } => { + HealRequest::new(HealType::Bucket { bucket: bucket.clone() }, HealOptions::default(), HealPriority::High) + } + HealEvent::MRFMetadataCorruption { meta_path, .. } => HealRequest::new( + HealType::MRF { + meta_path: meta_path.clone(), + }, + HealOptions::default(), + HealPriority::High, + ), + } + } + + /// Convert severity to priority + fn severity_to_priority(severity: &Severity) -> HealPriority { + match severity { + Severity::Low => HealPriority::Low, + Severity::Medium => HealPriority::Normal, + Severity::High => HealPriority::High, + Severity::Critical => HealPriority::Urgent, + } + } + + /// Get event description + pub fn description(&self) -> String { + match self { + HealEvent::ObjectCorruption { + bucket, + object, + corruption_type, + .. + } => { + format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}") + } + HealEvent::ObjectMissing { bucket, object, .. } => { + format!("Object missing: {bucket}/{object}") + } + HealEvent::MetadataCorruption { + bucket, + object, + corruption_type, + .. + } => { + format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}") + } + HealEvent::DiskStatusChange { + endpoint, + old_status, + new_status, + .. + } => { + format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}") + } + HealEvent::ECDecodeFailure { + bucket, + object, + missing_shards, + .. + } => { + format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}") + } + HealEvent::ChecksumMismatch { + bucket, + object, + expected_checksum, + actual_checksum, + .. + } => { + format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}") + } + HealEvent::BucketMetadataCorruption { + bucket, corruption_type, .. + } => { + format!("Bucket metadata corruption: {bucket} - {corruption_type:?}") + } + HealEvent::MRFMetadataCorruption { + meta_path, + corruption_type, + .. + } => { + format!("MRF metadata corruption: {meta_path} - {corruption_type:?}") + } + } + } + + /// Get event severity + pub fn severity(&self) -> Severity { + match self { + HealEvent::ObjectCorruption { severity, .. } => severity.clone(), + HealEvent::ObjectMissing { .. } => Severity::High, + HealEvent::MetadataCorruption { .. } => Severity::High, + HealEvent::DiskStatusChange { .. } => Severity::High, + HealEvent::ECDecodeFailure { .. } => Severity::Critical, + HealEvent::ChecksumMismatch { .. } => Severity::High, + HealEvent::BucketMetadataCorruption { .. } => Severity::High, + HealEvent::MRFMetadataCorruption { .. } => Severity::High, + } + } + + /// Get event timestamp + pub fn timestamp(&self) -> SystemTime { + SystemTime::now() + } +} + +/// Heal event handler +pub struct HealEventHandler { + /// Event queue + events: Vec, + /// Maximum number of events + max_events: usize, +} + +impl HealEventHandler { + pub fn new(max_events: usize) -> Self { + Self { + events: Vec::new(), + max_events, + } + } + + /// Add event + pub fn add_event(&mut self, event: HealEvent) { + if self.events.len() >= self.max_events { + // Remove oldest event + self.events.remove(0); + } + self.events.push(event); + } + + /// Get all events + pub fn get_events(&self) -> &[HealEvent] { + &self.events + } + + /// Clear events + pub fn clear_events(&mut self) { + self.events.clear(); + } + + /// Get event count + pub fn event_count(&self) -> usize { + self.events.len() + } + + /// Filter events by severity + pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> { + self.events.iter().filter(|event| event.severity() >= min_severity).collect() + } + + /// Filter events by type + pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> { + self.events + .iter() + .filter(|event| match event { + HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption", + HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing", + HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption", + HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange", + HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure", + HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch", + HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption", + HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption", + }) + .collect() + } +} + +impl Default for HealEventHandler { + fn default() -> Self { + Self::new(1000) + } +} diff --git a/crates/ahm/src/heal/manager.rs b/crates/ahm/src/heal/manager.rs new file mode 100644 index 000000000..044289e16 --- /dev/null +++ b/crates/ahm/src/heal/manager.rs @@ -0,0 +1,422 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use crate::heal::{ + progress::{HealProgress, HealStatistics}, + storage::HealStorageAPI, + task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType}, +}; +use rustfs_ecstore::disk::DiskAPI; +use rustfs_ecstore::disk::error::DiskError; +use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP; +use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tokio::{ + sync::{Mutex, RwLock}, + time::interval, +}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn}; + +/// Heal config +#[derive(Debug, Clone)] +pub struct HealConfig { + /// Whether to enable auto heal + pub enable_auto_heal: bool, + /// Heal interval + pub heal_interval: Duration, + /// Maximum concurrent heal tasks + pub max_concurrent_heals: usize, + /// Task timeout + pub task_timeout: Duration, + /// Queue size + pub queue_size: usize, +} + +impl Default for HealConfig { + fn default() -> Self { + Self { + enable_auto_heal: true, + heal_interval: Duration::from_secs(10), // 10 seconds + max_concurrent_heals: 4, + task_timeout: Duration::from_secs(300), // 5 minutes + queue_size: 1000, + } + } +} + +/// Heal state +#[derive(Debug, Default)] +pub struct HealState { + /// Whether running + pub is_running: bool, + /// Current heal cycle + pub current_cycle: u64, + /// Last heal time + pub last_heal_time: Option, + /// Total healed objects + pub total_healed_objects: u64, + /// Total heal failures + pub total_heal_failures: u64, + /// Current active heal tasks + pub active_heal_count: usize, +} + +/// Heal manager +pub struct HealManager { + /// Heal config + config: Arc>, + /// Heal state + state: Arc>, + /// Active heal tasks + active_heals: Arc>>>, + /// Heal queue + heal_queue: Arc>>, + /// Storage layer interface + storage: Arc, + /// Cancel token + cancel_token: CancellationToken, + /// Statistics + statistics: Arc>, +} + +impl HealManager { + /// Create new HealManager + pub fn new(storage: Arc, config: Option) -> Self { + let config = config.unwrap_or_default(); + Self { + config: Arc::new(RwLock::new(config)), + state: Arc::new(RwLock::new(HealState::default())), + active_heals: Arc::new(Mutex::new(HashMap::new())), + heal_queue: Arc::new(Mutex::new(VecDeque::new())), + storage, + cancel_token: CancellationToken::new(), + statistics: Arc::new(RwLock::new(HealStatistics::new())), + } + } + + /// Start HealManager + pub async fn start(&self) -> Result<()> { + let mut state = self.state.write().await; + if state.is_running { + warn!("HealManager is already running"); + return Ok(()); + } + state.is_running = true; + drop(state); + + info!("Starting HealManager"); + + // start scheduler + self.start_scheduler().await?; + + // start auto disk scanner + self.start_auto_disk_scanner().await?; + + info!("HealManager started successfully"); + Ok(()) + } + + /// Stop HealManager + pub async fn stop(&self) -> Result<()> { + info!("Stopping HealManager"); + + // cancel all tasks + self.cancel_token.cancel(); + + // wait for all tasks to complete + let mut active_heals = self.active_heals.lock().await; + for task in active_heals.values() { + if let Err(e) = task.cancel().await { + warn!("Failed to cancel task {}: {}", task.id, e); + } + } + active_heals.clear(); + + // update state + let mut state = self.state.write().await; + state.is_running = false; + + info!("HealManager stopped successfully"); + Ok(()) + } + + /// Submit heal request + pub async fn submit_heal_request(&self, request: HealRequest) -> Result { + let config = self.config.read().await; + let mut queue = self.heal_queue.lock().await; + + if queue.len() >= config.queue_size { + return Err(Error::ConfigurationError { + message: "Heal queue is full".to_string(), + }); + } + + let request_id = request.id.clone(); + queue.push_back(request); + drop(queue); + + info!("Submitted heal request: {}", request_id); + Ok(request_id) + } + + /// Get task status + pub async fn get_task_status(&self, task_id: &str) -> Result { + let active_heals = self.active_heals.lock().await; + if let Some(task) = active_heals.get(task_id) { + Ok(task.get_status().await) + } else { + Err(Error::TaskNotFound { + task_id: task_id.to_string(), + }) + } + } + + /// Get task progress + pub async fn get_active_tasks_count(&self) -> usize { + self.active_heals.lock().await.len() + } + + pub async fn get_task_progress(&self, task_id: &str) -> Result { + let active_heals = self.active_heals.lock().await; + if let Some(task) = active_heals.get(task_id) { + Ok(task.get_progress().await) + } else { + Err(Error::TaskNotFound { + task_id: task_id.to_string(), + }) + } + } + + /// Cancel task + pub async fn cancel_task(&self, task_id: &str) -> Result<()> { + let mut active_heals = self.active_heals.lock().await; + if let Some(task) = active_heals.get(task_id) { + task.cancel().await?; + active_heals.remove(task_id); + info!("Cancelled heal task: {}", task_id); + Ok(()) + } else { + Err(Error::TaskNotFound { + task_id: task_id.to_string(), + }) + } + } + + /// Get statistics + pub async fn get_statistics(&self) -> HealStatistics { + self.statistics.read().await.clone() + } + + /// Get active task count + pub async fn get_active_task_count(&self) -> usize { + let active_heals = self.active_heals.lock().await; + active_heals.len() + } + + /// Get queue length + pub async fn get_queue_length(&self) -> usize { + let queue = self.heal_queue.lock().await; + queue.len() + } + + /// Start scheduler + async fn start_scheduler(&self) -> Result<()> { + let config = self.config.clone(); + let heal_queue = self.heal_queue.clone(); + let active_heals = self.active_heals.clone(); + let cancel_token = self.cancel_token.clone(); + let statistics = self.statistics.clone(); + let storage = self.storage.clone(); + + tokio::spawn(async move { + let mut interval = interval(config.read().await.heal_interval); + + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + info!("Heal scheduler received shutdown signal"); + break; + } + _ = interval.tick() => { + Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage).await; + } + } + } + }); + + Ok(()) + } + + /// Start background task to auto scan local disks and enqueue erasure set heal requests + async fn start_auto_disk_scanner(&self) -> Result<()> { + let config = self.config.clone(); + let heal_queue = self.heal_queue.clone(); + let active_heals = self.active_heals.clone(); + let cancel_token = self.cancel_token.clone(); + let storage = self.storage.clone(); + + tokio::spawn(async move { + let mut interval = interval(config.read().await.heal_interval); + + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + info!("Auto disk scanner received shutdown signal"); + break; + } + _ = interval.tick() => { + // Build list of endpoints that need healing + let mut endpoints = Vec::new(); + for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() { + if let Some(disk) = disk_opt { + // detect unformatted disk via get_disk_id() + if let Err(err) = disk.get_disk_id().await { + if err == DiskError::UnformattedDisk { + endpoints.push(disk.endpoint()); + continue; + } + } + } + } + + if endpoints.is_empty() { + continue; + } + + // Get bucket list for erasure set healing + let buckets = match storage.list_buckets().await { + Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), + Err(e) => { + error!("Failed to get bucket list for auto healing: {}", e); + continue; + } + }; + + // Create erasure set heal requests for each endpoint + for ep in endpoints { + // skip if already queued or healing + let mut skip = false; + { + let queue = heal_queue.lock().await; + if queue.iter().any(|req| matches!(&req.heal_type, crate::heal::task::HealType::ErasureSet { set_disk_id, .. } if set_disk_id == &format!("{}_{}", ep.pool_idx, ep.set_idx))) { + skip = true; + } + } + if !skip { + let active = active_heals.lock().await; + if active.values().any(|task| matches!(&task.heal_type, crate::heal::task::HealType::ErasureSet { set_disk_id, .. } if set_disk_id == &format!("{}_{}", ep.pool_idx, ep.set_idx))) { + skip = true; + } + } + + if skip { + continue; + } + + // enqueue erasure set heal request for this disk + let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx); + let req = HealRequest::new( + HealType::ErasureSet { + buckets: buckets.clone(), + set_disk_id: set_disk_id.clone() + }, + HealOptions::default(), + HealPriority::Normal, + ); + let mut queue = heal_queue.lock().await; + queue.push_back(req); + info!("Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id); + } + } + } + } + }); + Ok(()) + } + + /// Process heal queue + async fn process_heal_queue( + heal_queue: &Arc>>, + active_heals: &Arc>>>, + config: &Arc>, + statistics: &Arc>, + storage: &Arc, + ) { + let config = config.read().await; + let mut active_heals_guard = active_heals.lock().await; + + // check if new heal tasks can be started + if active_heals_guard.len() >= config.max_concurrent_heals { + return; + } + + let mut queue = heal_queue.lock().await; + if let Some(request) = queue.pop_front() { + let task = Arc::new(HealTask::from_request(request, storage.clone())); + let task_id = task.id.clone(); + active_heals_guard.insert(task_id.clone(), task.clone()); + drop(active_heals_guard); + let active_heals_clone = active_heals.clone(); + let statistics_clone = statistics.clone(); + + // start heal task + tokio::spawn(async move { + info!("Starting heal task: {}", task_id); + let result = task.execute().await; + match result { + Ok(_) => { + info!("Heal task completed successfully: {}", task_id); + } + Err(e) => { + error!("Heal task failed: {} - {}", task_id, e); + } + } + let mut active_heals_guard = active_heals_clone.lock().await; + if let Some(completed_task) = active_heals_guard.remove(&task_id) { + // update statistics + let mut stats = statistics_clone.write().await; + match completed_task.get_status().await { + HealTaskStatus::Completed => { + stats.update_task_completion(true); + } + _ => { + stats.update_task_completion(false); + } + } + stats.update_running_tasks(active_heals_guard.len() as u64); + } + }); + + // update statistics + let mut stats = statistics.write().await; + stats.total_tasks += 1; + } + } +} + +impl std::fmt::Debug for HealManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HealManager") + .field("config", &"") + .field("state", &"") + .field("active_heals_count", &"") + .field("queue_length", &"") + .finish() + } +} diff --git a/crates/ecstore/src/heal/mod.rs b/crates/ahm/src/heal/mod.rs similarity index 60% rename from crates/ecstore/src/heal/mod.rs rename to crates/ahm/src/heal/mod.rs index a92bfc33b..aaa57d7ab 100644 --- a/crates/ecstore/src/heal/mod.rs +++ b/crates/ahm/src/heal/mod.rs @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod background_heal_ops; -pub mod data_scanner; -pub mod data_scanner_metric; -pub mod data_usage; -pub mod data_usage_cache; -pub mod error; -pub mod heal_commands; -pub mod heal_ops; -pub mod mrf; +pub mod channel; +pub mod erasure_healer; +pub mod event; +pub mod manager; +pub mod progress; +pub mod resume; +pub mod storage; +pub mod task; + +pub use erasure_healer::ErasureSetHealer; +pub use manager::HealManager; +pub use resume::{CheckpointManager, ResumeCheckpoint, ResumeManager, ResumeState, ResumeUtils}; +pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType}; diff --git a/crates/ahm/src/heal/progress.rs b/crates/ahm/src/heal/progress.rs new file mode 100644 index 000000000..f590a5a56 --- /dev/null +++ b/crates/ahm/src/heal/progress.rs @@ -0,0 +1,148 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; +use std::time::SystemTime; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct HealProgress { + /// Objects scanned + pub objects_scanned: u64, + /// Objects healed + pub objects_healed: u64, + /// Objects failed + pub objects_failed: u64, + /// Bytes processed + pub bytes_processed: u64, + /// Current object + pub current_object: Option, + /// Progress percentage + pub progress_percentage: f64, + /// Start time + pub start_time: Option, + /// Last update time + pub last_update_time: Option, + /// Estimated completion time + pub estimated_completion_time: Option, +} + +impl HealProgress { + pub fn new() -> Self { + Self { + start_time: Some(SystemTime::now()), + last_update_time: Some(SystemTime::now()), + ..Default::default() + } + } + + pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) { + self.objects_scanned = scanned; + self.objects_healed = healed; + self.objects_failed = failed; + self.bytes_processed = bytes; + self.last_update_time = Some(SystemTime::now()); + + // calculate progress percentage + let total = scanned + healed + failed; + if total > 0 { + self.progress_percentage = (healed as f64 / total as f64) * 100.0; + } + } + + pub fn set_current_object(&mut self, object: Option) { + self.current_object = object; + self.last_update_time = Some(SystemTime::now()); + } + + pub fn is_completed(&self) -> bool { + self.progress_percentage >= 100.0 + || self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned + } + + pub fn get_success_rate(&self) -> f64 { + let total = self.objects_healed + self.objects_failed; + if total > 0 { + (self.objects_healed as f64 / total as f64) * 100.0 + } else { + 0.0 + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealStatistics { + /// Total heal tasks + pub total_tasks: u64, + /// Successful tasks + pub successful_tasks: u64, + /// Failed tasks + pub failed_tasks: u64, + /// Running tasks + pub running_tasks: u64, + /// Total healed objects + pub total_objects_healed: u64, + /// Total healed bytes + pub total_bytes_healed: u64, + /// Last update time + pub last_update_time: SystemTime, +} + +impl Default for HealStatistics { + fn default() -> Self { + Self::new() + } +} + +impl HealStatistics { + pub fn new() -> Self { + Self { + total_tasks: 0, + successful_tasks: 0, + failed_tasks: 0, + running_tasks: 0, + total_objects_healed: 0, + total_bytes_healed: 0, + last_update_time: SystemTime::now(), + } + } + + pub fn update_task_completion(&mut self, success: bool) { + if success { + self.successful_tasks += 1; + } else { + self.failed_tasks += 1; + } + self.last_update_time = SystemTime::now(); + } + + pub fn update_running_tasks(&mut self, count: u64) { + self.running_tasks = count; + self.last_update_time = SystemTime::now(); + } + + pub fn add_healed_objects(&mut self, count: u64, bytes: u64) { + self.total_objects_healed += count; + self.total_bytes_healed += bytes; + self.last_update_time = SystemTime::now(); + } + + pub fn get_success_rate(&self) -> f64 { + let total = self.successful_tasks + self.failed_tasks; + if total > 0 { + (self.successful_tasks as f64 / total as f64) * 100.0 + } else { + 0.0 + } + } +} diff --git a/crates/ahm/src/heal/resume.rs b/crates/ahm/src/heal/resume.rs new file mode 100644 index 000000000..cf9d5cae5 --- /dev/null +++ b/crates/ahm/src/heal/resume.rs @@ -0,0 +1,696 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use rustfs_ecstore::disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET}; +use serde::{Deserialize, Serialize}; +use std::path::Path; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::RwLock; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +/// resume state file constants +const RESUME_STATE_FILE: &str = "ahm_resume_state.json"; +const RESUME_PROGRESS_FILE: &str = "ahm_progress.json"; +const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json"; + +/// resume state +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResumeState { + /// task id + pub task_id: String, + /// task type + pub task_type: String, + /// start time + pub start_time: u64, + /// last update time + pub last_update: u64, + /// completed + pub completed: bool, + /// total objects + pub total_objects: u64, + /// processed objects + pub processed_objects: u64, + /// successful objects + pub successful_objects: u64, + /// failed objects + pub failed_objects: u64, + /// skipped objects + pub skipped_objects: u64, + /// current bucket + pub current_bucket: Option, + /// current object + pub current_object: Option, + /// completed buckets + pub completed_buckets: Vec, + /// pending buckets + pub pending_buckets: Vec, + /// error message + pub error_message: Option, + /// retry count + pub retry_count: u32, + /// max retries + pub max_retries: u32, +} + +impl ResumeState { + pub fn new(task_id: String, task_type: String, buckets: Vec) -> Self { + Self { + task_id, + task_type, + start_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(), + last_update: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(), + completed: false, + total_objects: 0, + processed_objects: 0, + successful_objects: 0, + failed_objects: 0, + skipped_objects: 0, + current_bucket: None, + current_object: None, + completed_buckets: Vec::new(), + pending_buckets: buckets, + error_message: None, + retry_count: 0, + max_retries: 3, + } + } + + pub fn update_progress(&mut self, processed: u64, successful: u64, failed: u64, skipped: u64) { + self.processed_objects = processed; + self.successful_objects = successful; + self.failed_objects = failed; + self.skipped_objects = skipped; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn set_current_item(&mut self, bucket: Option, object: Option) { + self.current_bucket = bucket; + self.current_object = object; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn complete_bucket(&mut self, bucket: &str) { + if !self.completed_buckets.contains(&bucket.to_string()) { + self.completed_buckets.push(bucket.to_string()); + } + if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) { + self.pending_buckets.remove(pos); + } + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn mark_completed(&mut self) { + self.completed = true; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn set_error(&mut self, error: String) { + self.error_message = Some(error); + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn increment_retry(&mut self) { + self.retry_count += 1; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn can_retry(&self) -> bool { + self.retry_count < self.max_retries + } + + pub fn get_progress_percentage(&self) -> f64 { + if self.total_objects == 0 { + return 0.0; + } + (self.processed_objects as f64 / self.total_objects as f64) * 100.0 + } + + pub fn get_success_rate(&self) -> f64 { + let total = self.successful_objects + self.failed_objects; + if total == 0 { + return 0.0; + } + (self.successful_objects as f64 / total as f64) * 100.0 + } +} + +/// resume manager +pub struct ResumeManager { + disk: DiskStore, + state: Arc>, +} + +impl ResumeManager { + /// create new resume manager + pub async fn new(disk: DiskStore, task_id: String, task_type: String, buckets: Vec) -> Result { + let state = ResumeState::new(task_id, task_type, buckets); + let manager = Self { + disk, + state: Arc::new(RwLock::new(state)), + }; + + // save initial state + manager.save_state().await?; + Ok(manager) + } + + /// load resume state from disk + pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { + let state_data = Self::read_state_file(&disk, task_id).await?; + let state: ResumeState = serde_json::from_slice(&state_data).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to deserialize resume state: {e}"), + })?; + + Ok(Self { + disk, + state: Arc::new(RwLock::new(state)), + }) + } + + /// check if resume state exists + pub async fn has_resume_state(disk: &DiskStore, task_id: &str) -> bool { + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + match disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await { + Ok(data) => !data.is_empty(), + Err(_) => false, + } + } + + /// get current state + pub async fn get_state(&self) -> ResumeState { + self.state.read().await.clone() + } + + /// update progress + pub async fn update_progress(&self, processed: u64, successful: u64, failed: u64, skipped: u64) -> Result<()> { + let mut state = self.state.write().await; + state.update_progress(processed, successful, failed, skipped); + drop(state); + self.save_state().await + } + + /// set current item + pub async fn set_current_item(&self, bucket: Option, object: Option) -> Result<()> { + let mut state = self.state.write().await; + state.set_current_item(bucket, object); + drop(state); + self.save_state().await + } + + /// complete bucket + pub async fn complete_bucket(&self, bucket: &str) -> Result<()> { + let mut state = self.state.write().await; + state.complete_bucket(bucket); + drop(state); + self.save_state().await + } + + /// mark task completed + pub async fn mark_completed(&self) -> Result<()> { + let mut state = self.state.write().await; + state.mark_completed(); + drop(state); + self.save_state().await + } + + /// set error message + pub async fn set_error(&self, error: String) -> Result<()> { + let mut state = self.state.write().await; + state.set_error(error); + drop(state); + self.save_state().await + } + + /// increment retry count + pub async fn increment_retry(&self) -> Result<()> { + let mut state = self.state.write().await; + state.increment_retry(); + drop(state); + self.save_state().await + } + + /// cleanup resume state + pub async fn cleanup(&self) -> Result<()> { + let state = self.state.read().await; + let task_id = &state.task_id; + + // delete state files + let state_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + let progress_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_PROGRESS_FILE}")); + let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + + // ignore delete errors, files may not exist + let _ = self + .disk + .delete(RUSTFS_META_BUCKET, state_file.to_str().unwrap(), Default::default()) + .await; + let _ = self + .disk + .delete(RUSTFS_META_BUCKET, progress_file.to_str().unwrap(), Default::default()) + .await; + let _ = self + .disk + .delete(RUSTFS_META_BUCKET, checkpoint_file.to_str().unwrap(), Default::default()) + .await; + + info!("Cleaned up resume state for task: {}", task_id); + Ok(()) + } + + /// save state to disk + async fn save_state(&self) -> Result<()> { + let state = self.state.read().await; + let state_data = serde_json::to_vec(&*state).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize resume state: {e}"), + })?; + + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", state.task_id, RESUME_STATE_FILE)); + + self.disk + .write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), state_data.into()) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to save resume state: {e}"), + })?; + + debug!("Saved resume state for task: {}", state.task_id); + Ok(()) + } + + /// read state file from disk + async fn read_state_file(disk: &DiskStore, task_id: &str) -> Result> { + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}")); + + disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()) + .await + .map(|bytes| bytes.to_vec()) + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to read resume state file: {e}"), + }) + } +} + +/// resume checkpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResumeCheckpoint { + /// task id + pub task_id: String, + /// checkpoint time + pub checkpoint_time: u64, + /// current bucket index + pub current_bucket_index: usize, + /// current object index + pub current_object_index: usize, + /// processed objects + pub processed_objects: Vec, + /// failed objects + pub failed_objects: Vec, + /// skipped objects + pub skipped_objects: Vec, +} + +impl ResumeCheckpoint { + pub fn new(task_id: String) -> Self { + Self { + task_id, + checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(), + current_bucket_index: 0, + current_object_index: 0, + processed_objects: Vec::new(), + failed_objects: Vec::new(), + skipped_objects: Vec::new(), + } + } + + pub fn update_position(&mut self, bucket_index: usize, object_index: usize) { + self.current_bucket_index = bucket_index; + self.current_object_index = object_index; + self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + } + + pub fn add_processed_object(&mut self, object: String) { + if !self.processed_objects.contains(&object) { + self.processed_objects.push(object); + } + } + + pub fn add_failed_object(&mut self, object: String) { + if !self.failed_objects.contains(&object) { + self.failed_objects.push(object); + } + } + + pub fn add_skipped_object(&mut self, object: String) { + if !self.skipped_objects.contains(&object) { + self.skipped_objects.push(object); + } + } +} + +/// resume checkpoint manager +pub struct CheckpointManager { + disk: DiskStore, + checkpoint: Arc>, +} + +impl CheckpointManager { + /// create new checkpoint manager + pub async fn new(disk: DiskStore, task_id: String) -> Result { + let checkpoint = ResumeCheckpoint::new(task_id); + let manager = Self { + disk, + checkpoint: Arc::new(RwLock::new(checkpoint)), + }; + + // save initial checkpoint + manager.save_checkpoint().await?; + Ok(manager) + } + + /// load checkpoint from disk + pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { + let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?; + let checkpoint: ResumeCheckpoint = serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to deserialize checkpoint: {e}"), + })?; + + Ok(Self { + disk, + checkpoint: Arc::new(RwLock::new(checkpoint)), + }) + } + + /// check if checkpoint exists + pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool { + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + match disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await { + Ok(data) => !data.is_empty(), + Err(_) => false, + } + } + + /// get current checkpoint + pub async fn get_checkpoint(&self) -> ResumeCheckpoint { + self.checkpoint.read().await.clone() + } + + /// update position + pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.update_position(bucket_index, object_index); + drop(checkpoint); + self.save_checkpoint().await + } + + /// add processed object + pub async fn add_processed_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_processed_object(object); + drop(checkpoint); + self.save_checkpoint().await + } + + /// add failed object + pub async fn add_failed_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_failed_object(object); + drop(checkpoint); + self.save_checkpoint().await + } + + /// add skipped object + pub async fn add_skipped_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_skipped_object(object); + drop(checkpoint); + self.save_checkpoint().await + } + + /// cleanup checkpoint + pub async fn cleanup(&self) -> Result<()> { + let checkpoint = self.checkpoint.read().await; + let task_id = &checkpoint.task_id; + + let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + let _ = self + .disk + .delete(RUSTFS_META_BUCKET, checkpoint_file.to_str().unwrap(), Default::default()) + .await; + + info!("Cleaned up checkpoint for task: {}", task_id); + Ok(()) + } + + /// save checkpoint to disk + async fn save_checkpoint(&self) -> Result<()> { + let checkpoint = self.checkpoint.read().await; + let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize checkpoint: {e}"), + })?; + + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE)); + + self.disk + .write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), checkpoint_data.into()) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to save checkpoint: {e}"), + })?; + + debug!("Saved checkpoint for task: {}", checkpoint.task_id); + Ok(()) + } + + /// read checkpoint file from disk + async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result> { + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + + disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()) + .await + .map(|bytes| bytes.to_vec()) + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to read checkpoint file: {e}"), + }) + } +} + +/// resume utils +pub struct ResumeUtils; + +impl ResumeUtils { + /// generate unique task id + pub fn generate_task_id() -> String { + Uuid::new_v4().to_string() + } + + /// check if task can be resumed + pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool { + ResumeManager::has_resume_state(disk, task_id).await + } + + /// get all resumable task ids + pub async fn get_resumable_tasks(disk: &DiskStore) -> Result> { + // List all files in the buckets metadata directory + let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await { + Ok(entries) => entries, + Err(e) => { + debug!("Failed to list resume state files: {}", e); + return Ok(Vec::new()); + } + }; + + let mut task_ids = Vec::new(); + + // Filter files that end with ahm_resume_state.json and extract task IDs + for entry in entries { + if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) { + // Extract task ID from filename: {task_id}_ahm_resume_state.json + if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) { + if !task_id.is_empty() { + task_ids.push(task_id.to_string()); + } + } + } + } + + debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids); + Ok(task_ids) + } + + /// cleanup expired resume states + pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> { + let task_ids = Self::get_resumable_tasks(disk).await?; + let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + + for task_id in task_ids { + if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await { + let state = resume_manager.get_state().await; + let age_hours = (current_time - state.last_update) / 3600; + + if age_hours > max_age_hours { + info!("Cleaning up expired resume state for task: {} (age: {} hours)", task_id, age_hours); + if let Err(e) = resume_manager.cleanup().await { + warn!("Failed to cleanup expired resume state for task {}: {}", task_id, e); + } + } + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_resume_state_creation() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; + let state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), buckets); + + assert_eq!(state.task_id, task_id); + assert_eq!(state.task_type, "erasure_set"); + assert!(!state.completed); + assert_eq!(state.processed_objects, 0); + assert_eq!(state.pending_buckets.len(), 2); + } + + #[tokio::test] + async fn test_resume_state_progress() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string()]; + let mut state = ResumeState::new(task_id, "erasure_set".to_string(), buckets); + + state.update_progress(10, 8, 1, 1); + assert_eq!(state.processed_objects, 10); + assert_eq!(state.successful_objects, 8); + assert_eq!(state.failed_objects, 1); + assert_eq!(state.skipped_objects, 1); + + let progress = state.get_progress_percentage(); + assert_eq!(progress, 0.0); // total_objects is 0 + + state.total_objects = 100; + let progress = state.get_progress_percentage(); + assert_eq!(progress, 10.0); + } + + #[tokio::test] + async fn test_resume_state_bucket_completion() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; + let mut state = ResumeState::new(task_id, "erasure_set".to_string(), buckets); + + assert_eq!(state.pending_buckets.len(), 2); + assert_eq!(state.completed_buckets.len(), 0); + + state.complete_bucket("bucket1"); + assert_eq!(state.pending_buckets.len(), 1); + assert_eq!(state.completed_buckets.len(), 1); + assert!(state.completed_buckets.contains(&"bucket1".to_string())); + } + + #[tokio::test] + async fn test_resume_utils() { + let task_id1 = ResumeUtils::generate_task_id(); + let task_id2 = ResumeUtils::generate_task_id(); + + assert_ne!(task_id1, task_id2); + assert_eq!(task_id1.len(), 36); // UUID length + assert_eq!(task_id2.len(), 36); + } + + #[tokio::test] + async fn test_get_resumable_tasks_integration() { + use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk}; + use tempfile::TempDir; + + // Create a temporary directory for testing + let temp_dir = TempDir::new().unwrap(); + let disk_path = temp_dir.path().join("test_disk"); + std::fs::create_dir_all(&disk_path).unwrap(); + + // Create a local disk for testing + let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); + let disk_option = DiskOption { + cleanup: false, + health_check: false, + }; + let disk = new_disk(&endpoint, &disk_option).await.unwrap(); + + // Create necessary directories first (ignore if already exist) + let _ = disk.make_volume(RUSTFS_META_BUCKET).await; + let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; + + // Create some test resume state files + let task_ids = vec![ + "test-task-1".to_string(), + "test-task-2".to_string(), + "test-task-3".to_string(), + ]; + + // Save resume state files for each task + for task_id in &task_ids { + let state = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + vec!["bucket1".to_string(), "bucket2".to_string()], + ); + + let state_data = serde_json::to_vec(&state).unwrap(); + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + + disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) + .await + .unwrap(); + } + + // Also create some non-resume state files to test filtering + let non_resume_files = vec![ + "other_file.txt", + "task4_ahm_checkpoint.json", + "task5_ahm_progress.json", + "_ahm_resume_state.json", // Invalid: empty task ID + ]; + + for file_name in non_resume_files { + let file_path = format!("{BUCKET_META_PREFIX}/{file_name}"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, b"test data".to_vec().into()) + .await + .unwrap(); + } + + // Now call get_resumable_tasks to see if it finds the correct files + let found_task_ids = ResumeUtils::get_resumable_tasks(&disk).await.unwrap(); + + // Verify that only the valid resume state files are found + assert_eq!(found_task_ids.len(), 3); + for task_id in &task_ids { + assert!(found_task_ids.contains(task_id), "Task ID {task_id} not found"); + } + + // Verify that invalid files are not included + assert!(!found_task_ids.contains(&"".to_string())); + assert!(!found_task_ids.contains(&"task4".to_string())); + assert!(!found_task_ids.contains(&"task5".to_string())); + + // Clean up + temp_dir.close().unwrap(); + } +} diff --git a/crates/ahm/src/heal/storage.rs b/crates/ahm/src/heal/storage.rs new file mode 100644 index 000000000..43ca41638 --- /dev/null +++ b/crates/ahm/src/heal/storage.rs @@ -0,0 +1,506 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use async_trait::async_trait; +use rustfs_common::heal_channel::{HealOpts, HealScanMode}; +use rustfs_ecstore::{ + disk::{DiskStore, endpoint::Endpoint}, + store::ECStore, + store_api::{BucketInfo, ObjectIO, StorageAPI}, +}; +use rustfs_madmin::heal_commands::HealResultItem; +use std::sync::Arc; +use tracing::{debug, error, info, warn}; + +/// Disk status for heal operations +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiskStatus { + /// Ok + Ok, + /// Offline + Offline, + /// Corrupt + Corrupt, + /// Missing + Missing, + /// Permission denied + PermissionDenied, + /// Faulty + Faulty, + /// Root mount + RootMount, + /// Unknown + Unknown, + /// Unformatted + Unformatted, +} + +/// Heal storage layer interface +#[async_trait] +pub trait HealStorageAPI: Send + Sync { + /// Get object meta + async fn get_object_meta(&self, bucket: &str, object: &str) -> Result>; + + /// Get object data + async fn get_object_data(&self, bucket: &str, object: &str) -> Result>>; + + /// Put object data + async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>; + + /// Delete object + async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>; + + /// Check object integrity + async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result; + + /// EC decode rebuild + async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result>; + + /// Get disk status + async fn get_disk_status(&self, endpoint: &Endpoint) -> Result; + + /// Format disk + async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>; + + /// Get bucket info + async fn get_bucket_info(&self, bucket: &str) -> Result>; + + /// Fix bucket metadata + async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; + + /// Get all buckets + async fn list_buckets(&self) -> Result>; + + /// Check object exists + async fn object_exists(&self, bucket: &str, object: &str) -> Result; + + /// Get object size + async fn get_object_size(&self, bucket: &str, object: &str) -> Result>; + + /// Get object checksum + async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result>; + + /// Heal object using ecstore + async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: Option<&str>, + opts: &HealOpts, + ) -> Result<(HealResultItem, Option)>; + + /// Heal bucket using ecstore + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; + + /// Heal format using ecstore + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; + + /// List objects for healing + async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result>; + + /// Get disk for resume functionality + async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result; +} + +/// ECStore Heal storage layer implementation +pub struct ECStoreHealStorage { + ecstore: Arc, +} + +impl ECStoreHealStorage { + pub fn new(ecstore: Arc) -> Self { + Self { ecstore } + } +} + +#[async_trait] +impl HealStorageAPI for ECStoreHealStorage { + async fn get_object_meta(&self, bucket: &str, object: &str) -> Result> { + debug!("Getting object meta: {}/{}", bucket, object); + + match self.ecstore.get_object_info(bucket, object, &Default::default()).await { + Ok(info) => Ok(Some(info)), + Err(e) => { + error!("Failed to get object meta: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + } + } + + async fn get_object_data(&self, bucket: &str, object: &str) -> Result>> { + debug!("Getting object data: {}/{}", bucket, object); + + match (*self.ecstore) + .get_object_reader(bucket, object, None, Default::default(), &Default::default()) + .await + { + Ok(mut reader) => match reader.read_all().await { + Ok(data) => Ok(Some(data)), + Err(e) => { + error!("Failed to read object data: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + }, + Err(e) => { + error!("Failed to get object: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + } + } + + async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> { + debug!("Putting object data: {}/{} ({} bytes)", bucket, object, data.len()); + + let mut reader = rustfs_ecstore::store_api::PutObjReader::from_vec(data.to_vec()); + match (*self.ecstore) + .put_object(bucket, object, &mut reader, &Default::default()) + .await + { + Ok(_) => { + info!("Successfully put object: {}/{}", bucket, object); + Ok(()) + } + Err(e) => { + error!("Failed to put object: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + } + } + + async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> { + debug!("Deleting object: {}/{}", bucket, object); + + match self.ecstore.delete_object(bucket, object, Default::default()).await { + Ok(_) => { + info!("Successfully deleted object: {}/{}", bucket, object); + Ok(()) + } + Err(e) => { + error!("Failed to delete object: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + } + } + + async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result { + debug!("Verifying object integrity: {}/{}", bucket, object); + + // Try to get object info and data to verify integrity + match self.get_object_meta(bucket, object).await? { + Some(obj_info) => { + // Check if object has valid metadata + if obj_info.size < 0 { + warn!("Object has invalid size: {}/{}", bucket, object); + return Ok(false); + } + + // Try to read object data to verify it's accessible + match self.get_object_data(bucket, object).await { + Ok(Some(_)) => { + info!("Object integrity check passed: {}/{}", bucket, object); + Ok(true) + } + Ok(None) => { + warn!("Object data not found: {}/{}", bucket, object); + Ok(false) + } + Err(_) => { + warn!("Object data read failed: {}/{}", bucket, object); + Ok(false) + } + } + } + None => { + warn!("Object metadata not found: {}/{}", bucket, object); + Ok(false) + } + } + } + + async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result> { + debug!("EC decode rebuild: {}/{}", bucket, object); + + // Use ecstore's heal_object to rebuild the object + let heal_opts = HealOpts { + recursive: false, + dry_run: false, + remove: false, + recreate: true, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: None, + set: None, + }; + + match self.heal_object(bucket, object, None, &heal_opts).await { + Ok((_result, error)) => { + if error.is_some() { + return Err(Error::TaskExecutionFailed { + message: format!("Heal failed: {error:?}"), + }); + } + + // After healing, try to read the object data + match self.get_object_data(bucket, object).await? { + Some(data) => { + info!("EC decode rebuild successful: {}/{} ({} bytes)", bucket, object, data.len()); + Ok(data) + } + None => { + error!("Object not found after heal: {}/{}", bucket, object); + Err(Error::TaskExecutionFailed { + message: format!("Object not found after heal: {bucket}/{object}"), + }) + } + } + } + Err(e) => { + error!("Heal operation failed: {}/{} - {}", bucket, object, e); + Err(e) + } + } + } + + async fn get_disk_status(&self, endpoint: &Endpoint) -> Result { + debug!("Getting disk status: {:?}", endpoint); + + // TODO: implement disk status check using ecstore + // For now, return Ok status + info!("Disk status check: {:?} - OK", endpoint); + Ok(DiskStatus::Ok) + } + + async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> { + debug!("Formatting disk: {:?}", endpoint); + + // Use ecstore's heal_format + match self.heal_format(false).await { + Ok((_, error)) => { + if error.is_some() { + return Err(Error::other(format!("Format failed: {error:?}"))); + } + info!("Successfully formatted disk: {:?}", endpoint); + Ok(()) + } + Err(e) => { + error!("Failed to format disk: {:?} - {}", endpoint, e); + Err(e) + } + } + } + + async fn get_bucket_info(&self, bucket: &str) -> Result> { + debug!("Getting bucket info: {}", bucket); + + match self.ecstore.get_bucket_info(bucket, &Default::default()).await { + Ok(info) => Ok(Some(info)), + Err(e) => { + error!("Failed to get bucket info: {} - {}", bucket, e); + Err(Error::other(e)) + } + } + } + + async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { + debug!("Healing bucket metadata: {}", bucket); + + let heal_opts = HealOpts { + recursive: true, + dry_run: false, + remove: false, + recreate: false, + scan_mode: HealScanMode::Normal, + update_parity: false, + no_lock: false, + pool: None, + set: None, + }; + + match self.heal_bucket(bucket, &heal_opts).await { + Ok(_) => { + info!("Successfully healed bucket metadata: {}", bucket); + Ok(()) + } + Err(e) => { + error!("Failed to heal bucket metadata: {} - {}", bucket, e); + Err(e) + } + } + } + + async fn list_buckets(&self) -> Result> { + debug!("Listing buckets"); + + match self.ecstore.list_bucket(&Default::default()).await { + Ok(buckets) => Ok(buckets), + Err(e) => { + error!("Failed to list buckets: {}", e); + Err(Error::other(e)) + } + } + } + + async fn object_exists(&self, bucket: &str, object: &str) -> Result { + debug!("Checking object exists: {}/{}", bucket, object); + + match self.get_object_meta(bucket, object).await { + Ok(Some(_)) => Ok(true), + Ok(None) => Ok(false), + Err(_) => Ok(false), + } + } + + async fn get_object_size(&self, bucket: &str, object: &str) -> Result> { + debug!("Getting object size: {}/{}", bucket, object); + + match self.get_object_meta(bucket, object).await { + Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + } + + async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result> { + debug!("Getting object checksum: {}/{}", bucket, object); + + match self.get_object_meta(bucket, object).await { + Ok(Some(obj_info)) => { + // Convert checksum bytes to hex string + let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::(); + Ok(Some(checksum)) + } + Ok(None) => Ok(None), + Err(e) => Err(e), + } + } + + async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: Option<&str>, + opts: &HealOpts, + ) -> Result<(HealResultItem, Option)> { + debug!("Healing object: {}/{}", bucket, object); + + let version_id_str = version_id.unwrap_or(""); + + match self.ecstore.heal_object(bucket, object, version_id_str, opts).await { + Ok((result, ecstore_error)) => { + let error = ecstore_error.map(Error::other); + info!("Heal object completed: {}/{} - result: {:?}, error: {:?}", bucket, object, result, error); + Ok((result, error)) + } + Err(e) => { + error!("Heal object failed: {}/{} - {}", bucket, object, e); + Err(Error::other(e)) + } + } + } + + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + debug!("Healing bucket: {}", bucket); + + match self.ecstore.heal_bucket(bucket, opts).await { + Ok(result) => { + info!("Heal bucket completed: {} - result: {:?}", bucket, result); + Ok(result) + } + Err(e) => { + error!("Heal bucket failed: {} - {}", bucket, e); + Err(Error::other(e)) + } + } + } + + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + debug!("Healing format (dry_run: {})", dry_run); + + match self.ecstore.heal_format(dry_run).await { + Ok((result, ecstore_error)) => { + let error = ecstore_error.map(Error::other); + info!("Heal format completed - result: {:?}, error: {:?}", result, error); + Ok((result, error)) + } + Err(e) => { + error!("Heal format failed: {}", e); + Err(Error::other(e)) + } + } + } + + async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result> { + debug!("Listing objects for heal: {}/{}", bucket, prefix); + + // Use list_objects_v2 to get objects + match self + .ecstore + .clone() + .list_objects_v2(bucket, prefix, None, None, 1000, false, None) + .await + { + Ok(list_info) => { + let objects: Vec = list_info.objects.into_iter().map(|obj| obj.name).collect(); + info!("Found {} objects for heal in {}/{}", objects.len(), bucket, prefix); + Ok(objects) + } + Err(e) => { + error!("Failed to list objects for heal: {}/{} - {}", bucket, prefix, e); + Err(Error::other(e)) + } + } + } + + async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result { + debug!("Getting disk for resume: {}", set_disk_id); + + // Parse set_disk_id to extract pool and set indices + // Format: "pool_{pool_idx}_set_{set_idx}" + let parts: Vec<&str> = set_disk_id.split('_').collect(); + if parts.len() != 4 || parts[0] != "pool" || parts[2] != "set" { + return Err(Error::TaskExecutionFailed { + message: format!("Invalid set_disk_id format: {set_disk_id}"), + }); + } + + let pool_idx: usize = parts[1].parse().map_err(|_| Error::TaskExecutionFailed { + message: format!("Invalid pool index in set_disk_id: {set_disk_id}"), + })?; + + let set_idx: usize = parts[3].parse().map_err(|_| Error::TaskExecutionFailed { + message: format!("Invalid set index in set_disk_id: {set_disk_id}"), + })?; + + // Get the first available disk from the set + let disks = self + .ecstore + .get_disks(pool_idx, set_idx) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to get disks for pool {pool_idx} set {set_idx}: {e}"), + })?; + + // Find the first available disk + if let Some(disk_store) = disks.into_iter().flatten().next() { + info!("Found disk for resume: {:?}", disk_store); + return Ok(disk_store); + } + + Err(Error::TaskExecutionFailed { + message: format!("No available disk found for set_disk_id: {set_disk_id}"), + }) + } +} diff --git a/crates/ahm/src/heal/task.rs b/crates/ahm/src/heal/task.rs new file mode 100644 index 000000000..cf929b44e --- /dev/null +++ b/crates/ahm/src/heal/task.rs @@ -0,0 +1,855 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use crate::heal::ErasureSetHealer; +use crate::heal::{progress::HealProgress, storage::HealStorageAPI}; +use rustfs_common::heal_channel::{HealOpts, HealScanMode}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use tokio::sync::RwLock; +use tracing::{error, info, warn}; +use uuid::Uuid; + +/// Heal type +#[derive(Debug, Clone)] +pub enum HealType { + /// Object heal + Object { + bucket: String, + object: String, + version_id: Option, + }, + /// Bucket heal + Bucket { bucket: String }, + /// Erasure Set heal (includes disk format repair) + ErasureSet { buckets: Vec, set_disk_id: String }, + /// Metadata heal + Metadata { bucket: String, object: String }, + /// MRF heal + MRF { meta_path: String }, + /// EC decode heal + ECDecode { + bucket: String, + object: String, + version_id: Option, + }, +} + +/// Heal priority +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum HealPriority { + /// Low priority + Low = 0, + /// Normal priority + Normal = 1, + /// High priority + High = 2, + /// Urgent priority + Urgent = 3, +} + +impl Default for HealPriority { + fn default() -> Self { + Self::Normal + } +} + +/// Heal options +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealOptions { + /// Scan mode + pub scan_mode: HealScanMode, + /// Whether to remove corrupted data + pub remove_corrupted: bool, + /// Whether to recreate + pub recreate_missing: bool, + /// Whether to update parity + pub update_parity: bool, + /// Whether to recursively process + pub recursive: bool, + /// Whether to dry run + pub dry_run: bool, + /// Timeout + pub timeout: Option, + /// pool index + pub pool_index: Option, + /// set index + pub set_index: Option, +} + +impl Default for HealOptions { + fn default() -> Self { + Self { + scan_mode: HealScanMode::Normal, + remove_corrupted: false, + recreate_missing: true, + update_parity: true, + recursive: false, + dry_run: false, + timeout: Some(Duration::from_secs(300)), // 5 minutes default timeout + pool_index: None, + set_index: None, + } + } +} + +/// Heal task status +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum HealTaskStatus { + /// Pending + Pending, + /// Running + Running, + /// Completed + Completed, + /// Failed + Failed { error: String }, + /// Cancelled + Cancelled, + /// Timeout + Timeout, +} + +/// Heal request +#[derive(Debug, Clone)] +pub struct HealRequest { + /// Request ID + pub id: String, + /// Heal type + pub heal_type: HealType, + /// Heal options + pub options: HealOptions, + /// Priority + pub priority: HealPriority, + /// Created time + pub created_at: SystemTime, +} + +impl HealRequest { + pub fn new(heal_type: HealType, options: HealOptions, priority: HealPriority) -> Self { + Self { + id: Uuid::new_v4().to_string(), + heal_type, + options, + priority, + created_at: SystemTime::now(), + } + } + + pub fn object(bucket: String, object: String, version_id: Option) -> Self { + Self::new( + HealType::Object { + bucket, + object, + version_id, + }, + HealOptions::default(), + HealPriority::Normal, + ) + } + + pub fn bucket(bucket: String) -> Self { + Self::new(HealType::Bucket { bucket }, HealOptions::default(), HealPriority::Normal) + } + + pub fn metadata(bucket: String, object: String) -> Self { + Self::new(HealType::Metadata { bucket, object }, HealOptions::default(), HealPriority::High) + } + + pub fn ec_decode(bucket: String, object: String, version_id: Option) -> Self { + Self::new( + HealType::ECDecode { + bucket, + object, + version_id, + }, + HealOptions::default(), + HealPriority::Urgent, + ) + } +} + +/// Heal task +pub struct HealTask { + /// Task ID + pub id: String, + /// Heal type + pub heal_type: HealType, + /// Heal options + pub options: HealOptions, + /// Task status + pub status: Arc>, + /// Progress tracking + pub progress: Arc>, + /// Created time + pub created_at: SystemTime, + /// Started time + pub started_at: Arc>>, + /// Completed time + pub completed_at: Arc>>, + /// Cancel token + pub cancel_token: tokio_util::sync::CancellationToken, + /// Storage layer interface + pub storage: Arc, +} + +impl HealTask { + pub fn from_request(request: HealRequest, storage: Arc) -> Self { + Self { + id: request.id, + heal_type: request.heal_type, + options: request.options, + status: Arc::new(RwLock::new(HealTaskStatus::Pending)), + progress: Arc::new(RwLock::new(HealProgress::new())), + created_at: request.created_at, + started_at: Arc::new(RwLock::new(None)), + completed_at: Arc::new(RwLock::new(None)), + cancel_token: tokio_util::sync::CancellationToken::new(), + storage, + } + } + + pub async fn execute(&self) -> Result<()> { + // update status to running + { + let mut status = self.status.write().await; + *status = HealTaskStatus::Running; + } + { + let mut started_at = self.started_at.write().await; + *started_at = Some(SystemTime::now()); + } + + info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type); + + let result = match &self.heal_type { + HealType::Object { + bucket, + object, + version_id, + } => self.heal_object(bucket, object, version_id.as_deref()).await, + HealType::Bucket { bucket } => self.heal_bucket(bucket).await, + + HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await, + HealType::MRF { meta_path } => self.heal_mrf(meta_path).await, + HealType::ECDecode { + bucket, + object, + version_id, + } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await, + HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await, + }; + + // update completed time and status + { + let mut completed_at = self.completed_at.write().await; + *completed_at = Some(SystemTime::now()); + } + + match &result { + Ok(_) => { + let mut status = self.status.write().await; + *status = HealTaskStatus::Completed; + info!("Heal task completed successfully: {}", self.id); + } + Err(e) => { + let mut status = self.status.write().await; + *status = HealTaskStatus::Failed { error: e.to_string() }; + error!("Heal task failed: {} with error: {}", self.id, e); + } + } + + result + } + + pub async fn cancel(&self) -> Result<()> { + self.cancel_token.cancel(); + let mut status = self.status.write().await; + *status = HealTaskStatus::Cancelled; + info!("Heal task cancelled: {}", self.id); + Ok(()) + } + + pub async fn get_status(&self) -> HealTaskStatus { + self.status.read().await.clone() + } + + pub async fn get_progress(&self) -> HealProgress { + self.progress.read().await.clone() + } + + // specific heal implementation method + async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { + info!("Healing object: {}/{}", bucket, object); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("{bucket}/{object}"))); + progress.update_progress(0, 4, 0, 0); // 开始heal，总共4个步骤 + } + + // Step 1: Check if object exists and get metadata + info!("Step 1: Checking object existence and metadata"); + let object_exists = self.storage.object_exists(bucket, object).await?; + if !object_exists { + warn!("Object does not exist: {}/{}", bucket, object); + if self.options.recreate_missing { + info!("Attempting to recreate missing object: {}/{}", bucket, object); + return self.recreate_missing_object(bucket, object, version_id).await; + } else { + return Err(Error::TaskExecutionFailed { + message: format!("Object not found: {bucket}/{object}"), + }); + } + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(1, 3, 0, 0); + } + + // Step 2: directly call ecstore to perform heal + info!("Step 2: Performing heal using ecstore"); + let heal_opts = HealOpts { + recursive: self.options.recursive, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: false, + pool: self.options.pool_index, + set: self.options.set_index, + }; + + match self.storage.heal_object(bucket, object, version_id, &heal_opts).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("Heal operation failed: {}/{} - {}", bucket, object, e); + + // If heal failed and remove_corrupted is enabled, delete the corrupted object + if self.options.remove_corrupted { + warn!("Removing corrupted object: {}/{}", bucket, object); + if !self.options.dry_run { + self.storage.delete_object(bucket, object).await?; + info!("Successfully deleted corrupted object: {}/{}", bucket, object); + } else { + info!("Dry run mode - would delete corrupted object: {}/{}", bucket, object); + } + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal object {bucket}/{object}: {e}"), + }); + } + + // Step 3: Verify heal result + info!("Step 3: Verifying heal result"); + let object_size = result.object_size as u64; + info!( + "Heal completed successfully: {}/{} ({} bytes, {} drives healed)", + bucket, + object, + object_size, + result.after.drives.len() + ); + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, object_size, object_size); + } + Ok(()) + } + Err(e) => { + error!("Heal operation failed: {}/{} - {}", bucket, object, e); + + // If heal failed and remove_corrupted is enabled, delete the corrupted object + if self.options.remove_corrupted { + warn!("Removing corrupted object: {}/{}", bucket, object); + if !self.options.dry_run { + self.storage.delete_object(bucket, object).await?; + info!("Successfully deleted corrupted object: {}/{}", bucket, object); + } else { + info!("Dry run mode - would delete corrupted object: {}/{}", bucket, object); + } + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal object {bucket}/{object}: {e}"), + }) + } + } + } + + /// Recreate missing object (for EC decode scenarios) + async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { + info!("Attempting to recreate missing object: {}/{}", bucket, object); + + // Use ecstore's heal_object with recreate option + let heal_opts = HealOpts { + recursive: false, + dry_run: self.options.dry_run, + remove: false, + recreate: true, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: None, + set: None, + }; + + match self.storage.heal_object(bucket, object, version_id, &heal_opts).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e); + return Err(Error::TaskExecutionFailed { + message: format!("Failed to recreate missing object {bucket}/{object}: {e}"), + }); + } + + let object_size = result.object_size as u64; + info!("Successfully recreated missing object: {}/{} ({} bytes)", bucket, object, object_size); + + { + let mut progress = self.progress.write().await; + progress.update_progress(4, 4, object_size, object_size); + } + Ok(()) + } + Err(e) => { + error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e); + Err(Error::TaskExecutionFailed { + message: format!("Failed to recreate missing object {bucket}/{object}: {e}"), + }) + } + } + } + + async fn heal_bucket(&self, bucket: &str) -> Result<()> { + info!("Healing bucket: {}", bucket); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("bucket: {bucket}"))); + progress.update_progress(0, 3, 0, 0); + } + + // Step 1: Check if bucket exists + info!("Step 1: Checking bucket existence"); + let bucket_exists = self.storage.get_bucket_info(bucket).await?.is_some(); + if !bucket_exists { + warn!("Bucket does not exist: {}", bucket); + return Err(Error::TaskExecutionFailed { + message: format!("Bucket not found: {bucket}"), + }); + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(1, 3, 0, 0); + } + + // Step 2: Perform bucket heal using ecstore + info!("Step 2: Performing bucket heal using ecstore"); + let heal_opts = HealOpts { + recursive: self.options.recursive, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: false, + pool: self.options.pool_index, + set: self.options.set_index, + }; + + match self.storage.heal_bucket(bucket, &heal_opts).await { + Ok(result) => { + info!("Bucket heal completed successfully: {} ({} drives)", bucket, result.after.drives.len()); + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + Ok(()) + } + Err(e) => { + error!("Bucket heal failed: {} - {}", bucket, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal bucket {bucket}: {e}"), + }) + } + } + } + + async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { + info!("Healing metadata: {}/{}", bucket, object); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("metadata: {bucket}/{object}"))); + progress.update_progress(0, 3, 0, 0); + } + + // Step 1: Check if object exists + info!("Step 1: Checking object existence"); + let object_exists = self.storage.object_exists(bucket, object).await?; + if !object_exists { + warn!("Object does not exist: {}/{}", bucket, object); + return Err(Error::TaskExecutionFailed { + message: format!("Object not found: {bucket}/{object}"), + }); + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(1, 3, 0, 0); + } + + // Step 2: Perform metadata heal using ecstore + info!("Step 2: Performing metadata heal using ecstore"); + let heal_opts = HealOpts { + recursive: false, + dry_run: self.options.dry_run, + remove: false, + recreate: false, + scan_mode: HealScanMode::Deep, + update_parity: false, + no_lock: false, + pool: self.options.pool_index, + set: self.options.set_index, + }; + + match self.storage.heal_object(bucket, object, None, &heal_opts).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("Metadata heal failed: {}/{} - {}", bucket, object, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal metadata {bucket}/{object}: {e}"), + }); + } + + info!( + "Metadata heal completed successfully: {}/{} ({} drives)", + bucket, + object, + result.after.drives.len() + ); + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + Ok(()) + } + Err(e) => { + error!("Metadata heal failed: {}/{} - {}", bucket, object, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal metadata {bucket}/{object}: {e}"), + }) + } + } + } + + async fn heal_mrf(&self, meta_path: &str) -> Result<()> { + info!("Healing MRF: {}", meta_path); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("mrf: {meta_path}"))); + progress.update_progress(0, 2, 0, 0); + } + + // Parse meta_path to extract bucket and object + let parts: Vec<&str> = meta_path.split('/').collect(); + if parts.len() < 2 { + return Err(Error::TaskExecutionFailed { + message: format!("Invalid meta path format: {meta_path}"), + }); + } + + let bucket = parts[0]; + let object = parts[1..].join("/"); + + // Step 1: Perform MRF heal using ecstore + info!("Step 1: Performing MRF heal using ecstore"); + let heal_opts = HealOpts { + recursive: true, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: None, + set: None, + }; + + match self.storage.heal_object(bucket, &object, None, &heal_opts).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("MRF heal failed: {} - {}", meta_path, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(2, 2, 0, 0); + } + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal MRF {meta_path}: {e}"), + }); + } + + info!("MRF heal completed successfully: {} ({} drives)", meta_path, result.after.drives.len()); + + { + let mut progress = self.progress.write().await; + progress.update_progress(2, 2, 0, 0); + } + Ok(()) + } + Err(e) => { + error!("MRF heal failed: {} - {}", meta_path, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(2, 2, 0, 0); + } + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal MRF {meta_path}: {e}"), + }) + } + } + } + + async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { + info!("Healing EC decode: {}/{}", bucket, object); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}"))); + progress.update_progress(0, 3, 0, 0); + } + + // Step 1: Check if object exists + info!("Step 1: Checking object existence"); + let object_exists = self.storage.object_exists(bucket, object).await?; + if !object_exists { + warn!("Object does not exist: {}/{}", bucket, object); + return Err(Error::TaskExecutionFailed { + message: format!("Object not found: {bucket}/{object}"), + }); + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(1, 3, 0, 0); + } + + // Step 2: Perform EC decode heal using ecstore + info!("Step 2: Performing EC decode heal using ecstore"); + let heal_opts = HealOpts { + recursive: false, + dry_run: self.options.dry_run, + remove: false, + recreate: true, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: None, + set: None, + }; + + match self.storage.heal_object(bucket, object, version_id, &heal_opts).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("EC decode heal failed: {}/{} - {}", bucket, object, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal EC decode {bucket}/{object}: {e}"), + }); + } + + let object_size = result.object_size as u64; + info!( + "EC decode heal completed successfully: {}/{} ({} bytes, {} drives)", + bucket, + object, + object_size, + result.after.drives.len() + ); + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, object_size, object_size); + } + Ok(()) + } + Err(e) => { + error!("EC decode heal failed: {}/{} - {}", bucket, object, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 3, 0, 0); + } + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal EC decode {bucket}/{object}: {e}"), + }) + } + } + } + + async fn heal_erasure_set(&self, buckets: Vec, set_disk_id: String) -> Result<()> { + info!("Healing Erasure Set: {} ({} buckets)", set_disk_id, buckets.len()); + + // update progress + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len()))); + progress.update_progress(0, 4, 0, 0); + } + + let buckets = if buckets.is_empty() { + info!("No buckets specified, listing all buckets"); + let bucket_infos = self.storage.list_buckets().await?; + bucket_infos.into_iter().map(|info| info.name).collect() + } else { + buckets + }; + + // Step 1: Perform disk format heal using ecstore + info!("Step 1: Performing disk format heal using ecstore"); + match self.storage.heal_format(self.options.dry_run).await { + Ok((result, error)) => { + if let Some(e) = error { + error!("Disk format heal failed: {} - {}", set_disk_id, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(4, 4, 0, 0); + } + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal disk format for {set_disk_id}: {e}"), + }); + } + + info!( + "Disk format heal completed successfully: {} ({} drives)", + set_disk_id, + result.after.drives.len() + ); + } + Err(e) => { + error!("Disk format heal failed: {} - {}", set_disk_id, e); + { + let mut progress = self.progress.write().await; + progress.update_progress(4, 4, 0, 0); + } + return Err(Error::TaskExecutionFailed { + message: format!("Failed to heal disk format for {set_disk_id}: {e}"), + }); + } + } + + { + let mut progress = self.progress.write().await; + progress.update_progress(1, 4, 0, 0); + } + + // Step 2: Get disk for resume functionality + info!("Step 2: Getting disk for resume functionality"); + let disk = self.storage.get_disk_for_resume(&set_disk_id).await?; + + { + let mut progress = self.progress.write().await; + progress.update_progress(2, 4, 0, 0); + } + + // Step 3: Heal bucket structure + for bucket in buckets.iter() { + if let Err(err) = self.heal_bucket(bucket).await { + info!("{}", err.to_string()); + } + } + + // Step 3: Create erasure set healer with resume support + info!("Step 3: Creating erasure set healer with resume support"); + let erasure_healer = ErasureSetHealer::new(self.storage.clone(), self.progress.clone(), self.cancel_token.clone(), disk); + + { + let mut progress = self.progress.write().await; + progress.update_progress(3, 4, 0, 0); + } + + // Step 4: Execute erasure set heal with resume + info!("Step 4: Executing erasure set heal with resume"); + let result = erasure_healer.heal_erasure_set(&buckets, &set_disk_id).await; + + { + let mut progress = self.progress.write().await; + progress.update_progress(4, 4, 0, 0); + } + + match result { + Ok(_) => { + info!("Erasure set heal completed successfully: {} ({} buckets)", set_disk_id, buckets.len()); + Ok(()) + } + Err(e) => { + error!("Erasure set heal failed: {} - {}", set_disk_id, e); + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal erasure set {set_disk_id}: {e}"), + }) + } + } + } +} + +impl std::fmt::Debug for HealTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HealTask") + .field("id", &self.id) + .field("heal_type", &self.heal_type) + .field("options", &self.options) + .field("created_at", &self.created_at) + .finish() + } +} diff --git a/crates/ahm/src/lib.rs b/crates/ahm/src/lib.rs index d3d656194..17a70ff4e 100644 --- a/crates/ahm/src/lib.rs +++ b/crates/ahm/src/lib.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock}; use tokio_util::sync::CancellationToken; +use tracing::{error, info}; pub mod error; +pub mod heal; pub mod scanner; pub use error::{Error, Result}; -pub use scanner::{ - BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, Scanner, ScannerMetrics, load_data_usage_from_backend, - store_data_usage_in_backend, -}; +pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor}; +pub use scanner::Scanner; // Global cancellation token for AHM services (scanner and other background tasks) static GLOBAL_AHM_SERVICES_CANCEL_TOKEN: OnceLock = OnceLock::new(); @@ -52,3 +52,61 @@ pub fn shutdown_ahm_services() { cancel_token.cancel(); } } + +/// Global heal manager instance +static GLOBAL_HEAL_MANAGER: OnceLock> = OnceLock::new(); + +/// Global heal channel processor instance +static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock>> = OnceLock::new(); + +/// Initialize and start heal manager with channel processor +pub async fn init_heal_manager( + storage: Arc, + config: Option, +) -> Result> { + // Create heal manager + let heal_manager = Arc::new(HealManager::new(storage, config)); + + // Start heal manager + heal_manager.start().await?; + + // Store global instance + GLOBAL_HEAL_MANAGER + .set(heal_manager.clone()) + .map_err(|_| Error::Config("Heal manager already initialized".to_string()))?; + + // Initialize heal channel + let channel_receiver = rustfs_common::heal_channel::init_heal_channel(); + + // Create channel processor + let channel_processor = HealChannelProcessor::new(heal_manager.clone()); + + // Store channel processor instance first + GLOBAL_HEAL_CHANNEL_PROCESSOR + .set(Arc::new(tokio::sync::Mutex::new(channel_processor))) + .map_err(|_| Error::Config("Heal channel processor already initialized".to_string()))?; + + // Start channel processor in background + let receiver = channel_receiver; + tokio::spawn(async move { + if let Some(processor_guard) = GLOBAL_HEAL_CHANNEL_PROCESSOR.get() { + let mut processor = processor_guard.lock().await; + if let Err(e) = processor.start(receiver).await { + error!("Heal channel processor failed: {}", e); + } + } + }); + + info!("Heal manager with channel processor initialized successfully"); + Ok(heal_manager) +} + +/// Get global heal manager instance +pub fn get_heal_manager() -> Option<&'static Arc> { + GLOBAL_HEAL_MANAGER.get() +} + +/// Get global heal channel processor instance +pub fn get_heal_channel_processor() -> Option<&'static Arc>> { + GLOBAL_HEAL_CHANNEL_PROCESSOR.get() +} diff --git a/crates/ahm/src/scanner/data_scanner.rs b/crates/ahm/src/scanner/data_scanner.rs index 1e3e4fc2a..8083ecf73 100644 --- a/crates/ahm/src/scanner/data_scanner.rs +++ b/crates/ahm/src/scanner/data_scanner.rs @@ -22,20 +22,23 @@ use ecstore::{ disk::{DiskAPI, DiskStore, WalkDirOptions}, set_disk::SetDisks, }; -use rustfs_ecstore as ecstore; +use rustfs_ecstore::{self as ecstore, StorageAPI, data_usage::store_data_usage_in_backend}; use rustfs_filemeta::MetacacheReader; use tokio::sync::{Mutex, RwLock}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; -use super::{ - data_usage::DataUsageInfo, - metrics::{BucketMetrics, DiskMetrics, MetricsCollector, ScannerMetrics}, -}; +use super::metrics::{BucketMetrics, DiskMetrics, MetricsCollector, ScannerMetrics}; +use crate::heal::HealManager; use crate::{ + HealRequest, error::{Error, Result}, get_ahm_services_cancel_token, }; +use rustfs_common::{ + data_usage::DataUsageInfo, + metrics::{Metric, Metrics, globalMetrics}, +}; use rustfs_ecstore::disk::RUSTFS_META_BUCKET; @@ -116,7 +119,7 @@ pub struct Scanner { config: Arc>, /// Scanner state state: Arc>, - /// Metrics collector + /// Local metrics collector (for backward compatibility) metrics: Arc, /// Bucket metrics cache bucket_metrics: Arc>>, @@ -126,15 +129,15 @@ pub struct Scanner { data_usage_stats: Arc>>, /// Last data usage statistics collection time last_data_usage_collection: Arc>>, + /// Heal manager for auto-heal integration + heal_manager: Option>, } impl Scanner { /// Create a new scanner - pub fn new(config: Option) -> Self { + pub fn new(config: Option, heal_manager: Option>) -> Self { let config = config.unwrap_or_default(); - info!("Creating AHM scanner for all EC sets"); - Self { config: Arc::new(RwLock::new(config)), state: Arc::new(RwLock::new(ScannerState::default())), @@ -143,9 +146,15 @@ impl Scanner { disk_metrics: Arc::new(Mutex::new(HashMap::new())), data_usage_stats: Arc::new(Mutex::new(HashMap::new())), last_data_usage_collection: Arc::new(RwLock::new(None)), + heal_manager, } } + /// Set the heal manager after construction + pub fn set_heal_manager(&mut self, heal_manager: Arc) { + self.heal_manager = Some(heal_manager); + } + /// Start the scanner pub async fn start(&self) -> Result<()> { let mut state = self.state.write().await; @@ -279,10 +288,18 @@ impl Scanner { metrics } + /// Get global metrics from common crate + pub async fn get_global_metrics(&self) -> rustfs_madmin::metrics::ScannerMetrics { + globalMetrics.report().await + } + /// Perform a single scan cycle pub async fn scan_cycle(&self) -> Result<()> { let start_time = SystemTime::now(); + // Start global metrics collection for this cycle + let stop_fn = Metrics::time(Metric::ScanCycle); + info!("Starting scan cycle {} for all EC sets", self.metrics.get_metrics().current_cycle + 1); // Update state @@ -294,6 +311,14 @@ impl Scanner { state.scanning_disks.clear(); } + // Update global metrics cycle information + let cycle_info = rustfs_common::metrics::CurrentCycle { + current: self.state.read().await.current_cycle, + cycle_completed: vec![chrono::Utc::now()], + started: chrono::Utc::now(), + }; + globalMetrics.set_cycle(Some(cycle_info)).await; + self.metrics.set_current_cycle(self.state.read().await.current_cycle); self.metrics.increment_total_cycles(); @@ -385,6 +410,9 @@ impl Scanner { state.current_scan_duration = Some(scan_duration); } + // Complete global metrics collection for this cycle + stop_fn(); + info!( "Completed scan cycle in {:?} ({} successful, {} failed)", scan_duration, successful_scans, failed_scans @@ -392,6 +420,207 @@ impl Scanner { Ok(()) } + /// Verify object integrity and trigger healing if necessary + async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<()> { + debug!("Starting verify_object_integrity for {}/{}", bucket, object); + + let config = self.config.read().await; + if !config.enable_healing || config.scan_mode != ScanMode::Deep { + debug!("Healing disabled or not in deep scan mode, skipping verification"); + return Ok(()); + } + + if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() { + // First try the standard integrity check + let object_opts = ecstore::store_api::ObjectOptions::default(); + let mut integrity_failed = false; + + debug!("Running standard object verification for {}/{}", bucket, object); + match ecstore.verify_object_integrity(bucket, object, &object_opts).await { + Ok(_) => { + debug!("Standard verification passed for {}/{}", bucket, object); + // Standard verification passed, now check for missing data parts + match self.check_data_parts_integrity(bucket, object).await { + Ok(_) => { + // Object is completely healthy + debug!("Data parts integrity check passed for {}/{}", bucket, object); + self.metrics.increment_healthy_objects(); + } + Err(e) => { + // Data parts are missing or corrupt + debug!("Data parts integrity check failed for {}/{}: {}", bucket, object, e); + warn!("Data parts integrity check failed for {}/{}: {}. Triggering heal.", bucket, object, e); + integrity_failed = true; + } + } + } + Err(e) => { + // Standard object verification failed + debug!("Standard verification failed for {}/{}: {}", bucket, object, e); + warn!("Object verification failed for {}/{}: {}. Triggering heal.", bucket, object, e); + integrity_failed = true; + } + } + + debug!("integrity_failed = {} for {}/{}", integrity_failed, bucket, object); + if integrity_failed { + self.metrics.increment_corrupted_objects(); + + if let Some(heal_manager) = &self.heal_manager { + debug!("Submitting heal request for {}/{}", bucket, object); + let heal_request = HealRequest::object(bucket.to_string(), object.to_string(), None); + if let Err(e) = heal_manager.submit_heal_request(heal_request).await { + error!("Failed to submit heal task for {}/{}: {}", bucket, object, e); + } else { + debug!("Successfully submitted heal request for {}/{}", bucket, object); + } + } else { + debug!("No heal manager available for {}/{}", bucket, object); + } + } + } else { + debug!("No ECStore available for {}/{}", bucket, object); + } + + debug!("Completed verify_object_integrity for {}/{}", bucket, object); + Ok(()) + } + + /// Check data parts integrity by verifying all parts exist on disks + async fn check_data_parts_integrity(&self, bucket: &str, object: &str) -> Result<()> { + debug!("Checking data parts integrity for {}/{}", bucket, object); + + if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() { + // Get object info + let object_info = match ecstore.get_object_info(bucket, object, &Default::default()).await { + Ok(info) => info, + Err(e) => { + return Err(Error::Other(format!("Failed to get object info: {e}"))); + } + }; + + debug!( + "Object info for {}/{}: data_blocks={}, parity_blocks={}, parts={}", + bucket, + object, + object_info.data_blocks, + object_info.parity_blocks, + object_info.parts.len() + ); + + // Create FileInfo from ObjectInfo + let file_info = rustfs_filemeta::FileInfo { + volume: bucket.to_string(), + name: object.to_string(), + version_id: object_info.version_id, + is_latest: object_info.is_latest, + deleted: object_info.delete_marker, + size: object_info.size, + mod_time: object_info.mod_time, + parts: object_info + .parts + .iter() + .map(|p| rustfs_filemeta::ObjectPartInfo { + etag: p.etag.clone(), + number: 0, // Will be set by erasure info + size: p.size, + actual_size: p.actual_size, + mod_time: p.mod_time, + index: p.index.clone(), + checksums: p.checksums.clone(), + error: None, + }) + .collect(), + erasure: rustfs_filemeta::ErasureInfo { + algorithm: "ReedSolomon".to_string(), + data_blocks: object_info.data_blocks, + parity_blocks: object_info.parity_blocks, + block_size: 0, // Default value + index: 1, // Default index + distribution: (1..=object_info.data_blocks + object_info.parity_blocks).collect(), + checksums: vec![], + }, + ..Default::default() + }; + + // Get all disks from ECStore's disk_map + let mut has_missing_parts = false; + let mut total_disks_checked = 0; + let mut disks_with_errors = 0; + + debug!("Checking {} pools in disk_map", ecstore.disk_map.len()); + + for (pool_idx, pool_disks) in &ecstore.disk_map { + debug!("Checking pool {}, {} disks", pool_idx, pool_disks.len()); + + for (disk_idx, disk_option) in pool_disks.iter().enumerate() { + if let Some(disk) = disk_option { + total_disks_checked += 1; + debug!("Checking disk {} in pool {}: {}", disk_idx, pool_idx, disk.path().display()); + + match disk.check_parts(bucket, object, &file_info).await { + Ok(check_result) => { + debug!( + "check_parts returned {} results for disk {}", + check_result.results.len(), + disk.path().display() + ); + + // Check if any parts are missing or corrupt + for (part_idx, &result) in check_result.results.iter().enumerate() { + debug!("Part {} result: {} on disk {}", part_idx, result, disk.path().display()); + + if result == 4 || result == 5 { + // CHECK_PART_FILE_NOT_FOUND or CHECK_PART_FILE_CORRUPT + has_missing_parts = true; + disks_with_errors += 1; + warn!( + "Found missing or corrupt part {} for object {}/{} on disk {} (pool {}): result={}", + part_idx, + bucket, + object, + disk.path().display(), + pool_idx, + result + ); + break; + } + } + } + Err(e) => { + disks_with_errors += 1; + warn!("Failed to check parts on disk {}: {}", disk.path().display(), e); + // Continue checking other disks + } + } + + if has_missing_parts { + break; // No need to check other disks if we found missing parts + } + } else { + debug!("Disk {} in pool {} is None", disk_idx, pool_idx); + } + } + + if has_missing_parts { + break; // No need to check other pools if we found missing parts + } + } + + debug!( + "Data parts check completed for {}/{}: total_disks={}, disks_with_errors={}, has_missing_parts={}", + bucket, object, total_disks_checked, disks_with_errors, has_missing_parts + ); + + if has_missing_parts { + return Err(Error::Other(format!("Object has missing or corrupt data parts: {bucket}/{object}"))); + } + } + + debug!("Data parts integrity verified for {}/{}", bucket, object); + Ok(()) + } + /// Scan a single SetDisks (EC set) async fn scan_set_disks( &self, @@ -402,9 +631,16 @@ impl Scanner { info!("Scanning EC set {} in pool {}", set_index, pool_index); + // list all bucket for heal bucket + // Get online disks from this EC set let (disks, _) = set_disks.get_online_disks_with_healing(false).await; + // Check volume consistency across disks and heal missing buckets + if !disks.is_empty() { + self.check_and_heal_missing_volumes(&disks, set_index, pool_index).await?; + } + if disks.is_empty() { warn!("No online disks available for EC set {} in pool {}", set_index, pool_index); return Ok(Vec::new()); @@ -468,6 +704,9 @@ impl Scanner { async fn scan_disk(&self, disk: &DiskStore) -> Result>> { let disk_path = disk.path().to_string_lossy().to_string(); + // Start global metrics collection for disk scan + let stop_fn = Metrics::time(Metric::ScanBucketDrive); + info!("Scanning disk: {}", disk_path); // Update disk metrics @@ -495,6 +734,44 @@ impl Scanner { metrics.free_space = disk_info.free; metrics.is_online = disk.is_online().await; + // check disk status, if offline, submit erasure set heal task + if !metrics.is_online { + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + // Get bucket list for erasure set healing + let buckets = match rustfs_ecstore::new_object_layer_fn() { + Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await { + Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), + Err(e) => { + error!("Failed to get bucket list for disk healing: {}", e); + return Err(Error::Storage(e)); + } + }, + None => { + error!("No ECStore available for getting bucket list"); + return Err(Error::Storage(ecstore::error::StorageError::other("No ECStore available"))); + } + }; + + let set_disk_id = format!("pool_{}_set_{}", disk.endpoint().pool_idx, disk.endpoint().set_idx); + let req = HealRequest::new( + crate::heal::task::HealType::ErasureSet { buckets, set_disk_id }, + crate::heal::task::HealOptions::default(), + crate::heal::task::HealPriority::High, + ); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + warn!("disk offline, submit erasure set heal task: {} {}", task_id, disk_path); + } + Err(e) => { + error!("disk offline, submit erasure set heal task failed: {} {}", disk_path, e); + } + } + } + } + } + // Additional disk info for debugging debug!( "Disk {}: total={}, used={}, free={}, online={}", @@ -514,6 +791,43 @@ impl Scanner { Ok(volumes) => volumes, Err(e) => { error!("Failed to list volumes on disk {}: {}", disk_path, e); + + // disk access failed, submit erasure set heal task + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + // Get bucket list for erasure set healing + let buckets = match rustfs_ecstore::new_object_layer_fn() { + Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await { + Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), + Err(e) => { + error!("Failed to get bucket list for disk healing: {}", e); + return Err(Error::Storage(e)); + } + }, + None => { + error!("No ECStore available for getting bucket list"); + return Err(Error::Storage(ecstore::error::StorageError::other("No ECStore available"))); + } + }; + + let set_disk_id = format!("pool_{}_set_{}", disk.endpoint().pool_idx, disk.endpoint().set_idx); + let req = HealRequest::new( + crate::heal::task::HealType::ErasureSet { buckets, set_disk_id }, + crate::heal::task::HealOptions::default(), + crate::heal::task::HealPriority::Urgent, + ); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + warn!("disk access failed, submit erasure set heal task: {} {}", task_id, disk_path); + } + Err(heal_err) => { + error!("disk access failed, submit erasure set heal task failed: {} {}", disk_path, heal_err); + } + } + } + } + return Err(Error::Storage(e.into())); } }; @@ -556,6 +870,9 @@ impl Scanner { state.scanning_disks.retain(|d| d != &disk_path); } + // Complete global metrics collection for disk scan + stop_fn(); + Ok(disk_objects) } @@ -564,6 +881,9 @@ impl Scanner { /// This method collects all objects from a disk for a specific bucket. /// It returns a map of object names to their metadata for later analysis. async fn scan_volume(&self, disk: &DiskStore, bucket: &str) -> Result> { + // Start global metrics collection for volume scan + let stop_fn = Metrics::time(Metric::ScanObject); + info!("Scanning bucket: {} on disk: {}", bucket, disk.to_string()); // Initialize bucket metrics if not exists @@ -625,6 +945,28 @@ impl Scanner { if file_meta.versions.is_empty() { objects_with_issues += 1; warn!("Object {} has no versions", entry.name); + + // 对象元数据损坏，提交元数据heal任务 + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + let req = HealRequest::metadata(bucket.to_string(), entry.name.clone()); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + warn!( + "object metadata damaged, submit heal task: {} {} / {}", + task_id, bucket, entry.name + ); + } + Err(e) => { + error!( + "object metadata damaged, submit heal task failed: {} / {} {}", + bucket, entry.name, e + ); + } + } + } + } } else { // Store object metadata for later analysis object_metadata.insert(entry.name.clone(), file_meta.clone()); @@ -632,6 +974,28 @@ impl Scanner { } else { objects_with_issues += 1; warn!("Failed to parse metadata for object {}", entry.name); + + // 对象元数据解析失败，提交元数据heal任务 + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + let req = HealRequest::metadata(bucket.to_string(), entry.name.clone()); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + warn!( + "object metadata parse failed, submit heal task: {} {} / {}", + task_id, bucket, entry.name + ); + } + Err(e) => { + error!( + "object metadata parse failed, submit heal task failed: {} / {} {}", + bucket, entry.name, e + ); + } + } + } + } } } } @@ -659,6 +1023,9 @@ impl Scanner { state.scanning_buckets.retain(|b| b != bucket); } + // Complete global metrics collection for volume scan + stop_fn(); + debug!( "Completed scanning bucket: {} on disk {} ({} objects, {} issues)", bucket, @@ -734,27 +1101,42 @@ impl Scanner { objects_needing_heal += 1; let missing_disks: Vec = (0..disks.len()).filter(|&i| !locations.contains(&i)).collect(); warn!("Object {}/{} missing from disks: {:?}", bucket, object_name, missing_disks); - println!("Object {bucket}/{object_name} missing from disks: {missing_disks:?}"); - // TODO: Trigger heal for this object + + // submit heal task + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + use crate::heal::{HealPriority, HealRequest}; + let req = HealRequest::new( + crate::heal::HealType::Object { + bucket: bucket.clone(), + object: object_name.clone(), + version_id: None, + }, + crate::heal::HealOptions::default(), + HealPriority::High, + ); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + warn!( + "object missing, submit heal task: {} {} / {} (missing disks: {:?})", + task_id, bucket, object_name, missing_disks + ); + } + Err(e) => { + error!("object missing, submit heal task failed: {} / {} {}", bucket, object_name, e); + } + } + } + } } // Step 3: Deep scan EC verification let config = self.config.read().await; if config.scan_mode == ScanMode::Deep { - // Find the first disk that has this object to get metadata - if let Some(&first_disk_idx) = locations.first() { - if let Some(file_meta) = all_disk_objects[first_disk_idx] - .get(bucket) - .and_then(|objects| objects.get(object_name)) - { - if let Err(e) = self - .verify_ec_decode_with_locations(bucket, object_name, file_meta, locations, disks) - .await - { - objects_with_ec_issues += 1; - warn!("EC decode verification failed for object {}/{}: {}", bucket, object_name, e); - } - } + if let Err(e) = self.verify_object_integrity(bucket, object_name).await { + objects_with_ec_issues += 1; + warn!("Object integrity verification failed for object {}/{}: {}", bucket, object_name, e); } } } @@ -777,98 +1159,6 @@ impl Scanner { Ok(()) } - /// Verify EC decode capability for an object using known disk locations - /// - /// This method is optimized to use the known locations of object copies - /// instead of scanning all disks. - async fn verify_ec_decode_with_locations( - &self, - bucket: &str, - object: &str, - file_meta: &rustfs_filemeta::FileMeta, - locations: &[usize], - all_disks: &[DiskStore], - ) -> Result<()> { - // Get EC parameters from the latest version - let (data_blocks, _parity_blocks) = if let Some(latest_version) = file_meta.versions.last() { - if let Ok(version) = rustfs_filemeta::FileMetaVersion::try_from(latest_version.clone()) { - if let Some(obj) = version.object { - (obj.erasure_m, obj.erasure_n) - } else { - // Not an object version, skip EC verification - return Ok(()); - } - } else { - // Cannot parse version, skip EC verification - return Ok(()); - } - } else { - // No versions, skip EC verification - return Ok(()); - }; - - let read_quorum = data_blocks; // Need at least data_blocks to decode - - if locations.len() < read_quorum { - return Err(Error::Scanner(format!( - "Insufficient object copies for EC decode: need {}, have {}", - read_quorum, - locations.len() - ))); - } - - // Try to read object metadata from the known locations - let mut successful_reads = 0; - let mut errors = Vec::new(); - - for &disk_idx in locations { - if successful_reads >= read_quorum { - break; // We have enough copies for EC decode - } - - let disk = &all_disks[disk_idx]; - match disk.read_xl(bucket, object, false).await { - Ok(_) => { - successful_reads += 1; - debug!( - "Successfully read object {}/{} from disk {} (index: {})", - bucket, - object, - disk.to_string(), - disk_idx - ); - } - Err(e) => { - let error_msg = format!("{e}"); - errors.push(error_msg); - debug!( - "Failed to read object {}/{} from disk {} (index: {}): {}", - bucket, - object, - disk.to_string(), - disk_idx, - e - ); - } - } - } - - if successful_reads >= read_quorum { - debug!( - "EC decode verification passed for object {}/{} ({} successful reads from {} locations)", - bucket, - object, - successful_reads, - locations.len() - ); - Ok(()) - } else { - Err(Error::Scanner(format!( - "EC decode verification failed for object {bucket}/{object}: need {read_quorum} reads, got {successful_reads} (errors: {errors:?})" - ))) - } - } - /// Background scan loop with graceful shutdown async fn scan_loop(self) -> Result<()> { let config = self.config.read().await; @@ -981,7 +1271,7 @@ impl Scanner { // Offload persistence to background task let data_clone = data_usage.clone(); tokio::spawn(async move { - if let Err(e) = super::data_usage::store_data_usage_in_backend(data_clone, store).await { + if let Err(e) = store_data_usage_in_backend(data_clone, store).await { error!("Failed to store data usage statistics to backend: {}", e); } else { info!("Successfully stored data usage statistics to backend"); @@ -995,6 +1285,91 @@ impl Scanner { Ok(()) } + /// Check volume consistency across disks and heal missing buckets + async fn check_and_heal_missing_volumes(&self, disks: &[DiskStore], set_index: usize, pool_index: usize) -> Result<()> { + info!("Checking volume consistency for EC set {} in pool {}", set_index, pool_index); + + // Step 1: Collect bucket lists from all online disks + let mut disk_bucket_lists = Vec::new(); + let mut all_buckets = std::collections::HashSet::new(); + + for (disk_idx, disk) in disks.iter().enumerate() { + match disk.list_volumes().await { + Ok(volumes) => { + let bucket_names: Vec = volumes.iter().map(|v| v.name.clone()).collect(); + for bucket in &bucket_names { + all_buckets.insert(bucket.clone()); + } + disk_bucket_lists.push((disk_idx, bucket_names)); + debug!("Disk {} has {} buckets", disk_idx, volumes.len()); + } + Err(e) => { + warn!("Failed to list volumes on disk {}: {}", disk_idx, e); + disk_bucket_lists.push((disk_idx, Vec::new())); + } + } + } + + // Step 2: Find missing buckets on each disk + let mut missing_buckets_count = 0; + for (disk_idx, disk_buckets) in &disk_bucket_lists { + let disk_bucket_set: std::collections::HashSet<_> = disk_buckets.iter().collect(); + let missing_buckets: Vec<_> = all_buckets + .iter() + .filter(|bucket| !disk_bucket_set.contains(bucket)) + .collect(); + + if !missing_buckets.is_empty() { + missing_buckets_count += missing_buckets.len(); + warn!("Disk {} is missing {} buckets: {:?}", disk_idx, missing_buckets.len(), missing_buckets); + + // Step 3: Submit heal tasks for missing buckets + let enable_healing = self.config.read().await.enable_healing; + if enable_healing { + if let Some(heal_manager) = &self.heal_manager { + for bucket in missing_buckets { + let req = crate::heal::HealRequest::bucket(bucket.clone()); + match heal_manager.submit_heal_request(req).await { + Ok(task_id) => { + info!( + "Submitted bucket heal task {} for missing bucket '{}' on disk {}", + task_id, bucket, disk_idx + ); + } + Err(e) => { + error!("Failed to submit bucket heal task for '{}' on disk {}: {}", bucket, disk_idx, e); + } + } + } + } else { + warn!("Healing is enabled but no heal manager available"); + } + } else { + info!("Healing is disabled, skipping bucket heal tasks"); + } + } + } + + if missing_buckets_count > 0 { + warn!( + "Found {} missing bucket instances across {} disks in EC set {} (pool {})", + missing_buckets_count, + disks.len(), + set_index, + pool_index + ); + } else { + info!( + "All buckets are consistent across {} disks in EC set {} (pool {})", + disks.len(), + set_index, + pool_index + ); + } + + Ok(()) + } + /// Clone scanner for background tasks fn clone_for_background(&self) -> Self { Self { @@ -1005,6 +1380,7 @@ impl Scanner { disk_metrics: Arc::clone(&self.disk_metrics), data_usage_stats: Arc::clone(&self.data_usage_stats), last_data_usage_collection: Arc::clone(&self.last_data_usage_collection), + heal_manager: self.heal_manager.clone(), } } } @@ -1012,6 +1388,8 @@ impl Scanner { #[cfg(test)] mod tests { use super::*; + use crate::heal::manager::HealConfig; + use rustfs_ecstore::data_usage::load_data_usage_from_backend; use rustfs_ecstore::disk::endpoint::Endpoint; use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use rustfs_ecstore::store::ECStore; @@ -1019,10 +1397,20 @@ mod tests { StorageAPI, store_api::{MakeBucketOptions, ObjectIO, PutObjReader}, }; + use serial_test::serial; use std::fs; use std::net::SocketAddr; + use std::sync::OnceLock; + + // Global test environment cache to avoid repeated initialization + static GLOBAL_TEST_ENV: OnceLock<(Vec, Arc)> = OnceLock::new(); async fn prepare_test_env(test_dir: Option<&str>, port: Option) -> (Vec, Arc) { + // Check if global environment is already initialized + if let Some((disk_paths, ecstore)) = GLOBAL_TEST_ENV.get() { + return (disk_paths.clone(), ecstore.clone()); + } + // create temp dir as 4 disks let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_test"); let temp_dir = std::path::PathBuf::from(test_base_dir); @@ -1084,11 +1472,14 @@ mod tests { let buckets = buckets_list.into_iter().map(|v| v.name).collect(); rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; + // Store in global cache + let _ = GLOBAL_TEST_ENV.set((disk_paths.clone(), ecstore.clone())); + (disk_paths, ecstore) } #[tokio::test(flavor = "multi_thread")] - #[ignore] + #[serial] async fn test_scanner_basic_functionality() { const TEST_DIR_BASIC: &str = "/tmp/rustfs_ahm_test_basic"; let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_BASIC), Some(9001)).await; @@ -1121,7 +1512,7 @@ mod tests { .expect("put_object failed"); // create Scanner and test basic functionality - let scanner = Scanner::new(None); + let scanner = Scanner::new(None, None); // Test 1: Normal scan - verify object is found println!("=== Test 1: Normal scan ==="); @@ -1186,7 +1577,7 @@ mod tests { // test data usage statistics collection and validation #[tokio::test(flavor = "multi_thread")] - #[ignore] + #[serial] async fn test_scanner_usage_stats() { const TEST_DIR_USAGE_STATS: &str = "/tmp/rustfs_ahm_test_usage_stats"; let (_, ecstore) = prepare_test_env(Some(TEST_DIR_USAGE_STATS), Some(9002)).await; @@ -1200,7 +1591,7 @@ mod tests { .await .unwrap(); - let scanner = Scanner::new(None); + let scanner = Scanner::new(None, None); // enable statistics { @@ -1226,7 +1617,7 @@ mod tests { // verify correctness of persisted data tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let persisted = crate::scanner::data_usage::load_data_usage_from_backend(ecstore.clone()) + let persisted = load_data_usage_from_backend(ecstore.clone()) .await .expect("load persisted usage"); assert_eq!(persisted.objects_total_count, du_after.objects_total_count); @@ -1244,4 +1635,524 @@ mod tests { // clean up temp dir let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_USAGE_STATS)); } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn test_volume_healing_functionality() { + const TEST_DIR_VOLUME_HEAL: &str = "/tmp/rustfs_ahm_test_volume_heal"; + let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_VOLUME_HEAL), Some(9003)).await; + + // Create test buckets + let bucket1 = "test-bucket-1"; + let bucket2 = "test-bucket-2"; + + ecstore.make_bucket(bucket1, &Default::default()).await.unwrap(); + ecstore.make_bucket(bucket2, &Default::default()).await.unwrap(); + + // Add some test objects + let mut pr1 = PutObjReader::from_vec(b"test data 1".to_vec()); + ecstore + .put_object(bucket1, "obj1", &mut pr1, &Default::default()) + .await + .unwrap(); + + let mut pr2 = PutObjReader::from_vec(b"test data 2".to_vec()); + ecstore + .put_object(bucket2, "obj2", &mut pr2, &Default::default()) + .await + .unwrap(); + + // Simulate missing bucket on one disk by removing bucket directory + let disk1_bucket1_path = disk_paths[0].join(bucket1); + if disk1_bucket1_path.exists() { + println!("Removing bucket directory to simulate missing volume: {disk1_bucket1_path:?}"); + match fs::remove_dir_all(&disk1_bucket1_path) { + Ok(_) => println!("Successfully removed bucket directory from disk 0"), + Err(e) => println!("Failed to remove bucket directory: {e}"), + } + } + + // Create scanner without heal manager for now (testing the detection logic) + let scanner = Scanner::new(None, None); + + // Enable healing in config + { + let mut config = scanner.config.write().await; + config.enable_healing = true; + } + + println!("=== Testing volume healing functionality ==="); + + // Run scan cycle which should detect missing volume + // The new check_and_heal_missing_volumes function should be called + let scan_result = scanner.scan_cycle().await; + assert!(scan_result.is_ok(), "Scan cycle should succeed"); + + // Get metrics to verify scan completed + let metrics = scanner.get_metrics().await; + assert!(metrics.total_cycles > 0, "Should have completed scan cycles"); + println!("Volume healing detection test completed successfully"); + println!("Scan metrics: {metrics:?}"); + + // Clean up + let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_VOLUME_HEAL)); + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn test_scanner_detect_missing_data_parts() { + const TEST_DIR_MISSING_PARTS: &str = "/tmp/rustfs_ahm_test_missing_parts"; + let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_MISSING_PARTS), Some(9004)).await; + + // Create test bucket + let bucket_name = "test-bucket-parts"; + let object_name = "large-object-20mb"; + + ecstore.make_bucket(bucket_name, &Default::default()).await.unwrap(); + + // Create a 20MB object to ensure it has multiple parts (MIN_PART_SIZE is 16MB) + let large_data = vec![b'A'; 20 * 1024 * 1024]; // 20MB of 'A' characters + let mut put_reader = PutObjReader::from_vec(large_data); + let object_opts = rustfs_ecstore::store_api::ObjectOptions::default(); + + println!("=== Creating 20MB object ==="); + ecstore + .put_object(bucket_name, object_name, &mut put_reader, &object_opts) + .await + .expect("put_object failed for large object"); + + // Verify object was created and get its info + let obj_info = ecstore + .get_object_info(bucket_name, object_name, &object_opts) + .await + .expect("get_object_info failed"); + + println!( + "Object info: size={}, parts={}, inlined={}", + obj_info.size, + obj_info.parts.len(), + obj_info.inlined + ); + assert!(!obj_info.inlined, "20MB object should not be inlined"); + // Note: Even 20MB might be stored as single part depending on configuration + println!("Object has {} parts", obj_info.parts.len()); + + // Create HealManager and Scanner with shorter heal interval for testing + let heal_storage = Arc::new(crate::heal::storage::ECStoreHealStorage::new(ecstore.clone())); + let heal_config = HealConfig { + enable_auto_heal: true, + heal_interval: Duration::from_millis(100), // 100ms for faster testing + max_concurrent_heals: 4, + task_timeout: Duration::from_secs(300), + queue_size: 1000, + }; + let heal_manager = Arc::new(crate::heal::HealManager::new(heal_storage, Some(heal_config))); + heal_manager.start().await.unwrap(); + let scanner = Scanner::new(None, Some(heal_manager.clone())); + + // Enable healing to detect missing parts + { + let mut config = scanner.config.write().await; + config.enable_healing = true; + config.scan_mode = ScanMode::Deep; + } + + println!("=== Initial scan (all parts present) ==="); + let initial_scan = scanner.scan_cycle().await; + assert!(initial_scan.is_ok(), "Initial scan should succeed"); + + let initial_metrics = scanner.get_metrics().await; + println!("Initial scan metrics: objects_scanned={}", initial_metrics.objects_scanned); + + // Simulate data part loss by deleting part files from some disks + println!("=== Simulating data part loss ==="); + let mut deleted_parts = 0; + let mut deleted_part_paths = Vec::new(); // Track deleted file paths for later verification + + for (disk_idx, disk_path) in disk_paths.iter().enumerate() { + if disk_idx > 0 { + // Only delete from first two disks + break; + } + let bucket_path = disk_path.join(bucket_name); + let object_path = bucket_path.join(object_name); + + if !object_path.exists() { + continue; + } + + // Find the data directory (UUID) + if let Ok(entries) = fs::read_dir(&object_path) { + for entry in entries.flatten() { + let entry_path = entry.path(); + if entry_path.is_dir() { + // This is likely the data_dir, look for part files inside + let part_file_path = entry_path.join("part.1"); + if part_file_path.exists() { + match fs::remove_file(&part_file_path) { + Ok(_) => { + println!("Deleted part file: {part_file_path:?}"); + deleted_part_paths.push(part_file_path); // Store path for verification + deleted_parts += 1; + } + Err(e) => { + println!("Failed to delete part file {part_file_path:?}: {e}"); + } + } + } + } + } + } + } + + println!("Deleted {deleted_parts} part files to simulate data loss"); + assert!(deleted_parts > 0, "Should have deleted some part files"); + + // Scan again to detect missing parts + println!("=== Scan after data deletion (should detect missing data) ==="); + let scan_after_deletion = scanner.scan_cycle().await; + + // Wait a bit for the heal manager to process the queue + tokio::time::sleep(Duration::from_millis(200)).await; + + // Add debug information + println!("=== Debug: Checking heal manager state ==="); + let tasks_count = heal_manager.get_active_tasks_count().await; + println!("Active heal tasks count: {tasks_count}"); + + // Check heal statistics to see if any tasks were submitted + let heal_stats = heal_manager.get_statistics().await; + println!("Heal statistics:"); + println!(" - total_tasks: {}", heal_stats.total_tasks); + println!(" - successful_tasks: {}", heal_stats.successful_tasks); + println!(" - failed_tasks: {}", heal_stats.failed_tasks); + println!(" - running_tasks: {}", heal_stats.running_tasks); + + // Get scanner metrics to see what was scanned + let final_metrics = scanner.get_metrics().await; + println!("Scanner metrics after deletion scan:"); + println!(" - objects_scanned: {}", final_metrics.objects_scanned); + println!(" - healthy_objects: {}", final_metrics.healthy_objects); + println!(" - corrupted_objects: {}", final_metrics.corrupted_objects); + println!(" - objects_with_issues: {}", final_metrics.objects_with_issues); + + // Try to manually verify the object to see what happens + println!("=== Manual object verification ==="); + if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() { + match ecstore.verify_object_integrity(bucket_name, object_name, &object_opts).await { + Ok(_) => println!("Manual verification: Object is healthy"), + Err(e) => println!("Manual verification: Object verification failed: {e}"), + } + } + + // Check if a heal task was submitted (check total tasks instead of active tasks) + assert!(heal_stats.total_tasks > 0, "Heal task should have been submitted"); + println!("{} heal tasks submitted in total", heal_stats.total_tasks); + + // Scanner should handle missing parts gracefully but may detect errors + match scan_after_deletion { + Ok(_) => { + println!("Scanner completed successfully despite missing data"); + } + Err(e) => { + println!("Scanner detected errors (expected): {e}"); + // This is acceptable - scanner may report errors when data is missing + } + } + + let final_metrics = scanner.get_metrics().await; + println!("Final scan metrics: objects_scanned={}", final_metrics.objects_scanned); + + // Verify that scanner completed additional cycles + assert!( + final_metrics.total_cycles > initial_metrics.total_cycles, + "Should have completed additional scan cycles" + ); + + // Test object retrieval after data loss + println!("=== Testing object retrieval after data loss ==="); + let get_result = ecstore.get_object_info(bucket_name, object_name, &object_opts).await; + match get_result { + Ok(info) => { + println!("Object still accessible: size={}", info.size); + // EC should allow recovery if enough shards remain + } + Err(e) => { + println!("Object not accessible due to missing data: {e}"); + // This is expected if too many shards are missing + } + } + + println!("=== Test completed ==="); + println!("Scanner successfully handled missing data scenario"); + + // Verify that deleted part files have been restored by the healing process + println!("=== Verifying file recovery ==="); + let mut recovered_files = 0; + for deleted_path in &deleted_part_paths { + assert!(deleted_path.exists(), "Deleted file should have been recovered"); + println!("Recovered file: {deleted_path:?}"); + recovered_files += 1; + } + + // Assert that at least some files have been recovered + // Note: In a real scenario, healing might take longer, but our test setup should allow recovery + if heal_stats.successful_tasks > 0 { + assert!( + recovered_files == deleted_part_paths.len(), + "Expected at least some deleted files to be recovered by healing process. \ + Deleted {} files, recovered {} files, successful heal tasks: {}", + deleted_part_paths.len(), + recovered_files, + heal_stats.successful_tasks + ); + println!("Successfully recovered {}/{} deleted files", recovered_files, deleted_part_paths.len()); + } else { + println!("No successful heal tasks completed yet - healing may still be in progress"); + } + + // Clean up + let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_MISSING_PARTS)); + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn test_scanner_detect_missing_xl_meta() { + const TEST_DIR_MISSING_META: &str = "/tmp/rustfs_ahm_test_missing_meta"; + let (disk_paths, ecstore) = prepare_test_env(Some(TEST_DIR_MISSING_META), Some(9005)).await; + + // Create test bucket + let bucket_name = "test-bucket-meta"; + let object_name = "test-object-meta"; + + ecstore.make_bucket(bucket_name, &Default::default()).await.unwrap(); + + // Create a test object + let test_data = vec![b'B'; 5 * 1024 * 1024]; // 5MB of 'B' characters + let mut put_reader = PutObjReader::from_vec(test_data); + let object_opts = rustfs_ecstore::store_api::ObjectOptions::default(); + + println!("=== Creating test object ==="); + ecstore + .put_object(bucket_name, object_name, &mut put_reader, &object_opts) + .await + .expect("put_object failed"); + + // Verify object was created and get its info + let obj_info = ecstore + .get_object_info(bucket_name, object_name, &object_opts) + .await + .expect("get_object_info failed"); + + println!("Object info: size={}, parts={}", obj_info.size, obj_info.parts.len()); + + // Create HealManager and Scanner with shorter heal interval for testing + let heal_storage = Arc::new(crate::heal::storage::ECStoreHealStorage::new(ecstore.clone())); + let heal_config = HealConfig { + enable_auto_heal: true, + heal_interval: Duration::from_millis(100), // 100ms for faster testing + max_concurrent_heals: 4, + task_timeout: Duration::from_secs(300), + queue_size: 1000, + }; + let heal_manager = Arc::new(crate::heal::HealManager::new(heal_storage, Some(heal_config))); + heal_manager.start().await.unwrap(); + let scanner = Scanner::new(None, Some(heal_manager.clone())); + + // Enable healing to detect missing metadata + { + let mut config = scanner.config.write().await; + config.enable_healing = true; + config.scan_mode = ScanMode::Deep; + } + + println!("=== Initial scan (all metadata present) ==="); + let initial_scan = scanner.scan_cycle().await; + assert!(initial_scan.is_ok(), "Initial scan should succeed"); + + let initial_metrics = scanner.get_metrics().await; + println!("Initial scan metrics: objects_scanned={}", initial_metrics.objects_scanned); + + // Simulate xl.meta file loss by deleting xl.meta files from some disks + println!("=== Simulating xl.meta file loss ==="); + let mut deleted_meta_files = 0; + let mut deleted_meta_paths = Vec::new(); // Track deleted file paths for later verification + + for (disk_idx, disk_path) in disk_paths.iter().enumerate() { + if disk_idx >= 2 { + // Only delete from first two disks to ensure some copies remain for recovery + break; + } + let bucket_path = disk_path.join(bucket_name); + let object_path = bucket_path.join(object_name); + + if !object_path.exists() { + continue; + } + + // Delete xl.meta file + let xl_meta_path = object_path.join("xl.meta"); + if xl_meta_path.exists() { + match fs::remove_file(&xl_meta_path) { + Ok(_) => { + println!("Deleted xl.meta file: {xl_meta_path:?}"); + deleted_meta_paths.push(xl_meta_path); + deleted_meta_files += 1; + } + Err(e) => { + println!("Failed to delete xl.meta file {xl_meta_path:?}: {e}"); + } + } + } + } + + println!("Deleted {deleted_meta_files} xl.meta files to simulate metadata loss"); + assert!(deleted_meta_files > 0, "Should have deleted some xl.meta files"); + + // Scan again to detect missing metadata + println!("=== Scan after xl.meta deletion (should detect missing metadata) ==="); + let scan_after_deletion = scanner.scan_cycle().await; + + // Wait a bit for the heal manager to process the queue + tokio::time::sleep(Duration::from_millis(500)).await; + + // Add debug information + println!("=== Debug: Checking heal manager state ==="); + let tasks_count = heal_manager.get_active_tasks_count().await; + println!("Active heal tasks count: {tasks_count}"); + + // Check heal statistics to see if any tasks were submitted + let heal_stats = heal_manager.get_statistics().await; + println!("Heal statistics:"); + println!(" - total_tasks: {}", heal_stats.total_tasks); + println!(" - successful_tasks: {}", heal_stats.successful_tasks); + println!(" - failed_tasks: {}", heal_stats.failed_tasks); + println!(" - running_tasks: {}", heal_stats.running_tasks); + + // Get scanner metrics to see what was scanned + let final_metrics = scanner.get_metrics().await; + println!("Scanner metrics after deletion scan:"); + println!(" - objects_scanned: {}", final_metrics.objects_scanned); + println!(" - healthy_objects: {}", final_metrics.healthy_objects); + println!(" - corrupted_objects: {}", final_metrics.corrupted_objects); + println!(" - objects_with_issues: {}", final_metrics.objects_with_issues); + + // Try to manually verify the object to see what happens + println!("=== Manual object verification ==="); + if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() { + match ecstore.verify_object_integrity(bucket_name, object_name, &object_opts).await { + Ok(_) => println!("Manual verification: Object is healthy"), + Err(e) => println!("Manual verification: Object verification failed: {e}"), + } + } + + // Check if a heal task was submitted for metadata recovery + assert!(heal_stats.total_tasks > 0, "Heal task should have been submitted for missing xl.meta"); + println!("{} heal tasks submitted in total", heal_stats.total_tasks); + + // Scanner should handle missing metadata gracefully but may detect errors + match scan_after_deletion { + Ok(_) => { + println!("Scanner completed successfully despite missing metadata"); + } + Err(e) => { + println!("Scanner detected errors (expected): {e}"); + // This is acceptable - scanner may report errors when metadata is missing + } + } + + let final_metrics = scanner.get_metrics().await; + println!("Final scan metrics: objects_scanned={}", final_metrics.objects_scanned); + + // Verify that scanner completed additional cycles + assert!( + final_metrics.total_cycles > initial_metrics.total_cycles, + "Should have completed additional scan cycles" + ); + + // Test object retrieval after metadata loss + println!("=== Testing object retrieval after metadata loss ==="); + let get_result = ecstore.get_object_info(bucket_name, object_name, &object_opts).await; + match get_result { + Ok(info) => { + println!("Object still accessible: size={}", info.size); + // Object should still be accessible if enough metadata copies remain + } + Err(e) => { + println!("Object not accessible due to missing metadata: {e}"); + // This might happen if too many metadata files are missing + } + } + + // Wait a bit more for healing to complete + tokio::time::sleep(Duration::from_millis(1000)).await; + + // Check heal statistics again after waiting + let final_heal_stats = heal_manager.get_statistics().await; + println!("Final heal statistics:"); + println!(" - total_tasks: {}", final_heal_stats.total_tasks); + println!(" - successful_tasks: {}", final_heal_stats.successful_tasks); + println!(" - failed_tasks: {}", final_heal_stats.failed_tasks); + + // Verify that deleted xl.meta files have been restored by the healing process + println!("=== Verifying xl.meta file recovery ==="); + let mut recovered_files = 0; + for deleted_path in &deleted_meta_paths { + assert!(deleted_path.exists(), "Deleted xl.meta file should exist after healing"); + recovered_files += 1; + println!("Recovered xl.meta file: {deleted_path:?}"); + } + + // Assert that healing was attempted + assert!( + final_heal_stats.total_tasks > 0, + "Heal tasks should have been submitted for missing xl.meta files" + ); + + // Check if any heal tasks were successful + if final_heal_stats.successful_tasks > 0 { + println!("Healing completed successfully, checking file recovery..."); + if recovered_files > 0 { + println!( + "Successfully recovered {}/{} deleted xl.meta files", + recovered_files, + deleted_meta_paths.len() + ); + } else { + println!("No xl.meta files recovered yet - healing may have recreated metadata elsewhere"); + } + } else { + println!("No successful heal tasks completed yet - healing may still be in progress or failed"); + + // If healing failed, this is acceptable for this test scenario + // The important thing is that the scanner detected the issue and submitted heal tasks + if final_heal_stats.failed_tasks > 0 { + println!("Heal tasks failed - this is acceptable for missing xl.meta scenario"); + println!("The scanner correctly detected missing metadata and submitted heal requests"); + } + } + + // The key success criteria for this test is that: + // 1. Scanner detected missing xl.meta files + // 2. Scanner submitted heal tasks for the missing metadata + // 3. Scanner handled the situation gracefully without crashing + println!("=== Test completed ==="); + println!("Scanner successfully handled missing xl.meta scenario"); + println!("Key achievements:"); + println!(" - Scanner detected missing xl.meta files"); + println!(" - Scanner submitted {} heal tasks", final_heal_stats.total_tasks); + println!(" - Scanner handled the situation gracefully"); + if recovered_files > 0 { + println!( + " - Successfully recovered {}/{} xl.meta files", + recovered_files, + deleted_meta_paths.len() + ); + } else { + println!(" - Note: xl.meta file recovery may require additional time or manual intervention"); + } + + // Clean up + let _ = std::fs::remove_dir_all(std::path::Path::new(TEST_DIR_MISSING_META)); + } } diff --git a/crates/ahm/src/scanner/data_usage.rs b/crates/ahm/src/scanner/data_usage.rs deleted file mode 100644 index 2ab97bff4..000000000 --- a/crates/ahm/src/scanner/data_usage.rs +++ /dev/null @@ -1,671 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{collections::HashMap, sync::Arc, time::SystemTime}; - -use rustfs_ecstore::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore}; -use rustfs_utils::path::SLASH_SEPARATOR; -use serde::{Deserialize, Serialize}; -use tracing::{error, info, warn}; - -use crate::error::{Error, Result}; - -// Data usage storage constants -pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; -const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; -const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; -pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; - -// Data usage storage paths -lazy_static::lazy_static! { - pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", - rustfs_ecstore::disk::RUSTFS_META_BUCKET, - SLASH_SEPARATOR, - rustfs_ecstore::disk::BUCKET_META_PREFIX - ); - pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", - rustfs_ecstore::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, - DATA_USAGE_OBJ_NAME - ); - pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", - rustfs_ecstore::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, - DATA_USAGE_BLOOM_NAME - ); -} - -/// Bucket target usage info provides replication statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct BucketTargetUsageInfo { - pub replication_pending_size: u64, - pub replication_failed_size: u64, - pub replicated_size: u64, - pub replica_size: u64, - pub replication_pending_count: u64, - pub replication_failed_count: u64, - pub replicated_count: u64, -} - -/// Bucket usage info provides bucket-level statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct BucketUsageInfo { - pub size: u64, - // Following five fields suffixed with V1 are here for backward compatibility - // Total Size for objects that have not yet been replicated - pub replication_pending_size_v1: u64, - // Total size for objects that have witness one or more failures and will be retried - pub replication_failed_size_v1: u64, - // Total size for objects that have been replicated to destination - pub replicated_size_v1: u64, - // Total number of objects pending replication - pub replication_pending_count_v1: u64, - // Total number of objects that failed replication - pub replication_failed_count_v1: u64, - - pub objects_count: u64, - pub object_size_histogram: HashMap, - pub object_versions_histogram: HashMap, - pub versions_count: u64, - pub delete_markers_count: u64, - pub replica_size: u64, - pub replica_count: u64, - pub replication_info: HashMap, -} - -/// DataUsageInfo represents data usage stats of the underlying storage -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct DataUsageInfo { - /// Total capacity - pub total_capacity: u64, - /// Total used capacity - pub total_used_capacity: u64, - /// Total free capacity - pub total_free_capacity: u64, - - /// LastUpdate is the timestamp of when the data usage info was last updated - pub last_update: Option, - - /// Objects total count across all buckets - pub objects_total_count: u64, - /// Versions total count across all buckets - pub versions_total_count: u64, - /// Delete markers total count across all buckets - pub delete_markers_total_count: u64, - /// Objects total size across all buckets - pub objects_total_size: u64, - /// Replication info across all buckets - pub replication_info: HashMap, - - /// Total number of buckets in this cluster - pub buckets_count: u64, - /// Buckets usage info provides following information across all buckets - pub buckets_usage: HashMap, - /// Deprecated kept here for backward compatibility reasons - pub bucket_sizes: HashMap, -} - -/// Size summary for a single object or group of objects -#[derive(Debug, Default, Clone)] -pub struct SizeSummary { - /// Total size - pub total_size: usize, - /// Number of versions - pub versions: usize, - /// Number of delete markers - pub delete_markers: usize, - /// Replicated size - pub replicated_size: usize, - /// Replicated count - pub replicated_count: usize, - /// Pending size - pub pending_size: usize, - /// Failed size - pub failed_size: usize, - /// Replica size - pub replica_size: usize, - /// Replica count - pub replica_count: usize, - /// Pending count - pub pending_count: usize, - /// Failed count - pub failed_count: usize, - /// Replication target stats - pub repl_target_stats: HashMap, -} - -/// Replication target size summary -#[derive(Debug, Default, Clone)] -pub struct ReplTargetSizeSummary { - /// Replicated size - pub replicated_size: usize, - /// Replicated count - pub replicated_count: usize, - /// Pending size - pub pending_size: usize, - /// Failed size - pub failed_size: usize, - /// Pending count - pub pending_count: usize, - /// Failed count - pub failed_count: usize, -} - -impl DataUsageInfo { - /// Create a new DataUsageInfo - pub fn new() -> Self { - Self::default() - } - - /// Add object metadata to data usage statistics - pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) { - // This method is kept for backward compatibility - // For accurate version counting, use add_object_from_file_meta instead - let bucket_name = match self.extract_bucket_from_path(object_path) { - Ok(name) => name, - Err(_) => return, - }; - - // Update bucket statistics - if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { - bucket_usage.size += meta_object.size as u64; - bucket_usage.objects_count += 1; - bucket_usage.versions_count += 1; // Simplified: assume 1 version per object - - // Update size histogram - let total_size = meta_object.size as u64; - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if total_size >= min_size && total_size < max_size { - *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; - break; - } - } - - // Update version histogram (simplified - count as single version) - *bucket_usage - .object_versions_histogram - .entry("SINGLE_VERSION".to_string()) - .or_insert(0) += 1; - } else { - // Create new bucket usage - let mut bucket_usage = BucketUsageInfo { - size: meta_object.size as u64, - objects_count: 1, - versions_count: 1, - ..Default::default() - }; - bucket_usage.object_size_histogram.insert("0-1KB".to_string(), 1); - bucket_usage.object_versions_histogram.insert("SINGLE_VERSION".to_string(), 1); - self.buckets_usage.insert(bucket_name, bucket_usage); - } - - // Update global statistics - self.objects_total_size += meta_object.size as u64; - self.objects_total_count += 1; - self.versions_total_count += 1; - } - - /// Add object from FileMeta for accurate version counting - pub fn add_object_from_file_meta(&mut self, object_path: &str, file_meta: &rustfs_filemeta::FileMeta) { - let bucket_name = match self.extract_bucket_from_path(object_path) { - Ok(name) => name, - Err(_) => return, - }; - - // Calculate accurate statistics from all versions - let mut total_size = 0u64; - let mut versions_count = 0u64; - let mut delete_markers_count = 0u64; - let mut latest_object_size = 0u64; - - // Process all versions to get accurate counts - for version in &file_meta.versions { - match rustfs_filemeta::FileMetaVersion::try_from(version.clone()) { - Ok(ver) => { - if let Some(obj) = ver.object { - total_size += obj.size as u64; - versions_count += 1; - latest_object_size = obj.size as u64; // Keep track of latest object size - } else if ver.delete_marker.is_some() { - delete_markers_count += 1; - } - } - Err(_) => { - // Skip invalid versions - continue; - } - } - } - - // Update bucket statistics - if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { - bucket_usage.size += total_size; - bucket_usage.objects_count += 1; - bucket_usage.versions_count += versions_count; - bucket_usage.delete_markers_count += delete_markers_count; - - // Update size histogram based on latest object size - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if latest_object_size >= min_size && latest_object_size < max_size { - *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; - break; - } - } - - // Update version histogram based on actual version count - let version_ranges = [ - ("1", 1, 1), - ("2-5", 2, 5), - ("6-10", 6, 10), - ("11-50", 11, 50), - ("51-100", 51, 100), - ("100+", 101, usize::MAX), - ]; - - for (range_name, min_versions, max_versions) in version_ranges { - if versions_count as usize >= min_versions && versions_count as usize <= max_versions { - *bucket_usage - .object_versions_histogram - .entry(range_name.to_string()) - .or_insert(0) += 1; - break; - } - } - } else { - // Create new bucket usage - let mut bucket_usage = BucketUsageInfo { - size: total_size, - objects_count: 1, - versions_count, - delete_markers_count, - ..Default::default() - }; - - // Set size histogram - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if latest_object_size >= min_size && latest_object_size < max_size { - bucket_usage.object_size_histogram.insert(range_name.to_string(), 1); - break; - } - } - - // Set version histogram - let version_ranges = [ - ("1", 1, 1), - ("2-5", 2, 5), - ("6-10", 6, 10), - ("11-50", 11, 50), - ("51-100", 51, 100), - ("100+", 101, usize::MAX), - ]; - - for (range_name, min_versions, max_versions) in version_ranges { - if versions_count as usize >= min_versions && versions_count as usize <= max_versions { - bucket_usage.object_versions_histogram.insert(range_name.to_string(), 1); - break; - } - } - - self.buckets_usage.insert(bucket_name, bucket_usage); - // Update buckets count when adding new bucket - self.buckets_count = self.buckets_usage.len() as u64; - } - - // Update global statistics - self.objects_total_size += total_size; - self.objects_total_count += 1; - self.versions_total_count += versions_count; - self.delete_markers_total_count += delete_markers_count; - } - - /// Extract bucket name from object path - fn extract_bucket_from_path(&self, object_path: &str) -> Result { - let parts: Vec<&str> = object_path.split('/').collect(); - if parts.is_empty() { - return Err(Error::Scanner("Invalid object path: empty".to_string())); - } - Ok(parts[0].to_string()) - } - - /// Update capacity information - pub fn update_capacity(&mut self, total: u64, used: u64, free: u64) { - self.total_capacity = total; - self.total_used_capacity = used; - self.total_free_capacity = free; - self.last_update = Some(SystemTime::now()); - } - - /// Add bucket usage info - pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) { - self.buckets_usage.insert(bucket.clone(), usage); - self.buckets_count = self.buckets_usage.len() as u64; - self.last_update = Some(SystemTime::now()); - } - - /// Get bucket usage info - pub fn get_bucket_usage(&self, bucket: &str) -> Option<&BucketUsageInfo> { - self.buckets_usage.get(bucket) - } - - /// Calculate total statistics from all buckets - pub fn calculate_totals(&mut self) { - self.objects_total_count = 0; - self.versions_total_count = 0; - self.delete_markers_total_count = 0; - self.objects_total_size = 0; - - for usage in self.buckets_usage.values() { - self.objects_total_count += usage.objects_count; - self.versions_total_count += usage.versions_count; - self.delete_markers_total_count += usage.delete_markers_count; - self.objects_total_size += usage.size; - } - } - - /// Merge another DataUsageInfo into this one - pub fn merge(&mut self, other: &DataUsageInfo) { - // Merge bucket usage - for (bucket, usage) in &other.buckets_usage { - if let Some(existing) = self.buckets_usage.get_mut(bucket) { - existing.merge(usage); - } else { - self.buckets_usage.insert(bucket.clone(), usage.clone()); - } - } - - // Recalculate totals - self.calculate_totals(); - - // Ensure buckets_count stays consistent with buckets_usage - self.buckets_count = self.buckets_usage.len() as u64; - - // Update last update time - if let Some(other_update) = other.last_update { - if self.last_update.is_none() || other_update > self.last_update.unwrap() { - self.last_update = Some(other_update); - } - } - } -} - -impl BucketUsageInfo { - /// Create a new BucketUsageInfo - pub fn new() -> Self { - Self::default() - } - - /// Add size summary to this bucket usage - pub fn add_size_summary(&mut self, summary: &SizeSummary) { - self.size += summary.total_size as u64; - self.versions_count += summary.versions as u64; - self.delete_markers_count += summary.delete_markers as u64; - self.replica_size += summary.replica_size as u64; - self.replica_count += summary.replica_count as u64; - } - - /// Merge another BucketUsageInfo into this one - pub fn merge(&mut self, other: &BucketUsageInfo) { - self.size += other.size; - self.objects_count += other.objects_count; - self.versions_count += other.versions_count; - self.delete_markers_count += other.delete_markers_count; - self.replica_size += other.replica_size; - self.replica_count += other.replica_count; - - // Merge histograms - for (key, value) in &other.object_size_histogram { - *self.object_size_histogram.entry(key.clone()).or_insert(0) += value; - } - - for (key, value) in &other.object_versions_histogram { - *self.object_versions_histogram.entry(key.clone()).or_insert(0) += value; - } - - // Merge replication info - for (target, info) in &other.replication_info { - let entry = self.replication_info.entry(target.clone()).or_default(); - entry.replicated_size += info.replicated_size; - entry.replica_size += info.replica_size; - entry.replication_pending_size += info.replication_pending_size; - entry.replication_failed_size += info.replication_failed_size; - entry.replication_pending_count += info.replication_pending_count; - entry.replication_failed_count += info.replication_failed_count; - entry.replicated_count += info.replicated_count; - } - - // Merge backward compatibility fields - self.replication_pending_size_v1 += other.replication_pending_size_v1; - self.replication_failed_size_v1 += other.replication_failed_size_v1; - self.replicated_size_v1 += other.replicated_size_v1; - self.replication_pending_count_v1 += other.replication_pending_count_v1; - self.replication_failed_count_v1 += other.replication_failed_count_v1; - } -} - -impl SizeSummary { - /// Create a new SizeSummary - pub fn new() -> Self { - Self::default() - } - - /// Add another SizeSummary to this one - pub fn add(&mut self, other: &SizeSummary) { - self.total_size += other.total_size; - self.versions += other.versions; - self.delete_markers += other.delete_markers; - self.replicated_size += other.replicated_size; - self.replicated_count += other.replicated_count; - self.pending_size += other.pending_size; - self.failed_size += other.failed_size; - self.replica_size += other.replica_size; - self.replica_count += other.replica_count; - self.pending_count += other.pending_count; - self.failed_count += other.failed_count; - - // Merge replication target stats - for (target, stats) in &other.repl_target_stats { - let entry = self.repl_target_stats.entry(target.clone()).or_default(); - entry.replicated_size += stats.replicated_size; - entry.replicated_count += stats.replicated_count; - entry.pending_size += stats.pending_size; - entry.failed_size += stats.failed_size; - entry.pending_count += stats.pending_count; - entry.failed_count += stats.failed_count; - } - } -} - -/// Store data usage info to backend storage -pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc) -> Result<()> { - let data = - serde_json::to_vec(&data_usage_info).map_err(|e| Error::Config(format!("Failed to serialize data usage info: {e}")))?; - - // Save to backend using the same mechanism as original code - rustfs_ecstore::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data) - .await - .map_err(Error::Storage)?; - - Ok(()) -} - -/// Load data usage info from backend storage -pub async fn load_data_usage_from_backend(store: Arc) -> Result { - let buf = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await { - Ok(data) => data, - Err(e) => { - error!("Failed to read data usage info from backend: {}", e); - if e == rustfs_ecstore::error::Error::ConfigNotFound { - return Ok(DataUsageInfo::default()); - } - return Err(Error::Storage(e)); - } - }; - - let mut data_usage_info: DataUsageInfo = - serde_json::from_slice(&buf).map_err(|e| Error::Config(format!("Failed to deserialize data usage info: {e}")))?; - - warn!("Loaded data usage info from backend {:?}", &data_usage_info); - - // Handle backward compatibility like original code - if data_usage_info.buckets_usage.is_empty() { - data_usage_info.buckets_usage = data_usage_info - .bucket_sizes - .iter() - .map(|(bucket, &size)| { - ( - bucket.clone(), - BucketUsageInfo { - size, - ..Default::default() - }, - ) - }) - .collect(); - } - - if data_usage_info.bucket_sizes.is_empty() { - data_usage_info.bucket_sizes = data_usage_info - .buckets_usage - .iter() - .map(|(bucket, bui)| (bucket.clone(), bui.size)) - .collect(); - } - - for (bucket, bui) in &data_usage_info.buckets_usage { - if bui.replicated_size_v1 > 0 - || bui.replication_failed_count_v1 > 0 - || bui.replication_failed_size_v1 > 0 - || bui.replication_pending_count_v1 > 0 - { - if let Ok((cfg, _)) = get_replication_config(bucket).await { - if !cfg.role.is_empty() { - data_usage_info.replication_info.insert( - cfg.role.clone(), - BucketTargetUsageInfo { - replication_failed_size: bui.replication_failed_size_v1, - replication_failed_count: bui.replication_failed_count_v1, - replicated_size: bui.replicated_size_v1, - replication_pending_count: bui.replication_pending_count_v1, - replication_pending_size: bui.replication_pending_size_v1, - ..Default::default() - }, - ); - } - } - } - } - - Ok(data_usage_info) -} - -/// Example function showing how to use AHM data usage functionality -/// This demonstrates the integration pattern for DataUsageInfoHandler -pub async fn example_data_usage_integration() -> Result<()> { - // Get the global storage instance - let Some(store) = rustfs_ecstore::new_object_layer_fn() else { - return Err(Error::Config("Storage not initialized".to_string())); - }; - - // Load data usage from backend (this replaces the original load_data_usage_from_backend) - let data_usage = load_data_usage_from_backend(store).await?; - - info!( - "Loaded data usage info: {} buckets, {} total objects", - data_usage.buckets_count, data_usage.objects_total_count - ); - - // Example: Store updated data usage back to backend - // This would typically be called by the scanner after collecting new statistics - // store_data_usage_in_backend(data_usage, store).await?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_data_usage_info_creation() { - let mut info = DataUsageInfo::new(); - info.update_capacity(1000, 500, 500); - - assert_eq!(info.total_capacity, 1000); - assert_eq!(info.total_used_capacity, 500); - assert_eq!(info.total_free_capacity, 500); - assert!(info.last_update.is_some()); - } - - #[test] - fn test_bucket_usage_info_merge() { - let mut usage1 = BucketUsageInfo::new(); - usage1.size = 100; - usage1.objects_count = 10; - usage1.versions_count = 5; - - let mut usage2 = BucketUsageInfo::new(); - usage2.size = 200; - usage2.objects_count = 20; - usage2.versions_count = 10; - - usage1.merge(&usage2); - - assert_eq!(usage1.size, 300); - assert_eq!(usage1.objects_count, 30); - assert_eq!(usage1.versions_count, 15); - } - - #[test] - fn test_size_summary_add() { - let mut summary1 = SizeSummary::new(); - summary1.total_size = 100; - summary1.versions = 5; - - let mut summary2 = SizeSummary::new(); - summary2.total_size = 200; - summary2.versions = 10; - - summary1.add(&summary2); - - assert_eq!(summary1.total_size, 300); - assert_eq!(summary1.versions, 15); - } -} diff --git a/crates/ahm/src/scanner/histogram.rs b/crates/ahm/src/scanner/histogram.rs index 778569ced..f5d7b73bd 100644 --- a/crates/ahm/src/scanner/histogram.rs +++ b/crates/ahm/src/scanner/histogram.rs @@ -12,197 +12,258 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::atomic::{AtomicU64, Ordering}, + time::{Duration, SystemTime}, +}; -/// Size interval for object size histogram -#[derive(Debug, Clone)] -pub struct SizeInterval { - pub start: u64, - pub end: u64, - pub name: &'static str, +use serde::{Deserialize, Serialize}; +use tracing::info; + +/// Scanner metrics +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ScannerMetrics { + /// Total objects scanned since server start + pub objects_scanned: u64, + /// Total object versions scanned since server start + pub versions_scanned: u64, + /// Total directories scanned since server start + pub directories_scanned: u64, + /// Total bucket scans started since server start + pub bucket_scans_started: u64, + /// Total bucket scans finished since server start + pub bucket_scans_finished: u64, + /// Total objects with health issues found + pub objects_with_issues: u64, + /// Total heal tasks queued + pub heal_tasks_queued: u64, + /// Total heal tasks completed + pub heal_tasks_completed: u64, + /// Total heal tasks failed + pub heal_tasks_failed: u64, + /// Total healthy objects found + pub healthy_objects: u64, + /// Total corrupted objects found + pub corrupted_objects: u64, + /// Last scan activity time + pub last_activity: Option, + /// Current scan cycle + pub current_cycle: u64, + /// Total scan cycles completed + pub total_cycles: u64, + /// Current scan duration + pub current_scan_duration: Option, + /// Average scan duration + pub avg_scan_duration: Duration, + /// Objects scanned per second + pub objects_per_second: f64, + /// Buckets scanned per second + pub buckets_per_second: f64, + /// Storage metrics by bucket + pub bucket_metrics: HashMap, + /// Disk metrics + pub disk_metrics: HashMap, } -/// Version interval for object versions histogram -#[derive(Debug, Clone)] -pub struct VersionInterval { - pub start: u64, - pub end: u64, - pub name: &'static str, +/// Bucket-specific metrics +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct BucketMetrics { + /// Bucket name + pub bucket: String, + /// Total objects in bucket + pub total_objects: u64, + /// Total size of objects in bucket (bytes) + pub total_size: u64, + /// Objects with health issues + pub objects_with_issues: u64, + /// Last scan time + pub last_scan_time: Option, + /// Scan duration + pub scan_duration: Option, + /// Heal tasks queued for this bucket + pub heal_tasks_queued: u64, + /// Heal tasks completed for this bucket + pub heal_tasks_completed: u64, + /// Heal tasks failed for this bucket + pub heal_tasks_failed: u64, } -/// Object size histogram intervals -pub const OBJECTS_HISTOGRAM_INTERVALS: &[SizeInterval] = &[ - SizeInterval { - start: 0, - end: 1024 - 1, - name: "LESS_THAN_1_KiB", - }, - SizeInterval { - start: 1024, - end: 1024 * 1024 - 1, - name: "1_KiB_TO_1_MiB", - }, - SizeInterval { - start: 1024 * 1024, - end: 10 * 1024 * 1024 - 1, - name: "1_MiB_TO_10_MiB", - }, - SizeInterval { - start: 10 * 1024 * 1024, - end: 64 * 1024 * 1024 - 1, - name: "10_MiB_TO_64_MiB", - }, - SizeInterval { - start: 64 * 1024 * 1024, - end: 128 * 1024 * 1024 - 1, - name: "64_MiB_TO_128_MiB", - }, - SizeInterval { - start: 128 * 1024 * 1024, - end: 512 * 1024 * 1024 - 1, - name: "128_MiB_TO_512_MiB", - }, - SizeInterval { - start: 512 * 1024 * 1024, - end: u64::MAX, - name: "MORE_THAN_512_MiB", - }, -]; - -/// Object version count histogram intervals -pub const OBJECTS_VERSION_COUNT_INTERVALS: &[VersionInterval] = &[ - VersionInterval { - start: 1, - end: 1, - name: "1_VERSION", - }, - VersionInterval { - start: 2, - end: 10, - name: "2_TO_10_VERSIONS", - }, - VersionInterval { - start: 11, - end: 100, - name: "11_TO_100_VERSIONS", - }, - VersionInterval { - start: 101, - end: 1000, - name: "101_TO_1000_VERSIONS", - }, - VersionInterval { - start: 1001, - end: u64::MAX, - name: "MORE_THAN_1000_VERSIONS", - }, -]; - -/// Size histogram for object size distribution -#[derive(Debug, Clone, Default)] -pub struct SizeHistogram { - counts: Vec, +/// Disk-specific metrics +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct DiskMetrics { + /// Disk path + pub disk_path: String, + /// Total disk space (bytes) + pub total_space: u64, + /// Used disk space (bytes) + pub used_space: u64, + /// Free disk space (bytes) + pub free_space: u64, + /// Objects scanned on this disk + pub objects_scanned: u64, + /// Objects with issues on this disk + pub objects_with_issues: u64, + /// Last scan time + pub last_scan_time: Option, + /// Whether disk is online + pub is_online: bool, + /// Whether disk is being scanned + pub is_scanning: bool, } -/// Versions histogram for object version count distribution -#[derive(Debug, Clone, Default)] -pub struct VersionsHistogram { - counts: Vec, +/// Thread-safe metrics collector +pub struct MetricsCollector { + /// Atomic counters for real-time metrics + objects_scanned: AtomicU64, + versions_scanned: AtomicU64, + directories_scanned: AtomicU64, + bucket_scans_started: AtomicU64, + bucket_scans_finished: AtomicU64, + objects_with_issues: AtomicU64, + heal_tasks_queued: AtomicU64, + heal_tasks_completed: AtomicU64, + heal_tasks_failed: AtomicU64, + current_cycle: AtomicU64, + total_cycles: AtomicU64, + healthy_objects: AtomicU64, + corrupted_objects: AtomicU64, } -impl SizeHistogram { - /// Create a new size histogram +impl MetricsCollector { + /// Create a new metrics collector pub fn new() -> Self { Self { - counts: vec![0; OBJECTS_HISTOGRAM_INTERVALS.len()], + objects_scanned: AtomicU64::new(0), + versions_scanned: AtomicU64::new(0), + directories_scanned: AtomicU64::new(0), + bucket_scans_started: AtomicU64::new(0), + bucket_scans_finished: AtomicU64::new(0), + objects_with_issues: AtomicU64::new(0), + heal_tasks_queued: AtomicU64::new(0), + heal_tasks_completed: AtomicU64::new(0), + heal_tasks_failed: AtomicU64::new(0), + current_cycle: AtomicU64::new(0), + total_cycles: AtomicU64::new(0), + healthy_objects: AtomicU64::new(0), + corrupted_objects: AtomicU64::new(0), } } - /// Add a size to the histogram - pub fn add(&mut self, size: u64) { - for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() { - if size >= interval.start && size <= interval.end { - self.counts[idx] += 1; - break; - } + /// Increment objects scanned count + pub fn increment_objects_scanned(&self, count: u64) { + self.objects_scanned.fetch_add(count, Ordering::Relaxed); + } + + /// Increment versions scanned count + pub fn increment_versions_scanned(&self, count: u64) { + self.versions_scanned.fetch_add(count, Ordering::Relaxed); + } + + /// Increment directories scanned count + pub fn increment_directories_scanned(&self, count: u64) { + self.directories_scanned.fetch_add(count, Ordering::Relaxed); + } + + /// Increment bucket scans started count + pub fn increment_bucket_scans_started(&self, count: u64) { + self.bucket_scans_started.fetch_add(count, Ordering::Relaxed); + } + + /// Increment bucket scans finished count + pub fn increment_bucket_scans_finished(&self, count: u64) { + self.bucket_scans_finished.fetch_add(count, Ordering::Relaxed); + } + + /// Increment objects with issues count + pub fn increment_objects_with_issues(&self, count: u64) { + self.objects_with_issues.fetch_add(count, Ordering::Relaxed); + } + + /// Increment heal tasks queued count + pub fn increment_heal_tasks_queued(&self, count: u64) { + self.heal_tasks_queued.fetch_add(count, Ordering::Relaxed); + } + + /// Increment heal tasks completed count + pub fn increment_heal_tasks_completed(&self, count: u64) { + self.heal_tasks_completed.fetch_add(count, Ordering::Relaxed); + } + + /// Increment heal tasks failed count + pub fn increment_heal_tasks_failed(&self, count: u64) { + self.heal_tasks_failed.fetch_add(count, Ordering::Relaxed); + } + + /// Set current cycle + pub fn set_current_cycle(&self, cycle: u64) { + self.current_cycle.store(cycle, Ordering::Relaxed); + } + + /// Increment total cycles + pub fn increment_total_cycles(&self) { + self.total_cycles.fetch_add(1, Ordering::Relaxed); + } + + /// Increment healthy objects count + pub fn increment_healthy_objects(&self) { + self.healthy_objects.fetch_add(1, Ordering::Relaxed); + } + + /// Increment corrupted objects count + pub fn increment_corrupted_objects(&self) { + self.corrupted_objects.fetch_add(1, Ordering::Relaxed); + } + + /// Get current metrics snapshot + pub fn get_metrics(&self) -> ScannerMetrics { + ScannerMetrics { + objects_scanned: self.objects_scanned.load(Ordering::Relaxed), + versions_scanned: self.versions_scanned.load(Ordering::Relaxed), + directories_scanned: self.directories_scanned.load(Ordering::Relaxed), + bucket_scans_started: self.bucket_scans_started.load(Ordering::Relaxed), + bucket_scans_finished: self.bucket_scans_finished.load(Ordering::Relaxed), + objects_with_issues: self.objects_with_issues.load(Ordering::Relaxed), + heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed), + heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed), + heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed), + healthy_objects: self.healthy_objects.load(Ordering::Relaxed), + corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed), + last_activity: Some(SystemTime::now()), + current_cycle: self.current_cycle.load(Ordering::Relaxed), + total_cycles: self.total_cycles.load(Ordering::Relaxed), + current_scan_duration: None, // Will be set by scanner + avg_scan_duration: Duration::ZERO, // Will be calculated + objects_per_second: 0.0, // Will be calculated + buckets_per_second: 0.0, // Will be calculated + bucket_metrics: HashMap::new(), // Will be populated by scanner + disk_metrics: HashMap::new(), // Will be populated by scanner } } - /// Get the histogram as a map - pub fn to_map(&self) -> HashMap { - let mut result = HashMap::new(); - for (idx, count) in self.counts.iter().enumerate() { - let interval = &OBJECTS_HISTOGRAM_INTERVALS[idx]; - result.insert(interval.name.to_string(), *count); - } - result - } + /// Reset all metrics + pub fn reset(&self) { + self.objects_scanned.store(0, Ordering::Relaxed); + self.versions_scanned.store(0, Ordering::Relaxed); + self.directories_scanned.store(0, Ordering::Relaxed); + self.bucket_scans_started.store(0, Ordering::Relaxed); + self.bucket_scans_finished.store(0, Ordering::Relaxed); + self.objects_with_issues.store(0, Ordering::Relaxed); + self.heal_tasks_queued.store(0, Ordering::Relaxed); + self.heal_tasks_completed.store(0, Ordering::Relaxed); + self.heal_tasks_failed.store(0, Ordering::Relaxed); + self.current_cycle.store(0, Ordering::Relaxed); + self.total_cycles.store(0, Ordering::Relaxed); + self.healthy_objects.store(0, Ordering::Relaxed); + self.corrupted_objects.store(0, Ordering::Relaxed); - /// Merge another histogram into this one - pub fn merge(&mut self, other: &SizeHistogram) { - for (idx, count) in other.counts.iter().enumerate() { - self.counts[idx] += count; - } - } - - /// Get total count - pub fn total_count(&self) -> u64 { - self.counts.iter().sum() - } - - /// Reset the histogram - pub fn reset(&mut self) { - for count in &mut self.counts { - *count = 0; - } + info!("Scanner metrics reset"); } } -impl VersionsHistogram { - /// Create a new versions histogram - pub fn new() -> Self { - Self { - counts: vec![0; OBJECTS_VERSION_COUNT_INTERVALS.len()], - } - } - - /// Add a version count to the histogram - pub fn add(&mut self, versions: u64) { - for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() { - if versions >= interval.start && versions <= interval.end { - self.counts[idx] += 1; - break; - } - } - } - - /// Get the histogram as a map - pub fn to_map(&self) -> HashMap { - let mut result = HashMap::new(); - for (idx, count) in self.counts.iter().enumerate() { - let interval = &OBJECTS_VERSION_COUNT_INTERVALS[idx]; - result.insert(interval.name.to_string(), *count); - } - result - } - - /// Merge another histogram into this one - pub fn merge(&mut self, other: &VersionsHistogram) { - for (idx, count) in other.counts.iter().enumerate() { - self.counts[idx] += count; - } - } - - /// Get total count - pub fn total_count(&self) -> u64 { - self.counts.iter().sum() - } - - /// Reset the histogram - pub fn reset(&mut self) { - for count in &mut self.counts { - *count = 0; - } +impl Default for MetricsCollector { + fn default() -> Self { + Self::new() } } @@ -211,67 +272,35 @@ mod tests { use super::*; #[test] - fn test_size_histogram() { - let mut histogram = SizeHistogram::new(); - - // Add some sizes - histogram.add(512); // LESS_THAN_1_KiB - histogram.add(1024); // 1_KiB_TO_1_MiB - histogram.add(1024 * 1024); // 1_MiB_TO_10_MiB - histogram.add(5 * 1024 * 1024); // 1_MiB_TO_10_MiB - - let map = histogram.to_map(); - - assert_eq!(map.get("LESS_THAN_1_KiB"), Some(&1)); - assert_eq!(map.get("1_KiB_TO_1_MiB"), Some(&1)); - assert_eq!(map.get("1_MiB_TO_10_MiB"), Some(&2)); - assert_eq!(map.get("10_MiB_TO_64_MiB"), Some(&0)); + fn test_metrics_collector_creation() { + let collector = MetricsCollector::new(); + let metrics = collector.get_metrics(); + assert_eq!(metrics.objects_scanned, 0); + assert_eq!(metrics.versions_scanned, 0); } #[test] - fn test_versions_histogram() { - let mut histogram = VersionsHistogram::new(); + fn test_metrics_increment() { + let collector = MetricsCollector::new(); - // Add some version counts - histogram.add(1); // 1_VERSION - histogram.add(5); // 2_TO_10_VERSIONS - histogram.add(50); // 11_TO_100_VERSIONS - histogram.add(500); // 101_TO_1000_VERSIONS + collector.increment_objects_scanned(10); + collector.increment_versions_scanned(5); + collector.increment_objects_with_issues(2); - let map = histogram.to_map(); - - assert_eq!(map.get("1_VERSION"), Some(&1)); - assert_eq!(map.get("2_TO_10_VERSIONS"), Some(&1)); - assert_eq!(map.get("11_TO_100_VERSIONS"), Some(&1)); - assert_eq!(map.get("101_TO_1000_VERSIONS"), Some(&1)); + let metrics = collector.get_metrics(); + assert_eq!(metrics.objects_scanned, 10); + assert_eq!(metrics.versions_scanned, 5); + assert_eq!(metrics.objects_with_issues, 2); } #[test] - fn test_histogram_merge() { - let mut histogram1 = SizeHistogram::new(); - histogram1.add(1024); - histogram1.add(1024 * 1024); + fn test_metrics_reset() { + let collector = MetricsCollector::new(); - let mut histogram2 = SizeHistogram::new(); - histogram2.add(1024); - histogram2.add(5 * 1024 * 1024); + collector.increment_objects_scanned(10); + collector.reset(); - histogram1.merge(&histogram2); - - let map = histogram1.to_map(); - assert_eq!(map.get("1_KiB_TO_1_MiB"), Some(&2)); // 1 from histogram1 + 1 from histogram2 - assert_eq!(map.get("1_MiB_TO_10_MiB"), Some(&2)); // 1 from histogram1 + 1 from histogram2 - } - - #[test] - fn test_histogram_reset() { - let mut histogram = SizeHistogram::new(); - histogram.add(1024); - histogram.add(1024 * 1024); - - assert_eq!(histogram.total_count(), 2); - - histogram.reset(); - assert_eq!(histogram.total_count(), 0); + let metrics = collector.get_metrics(); + assert_eq!(metrics.objects_scanned, 0); } } diff --git a/crates/ahm/src/scanner/metrics.rs b/crates/ahm/src/scanner/metrics.rs index 10d010058..f5d7b73bd 100644 --- a/crates/ahm/src/scanner/metrics.rs +++ b/crates/ahm/src/scanner/metrics.rs @@ -42,6 +42,10 @@ pub struct ScannerMetrics { pub heal_tasks_completed: u64, /// Total heal tasks failed pub heal_tasks_failed: u64, + /// Total healthy objects found + pub healthy_objects: u64, + /// Total corrupted objects found + pub corrupted_objects: u64, /// Last scan activity time pub last_activity: Option, /// Current scan cycle @@ -122,6 +126,8 @@ pub struct MetricsCollector { heal_tasks_failed: AtomicU64, current_cycle: AtomicU64, total_cycles: AtomicU64, + healthy_objects: AtomicU64, + corrupted_objects: AtomicU64, } impl MetricsCollector { @@ -139,6 +145,8 @@ impl MetricsCollector { heal_tasks_failed: AtomicU64::new(0), current_cycle: AtomicU64::new(0), total_cycles: AtomicU64::new(0), + healthy_objects: AtomicU64::new(0), + corrupted_objects: AtomicU64::new(0), } } @@ -197,6 +205,16 @@ impl MetricsCollector { self.total_cycles.fetch_add(1, Ordering::Relaxed); } + /// Increment healthy objects count + pub fn increment_healthy_objects(&self) { + self.healthy_objects.fetch_add(1, Ordering::Relaxed); + } + + /// Increment corrupted objects count + pub fn increment_corrupted_objects(&self) { + self.corrupted_objects.fetch_add(1, Ordering::Relaxed); + } + /// Get current metrics snapshot pub fn get_metrics(&self) -> ScannerMetrics { ScannerMetrics { @@ -209,6 +227,8 @@ impl MetricsCollector { heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed), heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed), heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed), + healthy_objects: self.healthy_objects.load(Ordering::Relaxed), + corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed), last_activity: Some(SystemTime::now()), current_cycle: self.current_cycle.load(Ordering::Relaxed), total_cycles: self.total_cycles.load(Ordering::Relaxed), @@ -234,6 +254,8 @@ impl MetricsCollector { self.heal_tasks_failed.store(0, Ordering::Relaxed); self.current_cycle.store(0, Ordering::Relaxed); self.total_cycles.store(0, Ordering::Relaxed); + self.healthy_objects.store(0, Ordering::Relaxed); + self.corrupted_objects.store(0, Ordering::Relaxed); info!("Scanner metrics reset"); } diff --git a/crates/ahm/src/scanner/mod.rs b/crates/ahm/src/scanner/mod.rs index 025bbb385..d299c1432 100644 --- a/crates/ahm/src/scanner/mod.rs +++ b/crates/ahm/src/scanner/mod.rs @@ -13,13 +13,8 @@ // limitations under the License. pub mod data_scanner; -pub mod data_usage; pub mod histogram; pub mod metrics; -// Re-export main types for convenience pub use data_scanner::Scanner; -pub use data_usage::{ - BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, load_data_usage_from_backend, store_data_usage_in_backend, -}; pub use metrics::ScannerMetrics; diff --git a/crates/ahm/tests/heal_integration_test.rs b/crates/ahm/tests/heal_integration_test.rs new file mode 100644 index 000000000..edda32dc4 --- /dev/null +++ b/crates/ahm/tests/heal_integration_test.rs @@ -0,0 +1,410 @@ +use rustfs_ahm::heal::{ + manager::{HealConfig, HealManager}, + storage::{ECStoreHealStorage, HealStorageAPI}, + task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, +}; +use rustfs_common::heal_channel::{HealOpts, HealScanMode}; +use rustfs_ecstore::{ + disk::endpoint::Endpoint, + endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, + store::ECStore, + store_api::{ObjectIO, ObjectOptions, PutObjReader, StorageAPI}, +}; +use serial_test::serial; +use std::sync::Once; +use std::sync::OnceLock; +use std::{path::PathBuf, sync::Arc, time::Duration}; +use tokio::fs; +use tracing::info; +use walkdir::WalkDir; + +static GLOBAL_ENV: OnceLock<(Vec, Arc, Arc)> = OnceLock::new(); +static INIT: Once = Once::new(); + +fn init_tracing() { + INIT.call_once(|| { + let _ = tracing_subscriber::fmt::try_init(); + }); +} + +/// Test helper: Create test environment with ECStore +async fn setup_test_env() -> (Vec, Arc, Arc) { + init_tracing(); + + // Fast path: already initialized, just clone and return + if let Some((paths, ecstore, heal_storage)) = GLOBAL_ENV.get() { + return (paths.clone(), ecstore.clone(), heal_storage.clone()); + } + + // create temp dir as 4 disks with unique base dir + let test_base_dir = format!("/tmp/rustfs_ahm_heal_test_{}", uuid::Uuid::new_v4()); + let temp_dir = std::path::PathBuf::from(&test_base_dir); + if temp_dir.exists() { + fs::remove_dir_all(&temp_dir).await.ok(); + } + fs::create_dir_all(&temp_dir).await.unwrap(); + + // create 4 disk dirs + let disk_paths = vec![ + temp_dir.join("disk1"), + temp_dir.join("disk2"), + temp_dir.join("disk3"), + temp_dir.join("disk4"), + ]; + + for disk_path in &disk_paths { + fs::create_dir_all(disk_path).await.unwrap(); + } + + // create EndpointServerPools + let mut endpoints = Vec::new(); + for (i, disk_path) in disk_paths.iter().enumerate() { + let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); + // set correct index + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + endpoints.push(endpoint); + } + + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "test".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }; + + let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); + + // format disks (only first time) + rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap(); + + // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free + let port = 9001; // for simplicity + let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap(); + let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap(); + + // init bucket metadata system + let buckets_list = ecstore + .list_bucket(&rustfs_ecstore::store_api::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .unwrap(); + let buckets = buckets_list.into_iter().map(|v| v.name).collect(); + rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; + + // Create heal storage layer + let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone())); + + // Store in global once lock + let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone(), heal_storage.clone())); + + (disk_paths, ecstore, heal_storage) +} + +/// Test helper: Create a test bucket +async fn create_test_bucket(ecstore: &Arc, bucket_name: &str) { + (**ecstore) + .make_bucket(bucket_name, &Default::default()) + .await + .expect("Failed to create test bucket"); + info!("Created test bucket: {}", bucket_name); +} + +/// Test helper: Upload test object +async fn upload_test_object(ecstore: &Arc, bucket: &str, object: &str, data: &[u8]) { + let mut reader = PutObjReader::from_vec(data.to_vec()); + let object_info = (**ecstore) + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("Failed to upload test object"); + + info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_heal_object_basic() { + let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + + // Create test bucket and object + let bucket_name = "test-bucket"; + let object_name = "test-object.txt"; + let test_data = b"Hello, this is test data for healing!"; + + create_test_bucket(&ecstore, bucket_name).await; + upload_test_object(&ecstore, bucket_name, object_name, test_data).await; + + // ─── 1️⃣ delete single data shard file ───────────────────────────────────── + let obj_dir = disk_paths[0].join(bucket_name).join(object_name); + // find part file at depth 2, e.g. ...//part.1 + let target_part = WalkDir::new(&obj_dir) + .min_depth(2) + .max_depth(2) + .into_iter() + .filter_map(Result::ok) + .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false)) + .map(|e| e.into_path()) + .expect("Failed to locate part file to delete"); + + std::fs::remove_file(&target_part).expect("failed to delete part file"); + assert!(!target_part.exists()); + println!("✅ Deleted shard part file: {target_part:?}"); + + // Create heal manager with faster interval + let cfg = HealConfig { + heal_interval: Duration::from_millis(1), + ..Default::default() + }; + let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg)); + heal_manager.start().await.unwrap(); + + // Submit heal request for the object + let heal_request = HealRequest::new( + HealType::Object { + bucket: bucket_name.to_string(), + object: object_name.to_string(), + version_id: None, + }, + HealOptions { + dry_run: false, + recursive: false, + remove_corrupted: false, + recreate_missing: true, + scan_mode: HealScanMode::Normal, + update_parity: true, + timeout: Some(Duration::from_secs(300)), + pool_index: None, + set_index: None, + }, + HealPriority::Normal, + ); + + let task_id = heal_manager + .submit_heal_request(heal_request) + .await + .expect("Failed to submit heal request"); + + info!("Submitted heal request with task ID: {}", task_id); + + // Wait for task completion + tokio::time::sleep(tokio::time::Duration::from_secs(8)).await; + + // Attempt to fetch task status (might be removed if finished) + match heal_manager.get_task_status(&task_id).await { + Ok(status) => info!("Task status: {:?}", status), + Err(e) => info!("Task status not found (likely completed): {}", e), + } + + // ─── 2️⃣ verify each part file is restored ─────── + assert!(target_part.exists()); + + info!("Heal object basic test passed"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_heal_bucket_basic() { + let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + + // Create test bucket + let bucket_name = "test-bucket-heal"; + create_test_bucket(&ecstore, bucket_name).await; + + // ─── 1️⃣ delete bucket dir on disk ────────────── + let broken_bucket_path = disk_paths[0].join(bucket_name); + assert!(broken_bucket_path.exists(), "bucket dir does not exist on disk"); + std::fs::remove_dir_all(&broken_bucket_path).expect("failed to delete bucket dir on disk"); + assert!(!broken_bucket_path.exists(), "bucket dir still exists after deletion"); + println!("✅ Deleted bucket directory on disk: {broken_bucket_path:?}"); + + // Create heal manager with faster interval + let cfg = HealConfig { + heal_interval: Duration::from_millis(1), + ..Default::default() + }; + let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg)); + heal_manager.start().await.unwrap(); + + // Submit heal request for the bucket + let heal_request = HealRequest::new( + HealType::Bucket { + bucket: bucket_name.to_string(), + }, + HealOptions { + dry_run: false, + recursive: true, + remove_corrupted: false, + recreate_missing: false, + scan_mode: HealScanMode::Normal, + update_parity: false, + timeout: Some(Duration::from_secs(300)), + pool_index: None, + set_index: None, + }, + HealPriority::Normal, + ); + + let task_id = heal_manager + .submit_heal_request(heal_request) + .await + .expect("Failed to submit bucket heal request"); + + info!("Submitted bucket heal request with task ID: {}", task_id); + + // Wait for task completion + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // Attempt to fetch task status (optional) + if let Ok(status) = heal_manager.get_task_status(&task_id).await { + if status == HealTaskStatus::Completed { + info!("Bucket heal task status: {:?}", status); + } else { + panic!("Bucket heal task status: {status:?}"); + } + } + + // ─── 3️⃣ Verify bucket directory is restored on every disk ─────── + assert!(broken_bucket_path.exists(), "bucket dir does not exist on disk"); + + info!("Heal bucket basic test passed"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_heal_format_basic() { + let (disk_paths, _ecstore, heal_storage) = setup_test_env().await; + + // ─── 1️⃣ delete format.json on one disk ────────────── + let format_path = disk_paths[0].join(".rustfs.sys").join("format.json"); + assert!(format_path.exists(), "format.json does not exist on disk"); + std::fs::remove_file(&format_path).expect("failed to delete format.json on disk"); + assert!(!format_path.exists(), "format.json still exists after deletion"); + println!("✅ Deleted format.json on disk: {format_path:?}"); + + // Create heal manager with faster interval + let cfg = HealConfig { + heal_interval: Duration::from_secs(2), + ..Default::default() + }; + let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg)); + heal_manager.start().await.unwrap(); + + // Wait for task completion + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // ─── 2️⃣ verify format.json is restored ─────── + assert!(format_path.exists(), "format.json does not exist on disk after heal"); + + info!("Heal format basic test passed"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_heal_format_with_data() { + let (disk_paths, ecstore, heal_storage) = setup_test_env().await; + + // Create test bucket and object + let bucket_name = "test-bucket"; + let object_name = "test-object.txt"; + let test_data = b"Hello, this is test data for healing!"; + + create_test_bucket(&ecstore, bucket_name).await; + upload_test_object(&ecstore, bucket_name, object_name, test_data).await; + + let obj_dir = disk_paths[0].join(bucket_name).join(object_name); + let target_part = WalkDir::new(&obj_dir) + .min_depth(2) + .max_depth(2) + .into_iter() + .filter_map(Result::ok) + .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false)) + .map(|e| e.into_path()) + .expect("Failed to locate part file to delete"); + + // ─── 1️⃣ delete format.json on one disk ────────────── + let format_path = disk_paths[0].join(".rustfs.sys").join("format.json"); + std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]"); + std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory"); + println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]); + + // Create heal manager with faster interval + let cfg = HealConfig { + heal_interval: Duration::from_secs(2), + ..Default::default() + }; + let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg)); + heal_manager.start().await.unwrap(); + + // Wait for task completion + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // ─── 2️⃣ verify format.json is restored ─────── + assert!(format_path.exists(), "format.json does not exist on disk after heal"); + // ─── 3 verify each part file is restored ─────── + assert!(target_part.exists()); + + info!("Heal format basic test passed"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_heal_storage_api_direct() { + let (_disk_paths, ecstore, heal_storage) = setup_test_env().await; + + // Test direct heal storage API calls + + // Test heal_format + let format_result = heal_storage.heal_format(true).await; // dry run + assert!(format_result.is_ok()); + info!("Direct heal_format test passed"); + + // Test heal_bucket + let bucket_name = "test-bucket-direct"; + create_test_bucket(&ecstore, bucket_name).await; + + let heal_opts = HealOpts { + recursive: true, + dry_run: true, + remove: false, + recreate: false, + scan_mode: HealScanMode::Normal, + update_parity: false, + no_lock: false, + pool: None, + set: None, + }; + + let bucket_result = heal_storage.heal_bucket(bucket_name, &heal_opts).await; + assert!(bucket_result.is_ok()); + info!("Direct heal_bucket test passed"); + + // Test heal_object + let object_name = "test-object-direct.txt"; + let test_data = b"Test data for direct heal API"; + upload_test_object(&ecstore, bucket_name, object_name, test_data).await; + + let object_heal_opts = HealOpts { + recursive: false, + dry_run: true, + remove: false, + recreate: false, + scan_mode: HealScanMode::Normal, + update_parity: false, + no_lock: false, + pool: None, + set: None, + }; + + let object_result = heal_storage + .heal_object(bucket_name, object_name, None, &object_heal_opts) + .await; + assert!(object_result.is_ok()); + info!("Direct heal_object test passed"); + + info!("Direct heal storage API test passed"); +} diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index e8bcb3702..a2b57c663 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -28,5 +28,15 @@ categories = ["web-programming", "development-tools", "data-structures"] workspace = true [dependencies] -tokio.workspace = true +lazy_static = { workspace = true} +tokio = { workspace = true } tonic = { workspace = true } +uuid = { workspace = true } +chrono = { workspace = true } +rustfs-madmin = { workspace = true } +rustfs-filemeta = { workspace = true } +serde = { workspace = true } +path-clean = { workspace = true } +rmp-serde = { workspace = true } +async-trait = { workspace = true } +s3s = { workspace = true } diff --git a/crates/common/src/data_usage.rs b/crates/common/src/data_usage.rs new file mode 100644 index 000000000..b9b93b690 --- /dev/null +++ b/crates/common/src/data_usage.rs @@ -0,0 +1,1281 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use path_clean::PathClean; +use serde::{Deserialize, Serialize}; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::path::Path; +use std::{ + collections::{HashMap, HashSet}, + time::SystemTime, +}; + +#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)] +pub struct TierStats { + pub total_size: u64, + pub num_versions: i32, + pub num_objects: i32, +} + +impl TierStats { + pub fn add(&self, u: &TierStats) -> TierStats { + TierStats { + total_size: self.total_size + u.total_size, + num_versions: self.num_versions + u.num_versions, + num_objects: self.num_objects + u.num_objects, + } + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct AllTierStats { + pub tiers: HashMap, +} + +impl AllTierStats { + pub fn new() -> Self { + Self { tiers: HashMap::new() } + } + + pub fn add_sizes(&mut self, tiers: HashMap) { + for (tier, st) in tiers { + self.tiers + .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st)); + } + } + + pub fn merge(&mut self, other: AllTierStats) { + for (tier, st) in other.tiers { + self.tiers + .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st)); + } + } + + pub fn populate_stats(&self, stats: &mut HashMap) { + for (tier, st) in &self.tiers { + stats.insert( + tier.clone(), + TierStats { + total_size: st.total_size, + num_versions: st.num_versions, + num_objects: st.num_objects, + }, + ); + } + } +} + +/// Bucket target usage info provides replication statistics +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct BucketTargetUsageInfo { + pub replication_pending_size: u64, + pub replication_failed_size: u64, + pub replicated_size: u64, + pub replica_size: u64, + pub replication_pending_count: u64, + pub replication_failed_count: u64, + pub replicated_count: u64, +} + +/// Bucket usage info provides bucket-level statistics +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct BucketUsageInfo { + pub size: u64, + // Following five fields suffixed with V1 are here for backward compatibility + // Total Size for objects that have not yet been replicated + pub replication_pending_size_v1: u64, + // Total size for objects that have witness one or more failures and will be retried + pub replication_failed_size_v1: u64, + // Total size for objects that have been replicated to destination + pub replicated_size_v1: u64, + // Total number of objects pending replication + pub replication_pending_count_v1: u64, + // Total number of objects that failed replication + pub replication_failed_count_v1: u64, + + pub objects_count: u64, + pub object_size_histogram: HashMap, + pub object_versions_histogram: HashMap, + pub versions_count: u64, + pub delete_markers_count: u64, + pub replica_size: u64, + pub replica_count: u64, + pub replication_info: HashMap, +} + +/// DataUsageInfo represents data usage stats of the underlying storage +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct DataUsageInfo { + /// Total capacity + pub total_capacity: u64, + /// Total used capacity + pub total_used_capacity: u64, + /// Total free capacity + pub total_free_capacity: u64, + + /// LastUpdate is the timestamp of when the data usage info was last updated + pub last_update: Option, + + /// Objects total count across all buckets + pub objects_total_count: u64, + /// Versions total count across all buckets + pub versions_total_count: u64, + /// Delete markers total count across all buckets + pub delete_markers_total_count: u64, + /// Objects total size across all buckets + pub objects_total_size: u64, + /// Replication info across all buckets + pub replication_info: HashMap, + + /// Total number of buckets in this cluster + pub buckets_count: u64, + /// Buckets usage info provides following information across all buckets + pub buckets_usage: HashMap, + /// Deprecated kept here for backward compatibility reasons + pub bucket_sizes: HashMap, +} + +/// Size summary for a single object or group of objects +#[derive(Debug, Default, Clone)] +pub struct SizeSummary { + /// Total size + pub total_size: usize, + /// Number of versions + pub versions: usize, + /// Number of delete markers + pub delete_markers: usize, + /// Replicated size + pub replicated_size: usize, + /// Replicated count + pub replicated_count: usize, + /// Pending size + pub pending_size: usize, + /// Failed size + pub failed_size: usize, + /// Replica size + pub replica_size: usize, + /// Replica count + pub replica_count: usize, + /// Pending count + pub pending_count: usize, + /// Failed count + pub failed_count: usize, + /// Replication target stats + pub repl_target_stats: HashMap, +} + +/// Replication target size summary +#[derive(Debug, Default, Clone)] +pub struct ReplTargetSizeSummary { + /// Replicated size + pub replicated_size: usize, + /// Replicated count + pub replicated_count: usize, + /// Pending size + pub pending_size: usize, + /// Failed size + pub failed_size: usize, + /// Pending count + pub pending_count: usize, + /// Failed count + pub failed_count: usize, +} + +// ===== 缓存相关数据结构 ===== + +/// Data usage hash for path-based caching +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DataUsageHash(pub String); + +impl DataUsageHash { + pub fn string(&self) -> String { + self.0.clone() + } + + pub fn key(&self) -> String { + self.0.clone() + } + + pub fn mod_(&self, cycle: u32, cycles: u32) -> bool { + if cycles <= 1 { + return cycles == 1; + } + + let hash = self.calculate_hash(); + hash as u32 % cycles == cycle % cycles + } + + pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool { + if cycles <= 1 { + return cycles == 1; + } + + let hash = self.calculate_hash(); + (hash >> 32) as u32 % cycles == cycle % cycles + } + + fn calculate_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } +} + +/// Data usage hash map type +pub type DataUsageHashMap = HashSet; + +/// Size histogram for object size distribution +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SizeHistogram(Vec); + +impl Default for SizeHistogram { + fn default() -> Self { + Self(vec![0; 11]) // DATA_USAGE_BUCKET_LEN = 11 + } +} + +impl SizeHistogram { + pub fn add(&mut self, size: u64) { + let intervals = [ + (0, 1024), // LESS_THAN_1024_B + (1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB + (64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB + (256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB + (512 * 1024, 1024 * 1024 - 1), // BETWEEN_512_KB_AND_1_MB + (1024, 1024 * 1024 - 1), // BETWEEN_1024B_AND_1_MB + (1024 * 1024, 10 * 1024 * 1024 - 1), // BETWEEN_1_MB_AND_10_MB + (10 * 1024 * 1024, 64 * 1024 * 1024 - 1), // BETWEEN_10_MB_AND_64_MB + (64 * 1024 * 1024, 128 * 1024 * 1024 - 1), // BETWEEN_64_MB_AND_128_MB + (128 * 1024 * 1024, 512 * 1024 * 1024 - 1), // BETWEEN_128_MB_AND_512_MB + (512 * 1024 * 1024, u64::MAX), // GREATER_THAN_512_MB + ]; + + for (idx, (start, end)) in intervals.iter().enumerate() { + if size >= *start && size <= *end { + self.0[idx] += 1; + break; + } + } + } + + pub fn to_map(&self) -> HashMap { + let names = [ + "LESS_THAN_1024_B", + "BETWEEN_1024_B_AND_64_KB", + "BETWEEN_64_KB_AND_256_KB", + "BETWEEN_256_KB_AND_512_KB", + "BETWEEN_512_KB_AND_1_MB", + "BETWEEN_1024B_AND_1_MB", + "BETWEEN_1_MB_AND_10_MB", + "BETWEEN_10_MB_AND_64_MB", + "BETWEEN_64_MB_AND_128_MB", + "BETWEEN_128_MB_AND_512_MB", + "GREATER_THAN_512_MB", + ]; + + let mut res = HashMap::new(); + let mut spl_count = 0; + for (count, name) in self.0.iter().zip(names.iter()) { + if name == &"BETWEEN_1024B_AND_1_MB" { + res.insert(name.to_string(), spl_count); + } else if name.starts_with("BETWEEN_") && name.contains("_KB_") && name.contains("_MB") { + spl_count += count; + res.insert(name.to_string(), *count); + } else { + res.insert(name.to_string(), *count); + } + } + res + } +} + +/// Versions histogram for version count distribution +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct VersionsHistogram(Vec); + +impl Default for VersionsHistogram { + fn default() -> Self { + Self(vec![0; 7]) // DATA_USAGE_VERSION_LEN = 7 + } +} + +impl VersionsHistogram { + pub fn add(&mut self, count: u64) { + let intervals = [ + (0, 0), // UNVERSIONED + (1, 1), // SINGLE_VERSION + (2, 9), // BETWEEN_2_AND_10 + (10, 99), // BETWEEN_10_AND_100 + (100, 999), // BETWEEN_100_AND_1000 + (1000, 9999), // BETWEEN_1000_AND_10000 + (10000, u64::MAX), // GREATER_THAN_10000 + ]; + + for (idx, (start, end)) in intervals.iter().enumerate() { + if count >= *start && count <= *end { + self.0[idx] += 1; + break; + } + } + } + + pub fn to_map(&self) -> HashMap { + let names = [ + "UNVERSIONED", + "SINGLE_VERSION", + "BETWEEN_2_AND_10", + "BETWEEN_10_AND_100", + "BETWEEN_100_AND_1000", + "BETWEEN_1000_AND_10000", + "GREATER_THAN_10000", + ]; + + let mut res = HashMap::new(); + for (count, name) in self.0.iter().zip(names.iter()) { + res.insert(name.to_string(), *count); + } + res + } +} + +/// Replication statistics for a single target +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ReplicationStats { + pub pending_size: u64, + pub replicated_size: u64, + pub failed_size: u64, + pub failed_count: u64, + pub pending_count: u64, + pub missed_threshold_size: u64, + pub after_threshold_size: u64, + pub missed_threshold_count: u64, + pub after_threshold_count: u64, + pub replicated_count: u64, +} + +impl ReplicationStats { + pub fn empty(&self) -> bool { + self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0 + } +} + +/// Replication statistics for all targets +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ReplicationAllStats { + pub targets: HashMap, + pub replica_size: u64, + pub replica_count: u64, +} + +impl ReplicationAllStats { + pub fn empty(&self) -> bool { + if self.replica_size != 0 && self.replica_count != 0 { + return false; + } + for (_, v) in self.targets.iter() { + if !v.empty() { + return false; + } + } + true + } +} + +/// Data usage cache entry +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DataUsageEntry { + pub children: DataUsageHashMap, + // These fields do not include any children. + pub size: usize, + pub objects: usize, + pub versions: usize, + pub delete_markers: usize, + pub obj_sizes: SizeHistogram, + pub obj_versions: VersionsHistogram, + pub replication_stats: Option, + pub compacted: bool, +} + +impl DataUsageEntry { + pub fn add_child(&mut self, hash: &DataUsageHash) { + if self.children.contains(&hash.key()) { + return; + } + self.children.insert(hash.key()); + } + + pub fn add_sizes(&mut self, summary: &SizeSummary) { + self.size += summary.total_size; + self.versions += summary.versions; + self.delete_markers += summary.delete_markers; + self.obj_sizes.add(summary.total_size as u64); + self.obj_versions.add(summary.versions as u64); + + let replication_stats = if self.replication_stats.is_none() { + self.replication_stats = Some(ReplicationAllStats::default()); + self.replication_stats.as_mut().unwrap() + } else { + self.replication_stats.as_mut().unwrap() + }; + replication_stats.replica_size += summary.replica_size as u64; + replication_stats.replica_count += summary.replica_count as u64; + + for (arn, st) in &summary.repl_target_stats { + let tgt_stat = replication_stats + .targets + .entry(arn.to_string()) + .or_insert(ReplicationStats::default()); + tgt_stat.pending_size += st.pending_size as u64; + tgt_stat.failed_size += st.failed_size as u64; + tgt_stat.replicated_size += st.replicated_size as u64; + tgt_stat.replicated_count += st.replicated_count as u64; + tgt_stat.failed_count += st.failed_count as u64; + tgt_stat.pending_count += st.pending_count as u64; + } + } + + pub fn merge(&mut self, other: &DataUsageEntry) { + self.objects += other.objects; + self.versions += other.versions; + self.delete_markers += other.delete_markers; + self.size += other.size; + + if let Some(o_rep) = &other.replication_stats { + if self.replication_stats.is_none() { + self.replication_stats = Some(ReplicationAllStats::default()); + } + let s_rep = self.replication_stats.as_mut().unwrap(); + s_rep.targets.clear(); + s_rep.replica_size += o_rep.replica_size; + s_rep.replica_count += o_rep.replica_count; + for (arn, stat) in o_rep.targets.iter() { + let st = s_rep.targets.entry(arn.clone()).or_default(); + *st = ReplicationStats { + pending_size: stat.pending_size + st.pending_size, + failed_size: stat.failed_size + st.failed_size, + replicated_size: stat.replicated_size + st.replicated_size, + pending_count: stat.pending_count + st.pending_count, + failed_count: stat.failed_count + st.failed_count, + replicated_count: stat.replicated_count + st.replicated_count, + ..Default::default() + }; + } + } + + for (i, v) in other.obj_sizes.0.iter().enumerate() { + self.obj_sizes.0[i] += v; + } + + for (i, v) in other.obj_versions.0.iter().enumerate() { + self.obj_versions.0[i] += v; + } + } +} + +/// Data usage cache info +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DataUsageCacheInfo { + pub name: String, + pub next_cycle: u32, + pub last_update: Option, + pub skip_healing: bool, +} + +/// Data usage cache +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DataUsageCache { + pub info: DataUsageCacheInfo, + pub cache: HashMap, +} + +impl DataUsageCache { + pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) { + let hash = hash_path(path); + self.cache.insert(hash.key(), e); + if !parent.is_empty() { + let phash = hash_path(parent); + let p = { + let p = self.cache.entry(phash.key()).or_default(); + p.add_child(&hash); + p.clone() + }; + self.cache.insert(phash.key(), p); + } + } + + pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option, e: &DataUsageEntry) { + self.cache.insert(hash.key(), e.clone()); + if let Some(parent) = parent { + self.cache.entry(parent.key()).or_default().add_child(hash); + } + } + + pub fn find(&self, path: &str) -> Option { + self.cache.get(&hash_path(path).key()).cloned() + } + + pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { + self.cache.entry(h.string()).or_default().children.clone() + } + + pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { + let mut root = root.clone(); + for id in root.children.clone().iter() { + if let Some(e) = self.cache.get(id) { + let mut e = e.clone(); + if !e.children.is_empty() { + e = self.flatten(&e); + } + root.merge(&e); + } + } + root.children.clear(); + root + } + + pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option) { + if let Some(e) = src.cache.get(&hash.string()) { + self.cache.insert(hash.key(), e.clone()); + for ch in e.children.iter() { + if *ch == hash.key() { + return; + } + self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone())); + } + if let Some(parent) = parent { + let p = self.cache.entry(parent.key()).or_default(); + p.add_child(hash); + } + } + } + + pub fn delete_recursive(&mut self, hash: &DataUsageHash) { + let mut need_remove = Vec::new(); + if let Some(v) = self.cache.get(&hash.string()) { + for child in v.children.iter() { + need_remove.push(child.clone()); + } + } + self.cache.remove(&hash.string()); + need_remove.iter().for_each(|child| { + self.delete_recursive(&DataUsageHash(child.to_string())); + }); + } + + pub fn size_recursive(&self, path: &str) -> Option { + match self.find(path) { + Some(root) => { + if root.children.is_empty() { + return Some(root); + } + let mut flat = self.flatten(&root); + if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() { + flat.replication_stats = None; + } + Some(flat) + } + None => None, + } + } + + pub fn search_parent(&self, hash: &DataUsageHash) -> Option { + let want = hash.key(); + if let Some(last_index) = want.rfind('/') { + if let Some(v) = self.find(&want[0..last_index]) { + if v.children.contains(&want) { + let found = hash_path(&want[0..last_index]); + return Some(found); + } + } + } + + for (k, v) in self.cache.iter() { + if v.children.contains(&want) { + let found = DataUsageHash(k.clone()); + return Some(found); + } + } + None + } + + pub fn is_compacted(&self, hash: &DataUsageHash) -> bool { + match self.cache.get(&hash.key()) { + Some(due) => due.compacted, + None => false, + } + } + + pub fn force_compact(&mut self, limit: usize) { + if self.cache.len() < limit { + return; + } + let top = hash_path(&self.info.name).key(); + let top_e = match self.find(&top) { + Some(e) => e, + None => return, + }; + // Note: DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS constant would need to be passed as parameter + // or defined in common crate if needed + if top_e.children.len() > 250_000 { + // DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS + self.reduce_children_of(&hash_path(&self.info.name), limit, true); + } + if self.cache.len() <= limit { + return; + } + + let mut found = HashSet::new(); + found.insert(top); + mark(self, &top_e, &mut found); + self.cache.retain(|k, _| { + if !found.contains(k) { + return false; + } + true + }); + } + + pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) { + let e = match self.cache.get(&path.key()) { + Some(e) => e, + None => return, + }; + + if e.compacted { + return; + } + + if e.children.len() > limit && compact_self { + let mut flat = self.size_recursive(&path.key()).unwrap_or_default(); + flat.compacted = true; + self.delete_recursive(path); + self.replace_hashed(path, &None, &flat); + return; + } + let total = self.total_children_rec(&path.key()); + if total < limit { + return; + } + + let mut leaves = Vec::new(); + let mut remove = total - limit; + add(self, path, &mut leaves); + leaves.sort_by(|a, b| a.objects.cmp(&b.objects)); + + while remove > 0 && !leaves.is_empty() { + let e = leaves.first().unwrap(); + let candidate = e.path.clone(); + if candidate == *path && !compact_self { + break; + } + let removing = self.total_children_rec(&candidate.key()); + let mut flat = match self.size_recursive(&candidate.key()) { + Some(flat) => flat, + None => { + leaves.remove(0); + continue; + } + }; + + flat.compacted = true; + self.delete_recursive(&candidate); + self.replace_hashed(&candidate, &None, &flat); + + remove -= removing; + leaves.remove(0); + } + } + + pub fn total_children_rec(&self, path: &str) -> usize { + let root = self.find(path); + + if root.is_none() { + return 0; + } + let root = root.unwrap(); + if root.children.is_empty() { + return 0; + } + + let mut n = root.children.len(); + for ch in root.children.iter() { + n += self.total_children_rec(ch); + } + n + } + + pub fn merge(&mut self, o: &DataUsageCache) { + let mut existing_root = self.root(); + let other_root = o.root(); + if existing_root.is_none() && other_root.is_none() { + return; + } + if other_root.is_none() { + return; + } + if existing_root.is_none() { + *self = o.clone(); + return; + } + if o.info.last_update.gt(&self.info.last_update) { + self.info.last_update = o.info.last_update; + } + + existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap()); + self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap()); + let e_hash = self.root_hash(); + for key in other_root.as_ref().unwrap().children.iter() { + let entry = &o.cache[key]; + let flat = o.flatten(entry); + let mut existing = self.cache[key].clone(); + existing.merge(&flat); + self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing); + } + } + + pub fn root_hash(&self) -> DataUsageHash { + hash_path(&self.info.name) + } + + pub fn root(&self) -> Option { + self.find(&self.info.name) + } + + /// Convert cache to DataUsageInfo for a specific path + pub fn dui(&self, path: &str, buckets: &[String]) -> DataUsageInfo { + let e = match self.find(path) { + Some(e) => e, + None => return DataUsageInfo::default(), + }; + let flat = self.flatten(&e); + + let mut buckets_usage = HashMap::new(); + for bucket_name in buckets.iter() { + let e = match self.find(bucket_name) { + Some(e) => e, + None => continue, + }; + let flat = self.flatten(&e); + let mut bui = BucketUsageInfo { + size: flat.size as u64, + versions_count: flat.versions as u64, + objects_count: flat.objects as u64, + delete_markers_count: flat.delete_markers as u64, + object_size_histogram: flat.obj_sizes.to_map(), + object_versions_histogram: flat.obj_versions.to_map(), + ..Default::default() + }; + + if let Some(rs) = &flat.replication_stats { + bui.replica_size = rs.replica_size; + bui.replica_count = rs.replica_count; + + for (arn, stat) in rs.targets.iter() { + bui.replication_info.insert( + arn.clone(), + BucketTargetUsageInfo { + replication_pending_size: stat.pending_size, + replicated_size: stat.replicated_size, + replication_failed_size: stat.failed_size, + replication_pending_count: stat.pending_count, + replication_failed_count: stat.failed_count, + replicated_count: stat.replicated_count, + ..Default::default() + }, + ); + } + } + buckets_usage.insert(bucket_name.clone(), bui); + } + + DataUsageInfo { + last_update: self.info.last_update, + objects_total_count: flat.objects as u64, + versions_total_count: flat.versions as u64, + delete_markers_total_count: flat.delete_markers as u64, + objects_total_size: flat.size as u64, + buckets_count: e.children.len() as u64, + buckets_usage, + ..Default::default() + } + } + + pub fn marshal_msg(&self) -> Result, Box> { + let mut buf = Vec::new(); + self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?; + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result> { + let t: Self = rmp_serde::from_slice(buf)?; + Ok(t) + } + + // Note: load and save methods are storage-specific and should be implemented + // in the ecstore crate where storage access is available +} + +/// Trait for storage-specific operations on DataUsageCache +#[async_trait::async_trait] +pub trait DataUsageCacheStorage { + /// Load data usage cache from backend storage + async fn load(store: &dyn std::any::Any, name: &str) -> Result> + where + Self: Sized; + + /// Save data usage cache to backend storage + async fn save(&self, name: &str) -> Result<(), Box>; +} + +// Helper structs and functions for cache operations +#[derive(Default, Clone)] +struct Inner { + objects: usize, + path: DataUsageHash, +} + +fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec) { + let e = match data_usage_cache.cache.get(&path.key()) { + Some(e) => e, + None => return, + }; + if !e.children.is_empty() { + return; + } + + let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default(); + leaves.push(Inner { + objects: sz.objects, + path: path.clone(), + }); + for ch in e.children.iter() { + add(data_usage_cache, &DataUsageHash(ch.clone()), leaves); + } +} + +fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet) { + for k in entry.children.iter() { + found.insert(k.to_string()); + if let Some(ch) = duc.cache.get(k) { + mark(duc, ch, found); + } + } +} + +/// Hash a path for data usage caching +pub fn hash_path(data: &str) -> DataUsageHash { + DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string()) +} + +impl DataUsageInfo { + /// Create a new DataUsageInfo + pub fn new() -> Self { + Self::default() + } + + /// Add object metadata to data usage statistics + pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) { + // This method is kept for backward compatibility + // For accurate version counting, use add_object_from_file_meta instead + let bucket_name = match self.extract_bucket_from_path(object_path) { + Ok(name) => name, + Err(_) => return, + }; + + // Update bucket statistics + if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { + bucket_usage.size += meta_object.size as u64; + bucket_usage.objects_count += 1; + bucket_usage.versions_count += 1; // Simplified: assume 1 version per object + + // Update size histogram + let total_size = meta_object.size as u64; + let size_ranges = [ + ("0-1KB", 0, 1024), + ("1KB-1MB", 1024, 1024 * 1024), + ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), + ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), + ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), + ("1GB+", 1024 * 1024 * 1024, u64::MAX), + ]; + + for (range_name, min_size, max_size) in size_ranges { + if total_size >= min_size && total_size < max_size { + *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; + break; + } + } + + // Update version histogram (simplified - count as single version) + *bucket_usage + .object_versions_histogram + .entry("SINGLE_VERSION".to_string()) + .or_insert(0) += 1; + } else { + // Create new bucket usage + let mut bucket_usage = BucketUsageInfo { + size: meta_object.size as u64, + objects_count: 1, + versions_count: 1, + ..Default::default() + }; + bucket_usage.object_size_histogram.insert("0-1KB".to_string(), 1); + bucket_usage.object_versions_histogram.insert("SINGLE_VERSION".to_string(), 1); + self.buckets_usage.insert(bucket_name, bucket_usage); + } + + // Update global statistics + self.objects_total_size += meta_object.size as u64; + self.objects_total_count += 1; + self.versions_total_count += 1; + } + + /// Add object from FileMeta for accurate version counting + pub fn add_object_from_file_meta(&mut self, object_path: &str, file_meta: &rustfs_filemeta::FileMeta) { + let bucket_name = match self.extract_bucket_from_path(object_path) { + Ok(name) => name, + Err(_) => return, + }; + + // Calculate accurate statistics from all versions + let mut total_size = 0u64; + let mut versions_count = 0u64; + let mut delete_markers_count = 0u64; + let mut latest_object_size = 0u64; + + // Process all versions to get accurate counts + for version in &file_meta.versions { + match rustfs_filemeta::FileMetaVersion::try_from(version.clone()) { + Ok(ver) => { + if let Some(obj) = ver.object { + total_size += obj.size as u64; + versions_count += 1; + latest_object_size = obj.size as u64; // Keep track of latest object size + } else if ver.delete_marker.is_some() { + delete_markers_count += 1; + } + } + Err(_) => { + // Skip invalid versions + continue; + } + } + } + + // Update bucket statistics + if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { + bucket_usage.size += total_size; + bucket_usage.objects_count += 1; + bucket_usage.versions_count += versions_count; + bucket_usage.delete_markers_count += delete_markers_count; + + // Update size histogram based on latest object size + let size_ranges = [ + ("0-1KB", 0, 1024), + ("1KB-1MB", 1024, 1024 * 1024), + ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), + ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), + ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), + ("1GB+", 1024 * 1024 * 1024, u64::MAX), + ]; + + for (range_name, min_size, max_size) in size_ranges { + if latest_object_size >= min_size && latest_object_size < max_size { + *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; + break; + } + } + + // Update version histogram based on actual version count + let version_ranges = [ + ("1", 1, 1), + ("2-5", 2, 5), + ("6-10", 6, 10), + ("11-50", 11, 50), + ("51-100", 51, 100), + ("100+", 101, usize::MAX), + ]; + + for (range_name, min_versions, max_versions) in version_ranges { + if versions_count as usize >= min_versions && versions_count as usize <= max_versions { + *bucket_usage + .object_versions_histogram + .entry(range_name.to_string()) + .or_insert(0) += 1; + break; + } + } + } else { + // Create new bucket usage + let mut bucket_usage = BucketUsageInfo { + size: total_size, + objects_count: 1, + versions_count, + delete_markers_count, + ..Default::default() + }; + + // Set size histogram + let size_ranges = [ + ("0-1KB", 0, 1024), + ("1KB-1MB", 1024, 1024 * 1024), + ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), + ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), + ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), + ("1GB+", 1024 * 1024 * 1024, u64::MAX), + ]; + + for (range_name, min_size, max_size) in size_ranges { + if latest_object_size >= min_size && latest_object_size < max_size { + bucket_usage.object_size_histogram.insert(range_name.to_string(), 1); + break; + } + } + + // Set version histogram + let version_ranges = [ + ("1", 1, 1), + ("2-5", 2, 5), + ("6-10", 6, 10), + ("11-50", 11, 50), + ("51-100", 51, 100), + ("100+", 101, usize::MAX), + ]; + + for (range_name, min_versions, max_versions) in version_ranges { + if versions_count as usize >= min_versions && versions_count as usize <= max_versions { + bucket_usage.object_versions_histogram.insert(range_name.to_string(), 1); + break; + } + } + + self.buckets_usage.insert(bucket_name, bucket_usage); + // Update buckets count when adding new bucket + self.buckets_count = self.buckets_usage.len() as u64; + } + + // Update global statistics + self.objects_total_size += total_size; + self.objects_total_count += 1; + self.versions_total_count += versions_count; + self.delete_markers_total_count += delete_markers_count; + } + + /// Extract bucket name from object path + pub fn extract_bucket_from_path(&self, object_path: &str) -> Result> { + let parts: Vec<&str> = object_path.split('/').collect(); + if parts.is_empty() { + return Err("Invalid object path: empty".into()); + } + Ok(parts[0].to_string()) + } + + /// Update capacity information + pub fn update_capacity(&mut self, total: u64, used: u64, free: u64) { + self.total_capacity = total; + self.total_used_capacity = used; + self.total_free_capacity = free; + self.last_update = Some(SystemTime::now()); + } + + /// Add bucket usage info + pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) { + self.buckets_usage.insert(bucket.clone(), usage); + self.buckets_count = self.buckets_usage.len() as u64; + self.last_update = Some(SystemTime::now()); + } + + /// Get bucket usage info + pub fn get_bucket_usage(&self, bucket: &str) -> Option<&BucketUsageInfo> { + self.buckets_usage.get(bucket) + } + + /// Calculate total statistics from all buckets + pub fn calculate_totals(&mut self) { + self.objects_total_count = 0; + self.versions_total_count = 0; + self.delete_markers_total_count = 0; + self.objects_total_size = 0; + + for usage in self.buckets_usage.values() { + self.objects_total_count += usage.objects_count; + self.versions_total_count += usage.versions_count; + self.delete_markers_total_count += usage.delete_markers_count; + self.objects_total_size += usage.size; + } + } + + /// Merge another DataUsageInfo into this one + pub fn merge(&mut self, other: &DataUsageInfo) { + // Merge bucket usage + for (bucket, usage) in &other.buckets_usage { + if let Some(existing) = self.buckets_usage.get_mut(bucket) { + existing.merge(usage); + } else { + self.buckets_usage.insert(bucket.clone(), usage.clone()); + } + } + + // Recalculate totals + self.calculate_totals(); + + // Ensure buckets_count stays consistent with buckets_usage + self.buckets_count = self.buckets_usage.len() as u64; + + // Update last update time + if let Some(other_update) = other.last_update { + if self.last_update.is_none() || other_update > self.last_update.unwrap() { + self.last_update = Some(other_update); + } + } + } +} + +impl BucketUsageInfo { + /// Create a new BucketUsageInfo + pub fn new() -> Self { + Self::default() + } + + /// Add size summary to this bucket usage + pub fn add_size_summary(&mut self, summary: &SizeSummary) { + self.size += summary.total_size as u64; + self.versions_count += summary.versions as u64; + self.delete_markers_count += summary.delete_markers as u64; + self.replica_size += summary.replica_size as u64; + self.replica_count += summary.replica_count as u64; + } + + /// Merge another BucketUsageInfo into this one + pub fn merge(&mut self, other: &BucketUsageInfo) { + self.size += other.size; + self.objects_count += other.objects_count; + self.versions_count += other.versions_count; + self.delete_markers_count += other.delete_markers_count; + self.replica_size += other.replica_size; + self.replica_count += other.replica_count; + + // Merge histograms + for (key, value) in &other.object_size_histogram { + *self.object_size_histogram.entry(key.clone()).or_insert(0) += value; + } + + for (key, value) in &other.object_versions_histogram { + *self.object_versions_histogram.entry(key.clone()).or_insert(0) += value; + } + + // Merge replication info + for (target, info) in &other.replication_info { + let entry = self.replication_info.entry(target.clone()).or_default(); + entry.replicated_size += info.replicated_size; + entry.replica_size += info.replica_size; + entry.replication_pending_size += info.replication_pending_size; + entry.replication_failed_size += info.replication_failed_size; + entry.replication_pending_count += info.replication_pending_count; + entry.replication_failed_count += info.replication_failed_count; + entry.replicated_count += info.replicated_count; + } + + // Merge backward compatibility fields + self.replication_pending_size_v1 += other.replication_pending_size_v1; + self.replication_failed_size_v1 += other.replication_failed_size_v1; + self.replicated_size_v1 += other.replicated_size_v1; + self.replication_pending_count_v1 += other.replication_pending_count_v1; + self.replication_failed_count_v1 += other.replication_failed_count_v1; + } +} + +impl SizeSummary { + /// Create a new SizeSummary + pub fn new() -> Self { + Self::default() + } + + /// Add another SizeSummary to this one + pub fn add(&mut self, other: &SizeSummary) { + self.total_size += other.total_size; + self.versions += other.versions; + self.delete_markers += other.delete_markers; + self.replicated_size += other.replicated_size; + self.replicated_count += other.replicated_count; + self.pending_size += other.pending_size; + self.failed_size += other.failed_size; + self.replica_size += other.replica_size; + self.replica_count += other.replica_count; + self.pending_count += other.pending_count; + self.failed_count += other.failed_count; + + // Merge replication target stats + for (target, stats) in &other.repl_target_stats { + let entry = self.repl_target_stats.entry(target.clone()).or_default(); + entry.replicated_size += stats.replicated_size; + entry.replicated_count += stats.replicated_count; + entry.pending_size += stats.pending_size; + entry.failed_size += stats.failed_size; + entry.pending_count += stats.pending_count; + entry.failed_count += stats.failed_count; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_data_usage_info_creation() { + let mut info = DataUsageInfo::new(); + info.update_capacity(1000, 500, 500); + + assert_eq!(info.total_capacity, 1000); + assert_eq!(info.total_used_capacity, 500); + assert_eq!(info.total_free_capacity, 500); + assert!(info.last_update.is_some()); + } + + #[test] + fn test_bucket_usage_info_merge() { + let mut usage1 = BucketUsageInfo::new(); + usage1.size = 100; + usage1.objects_count = 10; + usage1.versions_count = 5; + + let mut usage2 = BucketUsageInfo::new(); + usage2.size = 200; + usage2.objects_count = 20; + usage2.versions_count = 10; + + usage1.merge(&usage2); + + assert_eq!(usage1.size, 300); + assert_eq!(usage1.objects_count, 30); + assert_eq!(usage1.versions_count, 15); + } + + #[test] + fn test_size_summary_add() { + let mut summary1 = SizeSummary::new(); + summary1.total_size = 100; + summary1.versions = 5; + + let mut summary2 = SizeSummary::new(); + summary2.total_size = 200; + summary2.versions = 10; + + summary1.add(&summary2); + + assert_eq!(summary1.total_size, 300); + assert_eq!(summary1.versions, 15); + } +} diff --git a/crates/common/src/heal_channel.rs b/crates/common/src/heal_channel.rs new file mode 100644 index 000000000..6988e06b8 --- /dev/null +++ b/crates/common/src/heal_channel.rs @@ -0,0 +1,427 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus}; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::{self, Display}, + sync::OnceLock, +}; +use tokio::sync::mpsc; +use uuid::Uuid; + +pub const HEAL_DELETE_DANGLING: bool = true; +pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs"; +pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs"; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub enum HealItemType { + Metadata, + Bucket, + BucketMetadata, + Object, +} + +impl HealItemType { + pub fn to_str(&self) -> &str { + match self { + HealItemType::Metadata => "metadata", + HealItemType::Bucket => "bucket", + HealItemType::BucketMetadata => "bucket-metadata", + HealItemType::Object => "object", + } + } +} + +impl Display for HealItemType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_str()) + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub enum DriveState { + Ok, + Offline, + Corrupt, + Missing, + PermissionDenied, + Faulty, + RootMount, + Unknown, + Unformatted, // only returned by disk +} + +impl DriveState { + pub fn to_str(&self) -> &str { + match self { + DriveState::Ok => "ok", + DriveState::Offline => "offline", + DriveState::Corrupt => "corrupt", + DriveState::Missing => "missing", + DriveState::PermissionDenied => "permission-denied", + DriveState::Faulty => "faulty", + DriveState::RootMount => "root-mount", + DriveState::Unknown => "unknown", + DriveState::Unformatted => "unformatted", + } + } +} + +impl Display for DriveState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_str()) + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum HealScanMode { + Unknown, + Normal, + Deep, +} + +impl Default for HealScanMode { + fn default() -> Self { + Self::Normal + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct HealOpts { + pub recursive: bool, + #[serde(rename = "dryRun")] + pub dry_run: bool, + pub remove: bool, + pub recreate: bool, + #[serde(rename = "scanMode")] + pub scan_mode: HealScanMode, + #[serde(rename = "updateParity")] + pub update_parity: bool, + #[serde(rename = "nolock")] + pub no_lock: bool, + pub pool: Option, + pub set: Option, +} + +/// Heal channel command type +#[derive(Debug, Clone)] +pub enum HealChannelCommand { + /// Start a new heal task + Start(HealChannelRequest), + /// Query heal task status + Query { heal_path: String, client_token: String }, + /// Cancel heal task + Cancel { heal_path: String }, +} + +/// Heal request from admin to ahm +#[derive(Debug, Clone, Default)] +pub struct HealChannelRequest { + /// Unique request ID + pub id: String, + /// Disk ID for heal disk/erasure set task + pub disk: Option, + /// Bucket name + pub bucket: String, + /// Object prefix (optional) + pub object_prefix: Option, + /// Force start heal + pub force_start: bool, + /// Priority + pub priority: HealChannelPriority, + /// Pool index (optional) + pub pool_index: Option, + /// Set index (optional) + pub set_index: Option, + /// Scan mode (optional) + pub scan_mode: Option, + /// Whether to remove corrupted data + pub remove_corrupted: Option, + /// Whether to recreate missing data + pub recreate_missing: Option, + /// Whether to update parity + pub update_parity: Option, + /// Whether to recursively process + pub recursive: Option, + /// Whether to dry run + pub dry_run: Option, + /// Timeout in seconds (optional) + pub timeout_seconds: Option, +} + +/// Heal response from ahm to admin +#[derive(Debug, Clone)] +pub struct HealChannelResponse { + /// Request ID + pub request_id: String, + /// Success status + pub success: bool, + /// Response data (if successful) + pub data: Option>, + /// Error message (if failed) + pub error: Option, +} + +/// Heal priority +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealChannelPriority { + /// Low priority + Low, + /// Normal priority + Normal, + /// High priority + High, + /// Critical priority + Critical, +} + +impl Default for HealChannelPriority { + fn default() -> Self { + Self::Normal + } +} + +/// Heal channel sender +pub type HealChannelSender = mpsc::UnboundedSender; + +/// Heal channel receiver +pub type HealChannelReceiver = mpsc::UnboundedReceiver; + +/// Global heal channel sender +static GLOBAL_HEAL_CHANNEL_SENDER: OnceLock = OnceLock::new(); + +/// Initialize global heal channel +pub fn init_heal_channel() -> HealChannelReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + GLOBAL_HEAL_CHANNEL_SENDER + .set(tx) + .expect("Heal channel sender already initialized"); + rx +} + +/// Get global heal channel sender +pub fn get_heal_channel_sender() -> Option<&'static HealChannelSender> { + GLOBAL_HEAL_CHANNEL_SENDER.get() +} + +/// Send heal command through global channel +pub async fn send_heal_command(command: HealChannelCommand) -> Result<(), String> { + if let Some(sender) = get_heal_channel_sender() { + sender + .send(command) + .map_err(|e| format!("Failed to send heal command: {e}"))?; + Ok(()) + } else { + Err("Heal channel not initialized".to_string()) + } +} + +/// Send heal start request +pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> { + send_heal_command(HealChannelCommand::Start(request)).await +} + +/// Send heal query request +pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<(), String> { + send_heal_command(HealChannelCommand::Query { heal_path, client_token }).await +} + +/// Send heal cancel request +pub async fn cancel_heal_task(heal_path: String) -> Result<(), String> { + send_heal_command(HealChannelCommand::Cancel { heal_path }).await +} + +/// Create a new heal request +pub fn create_heal_request( + bucket: String, + object_prefix: Option, + force_start: bool, + priority: Option, +) -> HealChannelRequest { + HealChannelRequest { + id: Uuid::new_v4().to_string(), + bucket, + object_prefix, + force_start, + priority: priority.unwrap_or_default(), + pool_index: None, + set_index: None, + scan_mode: None, + remove_corrupted: None, + recreate_missing: None, + update_parity: None, + recursive: None, + dry_run: None, + timeout_seconds: None, + disk: None, + } +} + +/// Create a new heal request with advanced options +pub fn create_heal_request_with_options( + bucket: String, + object_prefix: Option, + force_start: bool, + priority: Option, + pool_index: Option, + set_index: Option, +) -> HealChannelRequest { + HealChannelRequest { + id: Uuid::new_v4().to_string(), + bucket, + object_prefix, + force_start, + priority: priority.unwrap_or_default(), + pool_index, + set_index, + ..Default::default() + } +} + +/// Create a heal response +pub fn create_heal_response( + request_id: String, + success: bool, + data: Option>, + error: Option, +) -> HealChannelResponse { + HealChannelResponse { + request_id, + success, + data, + error, + } +} + +fn lc_get_prefix(rule: &LifecycleRule) -> String { + if let Some(p) = &rule.prefix { + return p.to_string(); + } else if let Some(filter) = &rule.filter { + if let Some(p) = &filter.prefix { + return p.to_string(); + } else if let Some(and) = &filter.and { + if let Some(p) = &and.prefix { + return p.to_string(); + } + } + } + + "".into() +} + +pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool { + if config.rules.is_empty() { + return false; + } + + for rule in config.rules.iter() { + if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) { + continue; + } + let rule_prefix = lc_get_prefix(rule); + if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix) + { + continue; + } + + if let Some(e) = &rule.noncurrent_version_expiration { + if let Some(true) = e.noncurrent_days.map(|d| d > 0) { + return true; + } + if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) { + return true; + } + } + + if rule.noncurrent_version_transitions.is_some() { + return true; + } + if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) { + return true; + } + + if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) { + return true; + } + + if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) { + return true; + } + + if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) { + return true; + } + + if rule.transitions.is_some() { + return true; + } + } + false +} + +pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool { + if config.rules.is_empty() { + return false; + } + + for rule in config.rules.iter() { + if rule + .status + .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)) + { + continue; + } + if !prefix.is_empty() { + if let Some(filter) = &rule.filter { + if let Some(r_prefix) = &filter.prefix { + if !r_prefix.is_empty() { + // incoming prefix must be in rule prefix + if !recursive && !prefix.starts_with(r_prefix) { + continue; + } + // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix + // does not match + if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) { + continue; + } + } + } + } + } + return true; + } + false +} + +pub async fn send_heal_disk(set_disk_id: String, priority: Option) -> Result<(), String> { + let req = HealChannelRequest { + id: Uuid::new_v4().to_string(), + bucket: "".to_string(), + object_prefix: None, + disk: Some(set_disk_id), + force_start: false, + priority: priority.unwrap_or_default(), + pool_index: None, + set_index: None, + scan_mode: None, + remove_corrupted: None, + recreate_missing: None, + update_parity: None, + recursive: None, + dry_run: None, + timeout_seconds: None, + }; + send_heal_request(req).await +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 16de991db..09dc164a3 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -14,8 +14,11 @@ pub mod bucket_stats; // pub mod error; +pub mod data_usage; pub mod globals; +pub mod heal_channel; pub mod last_minute; +pub mod metrics; // is ',' pub static DEFAULT_DELIMITER: u8 = 44; diff --git a/crates/ecstore/src/heal/data_scanner_metric.rs b/crates/common/src/metrics.rs similarity index 73% rename from crates/ecstore/src/heal/data_scanner_metric.rs rename to crates/common/src/metrics.rs index 8e0b8ccc7..d88e5a3a6 100644 --- a/crates/ecstore/src/heal/data_scanner_metric.rs +++ b/crates/common/src/metrics.rs @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::data_scanner::CurrentScannerCycle; -use crate::bucket::lifecycle::lifecycle; -use chrono::Utc; +use chrono::{DateTime, Utc}; use lazy_static::lazy_static; -use rustfs_common::last_minute::{AccElem, LastMinuteLatency}; use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics; use std::{ collections::HashMap, + fmt::Display, pin::Pin, sync::{ Arc, @@ -29,12 +27,58 @@ use std::{ }; use tokio::sync::{Mutex, RwLock}; +use crate::last_minute::{AccElem, LastMinuteLatency}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IlmAction { + NoneAction = 0, + DeleteAction, + DeleteVersionAction, + TransitionAction, + TransitionVersionAction, + DeleteRestoredAction, + DeleteRestoredVersionAction, + DeleteAllVersionsAction, + DelMarkerDeleteAllVersionsAction, + ActionCount, +} + +impl IlmAction { + pub fn delete_restored(&self) -> bool { + *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_versioned(&self) -> bool { + *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_all(&self) -> bool { + *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction + } + + pub fn delete(&self) -> bool { + if self.delete_restored() { + return true; + } + *self == Self::DeleteVersionAction + || *self == Self::DeleteAction + || *self == Self::DeleteAllVersionsAction + || *self == Self::DelMarkerDeleteAllVersionsAction + } +} + +impl Display for IlmAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + lazy_static! { - pub static ref globalScannerMetrics: Arc = Arc::new(ScannerMetrics::new()); + pub static ref globalMetrics: Arc = Arc::new(Metrics::new()); } #[derive(Clone, Debug, PartialEq, PartialOrd)] -pub enum ScannerMetric { +pub enum Metric { // START Realtime metrics, that only records // last minute latencies and total operation count. ReadMetadata = 0, @@ -69,7 +113,7 @@ pub enum ScannerMetric { Last, } -impl ScannerMetric { +impl Metric { /// Convert to string representation for metrics pub fn as_str(self) -> &'static str { match self { @@ -203,7 +247,7 @@ impl CurrentPathTracker { } /// Main scanner metrics structure -pub struct ScannerMetrics { +pub struct Metrics { // All fields must be accessed atomically and aligned. operations: Vec, latency: Vec, @@ -213,94 +257,102 @@ pub struct ScannerMetrics { current_paths: Arc>>>, // Cycle information - cycle_info: Arc>>, + cycle_info: Arc>>, } -impl ScannerMetrics { - pub fn new() -> Self { - let operations = (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(); +// This is a placeholder. We'll need to define this struct. +#[derive(Clone, Debug)] +pub struct CurrentCycle { + pub current: u64, + pub cycle_completed: Vec>, + pub started: DateTime, +} - let latency = (0..ScannerMetric::LastRealtime as usize) +impl Metrics { + pub fn new() -> Self { + let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(); + + let latency = (0..Metric::LastRealtime as usize) .map(|_| LockedLastMinuteLatency::new()) .collect(); Self { operations, latency, - actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(), - actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize], + actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(), + actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize], current_paths: Arc::new(RwLock::new(HashMap::new())), cycle_info: Arc::new(RwLock::new(None)), } } /// Log scanner action with custom metadata - compatible with existing usage - pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap) { + pub fn log(metric: Metric) -> impl Fn(&HashMap) { let metric = metric as usize; let start_time = SystemTime::now(); move |_custom: &HashMap| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); + globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task for this) - if (metric) < ScannerMetric::LastRealtime as usize { + if (metric) < Metric::LastRealtime as usize { let metric_index = metric; tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; + globalMetrics.latency[metric_index].add(duration).await; }); } // Log trace metrics - if metric as u8 > ScannerMetric::StartTrace as u8 { + if metric as u8 > Metric::StartTrace as u8 { //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); } } } /// Time scanner action with size - returns function that takes size - pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) { + pub fn time_size(metric: Metric) -> impl Fn(u64) { let metric = metric as usize; let start_time = SystemTime::now(); move |size: u64| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); + globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics with size (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { + if (metric) < Metric::LastRealtime as usize { let metric_index = metric; tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add_size(duration, size).await; + globalMetrics.latency[metric_index].add_size(duration, size).await; }); } } } /// Time a scanner action - returns a closure to call when done - pub fn time(metric: ScannerMetric) -> impl Fn() { + pub fn time(metric: Metric) -> impl Fn() { let metric = metric as usize; let start_time = SystemTime::now(); move || { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); + globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { + if (metric) < Metric::LastRealtime as usize { let metric_index = metric; tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; + globalMetrics.latency[metric_index].add(duration).await; }); } } } /// Time N scanner actions - returns function that takes count, then returns completion function - pub fn time_n(metric: ScannerMetric) -> Box Box + Send + Sync> { + pub fn time_n(metric: Metric) -> Box Box + Send + Sync> { let metric = metric as usize; let start_time = SystemTime::now(); Box::new(move |count: usize| { @@ -308,22 +360,23 @@ impl ScannerMetrics { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed); + globalMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { + if (metric) < Metric::LastRealtime as usize { let metric_index = metric; tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; + globalMetrics.latency[metric_index].add(duration).await; }); } }) }) } - pub fn time_ilm(a: lifecycle::IlmAction) -> Box Box + Send + Sync> { + /// Time ILM action with versions - returns function that takes versions, then returns completion function + pub fn time_ilm(a: IlmAction) -> Box Box + Send + Sync> { let a_clone = a as usize; - if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { + if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize { return Box::new(move |_: u64| Box::new(move || {})); } let start = SystemTime::now(); @@ -331,50 +384,50 @@ impl ScannerMetrics { Box::new(move || { let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); tokio::spawn(async move { - globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed); - globalScannerMetrics.actions_latency[a_clone].add(duration).await; + globalMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed); + globalMetrics.actions_latency[a_clone].add(duration).await; }); }) }) } /// Increment time with specific duration - pub async fn inc_time(metric: ScannerMetric, duration: Duration) { + pub async fn inc_time(metric: Metric, duration: Duration) { let metric = metric as usize; // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); + globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics - if (metric) < ScannerMetric::LastRealtime as usize { - globalScannerMetrics.latency[metric].add(duration).await; + if (metric) < Metric::LastRealtime as usize { + globalMetrics.latency[metric].add(duration).await; } } /// Get lifetime operation count for a metric - pub fn lifetime(&self, metric: ScannerMetric) -> u64 { + pub fn lifetime(&self, metric: Metric) -> u64 { let metric = metric as usize; - if (metric) >= ScannerMetric::Last as usize { + if (metric) >= Metric::Last as usize { return 0; } self.operations[metric].load(Ordering::Relaxed) } /// Get last minute statistics for a metric - pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem { + pub async fn last_minute(&self, metric: Metric) -> AccElem { let metric = metric as usize; - if (metric) >= ScannerMetric::LastRealtime as usize { + if (metric) >= Metric::LastRealtime as usize { return AccElem::default(); } self.latency[metric].total().await } /// Set current cycle information - pub async fn set_cycle(&self, cycle: Option) { + pub async fn set_cycle(&self, cycle: Option) { *self.cycle_info.write().await = cycle; } /// Get current cycle information - pub async fn get_cycle(&self) -> Option { + pub async fn get_cycle(&self) -> Option { self.cycle_info.read().await.clone() } @@ -411,20 +464,20 @@ impl ScannerMetrics { metrics.active_paths = self.get_current_paths().await; // Lifetime operations - for i in 0..ScannerMetric::Last as usize { + for i in 0..Metric::Last as usize { let count = self.operations[i].load(Ordering::Relaxed); if count > 0 { - if let Some(metric) = ScannerMetric::from_index(i) { + if let Some(metric) = Metric::from_index(i) { metrics.life_time_ops.insert(metric.as_str().to_string(), count); } } } // Last minute statistics for realtime metrics - for i in 0..ScannerMetric::LastRealtime as usize { + for i in 0..Metric::LastRealtime as usize { let last_min = self.latency[i].total().await; if last_min.n > 0 { - if let Some(_metric) = ScannerMetric::from_index(i) { + if let Some(_metric) = Metric::from_index(i) { // Convert to madmin TimedAction format if needed // This would require implementing the conversion } @@ -448,11 +501,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, let tracker_clone = Arc::clone(&tracker); let disk_clone = disk_name.clone(); tokio::spawn(async move { - globalScannerMetrics - .current_paths - .write() - .await - .insert(disk_clone, tracker_clone); + globalMetrics.current_paths.write().await.insert(disk_clone, tracker_clone); }); let update_fn = { @@ -471,7 +520,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, Arc::new(move || -> Pin + Send>> { let disk_name = disk_name.clone(); Box::pin(async move { - globalScannerMetrics.current_paths.write().await.remove(&disk_name); + globalMetrics.current_paths.write().await.remove(&disk_name); }) }) }; @@ -479,7 +528,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, (update_fn, done_fn) } -impl Default for ScannerMetrics { +impl Default for Metrics { fn default() -> Self { Self::new() } diff --git a/crates/ecstore/src/admin_server_info.rs b/crates/ecstore/src/admin_server_info.rs index 0ee4b1b84..4ee7d94c4 100644 --- a/crates/ecstore/src/admin_server_info.rs +++ b/crates/ecstore/src/admin_server_info.rs @@ -12,23 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend}; use crate::error::{Error, Result}; use crate::{ disk::endpoint::Endpoint, global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints}, - heal::{ - data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend}, - data_usage_cache::DataUsageCache, - heal_commands::{DRIVE_STATE_OK, DRIVE_STATE_UNFORMATTED}, - }, new_object_layer_fn, notification_sys::get_global_notification_sys, store_api::StorageAPI, }; -use rustfs_common::{ - // error::{Error, Result}, - globals::GLOBAL_Local_Node_Name, -}; + +use crate::data_usage::load_data_usage_cache; +use rustfs_common::{globals::GLOBAL_Local_Node_Name, heal_channel::DriveState}; use rustfs_madmin::{ BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties, }; @@ -318,7 +313,7 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend for disk in disks_info { let ep = &disk.endpoint; let state = &disk.state; - if *state != DRIVE_STATE_OK && *state != DRIVE_STATE_UNFORMATTED { + if *state != DriveState::Ok.to_string() && *state != DriveState::Unformatted.to_string() { *offline_disks.get_mut(ep).unwrap() += 1; continue; } @@ -359,13 +354,13 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result::new()); erasure_set.objects_count = data_usage_info.objects_total_count; erasure_set.versions_count = data_usage_info.versions_total_count; erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count; diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 14cfe9985..29b2097f6 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -22,6 +22,10 @@ use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use futures::Future; use http::HeaderMap; use lazy_static::lazy_static; +use rustfs_common::data_usage::TierStats; +use rustfs_common::heal_channel::rep_has_active_rules; +use rustfs_common::metrics::{IlmAction, Metrics}; +use rustfs_utils::path::encode_dir_object; use s3s::Body; use sha2::{Digest, Sha256}; use std::any::Any; @@ -31,6 +35,7 @@ use std::io::Write; use std::pin::Pin; use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::{Arc, Mutex}; +use time::OffsetDateTime; use tokio::select; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, mpsc}; @@ -41,9 +46,10 @@ use xxhash_rust::xxh64; //use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger}; //use rustfs_notify::{initialize, notification_system}; use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}; -use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; +use super::lifecycle::{self, ExpirationOptions, Lifecycle, TransitionOptions}; use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; use super::tier_sweeper::{Jentry, delete_object_from_remote_tier}; +use crate::bucket::object_lock::objectlock_sys::enforce_retention_for_deletion; use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys}; use crate::client::object_api_utils::new_getobjectreader; use crate::error::Error; @@ -52,16 +58,11 @@ use crate::event::name::EventName; use crate::event_notification::{EventArgs, send_event}; use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; -use crate::heal::{ - data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object}, - data_scanner_metric::ScannerMetrics, - data_usage_cache::TierStats, -}; use crate::store::ECStore; use crate::store_api::StorageAPI; use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete}; use crate::tier::warm_backend::WarmBackendGetOpts; -use s3s::dto::BucketLifecycleConfiguration; +use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; pub type TraceFn = @@ -631,7 +632,7 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { if !lc.is_none() { let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await; match event.action { - lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + IlmAction::TransitionAction | IlmAction::TransitionVersionAction => { if oi.delete_marker || oi.is_dir { return; } @@ -728,7 +729,7 @@ pub fn gen_transition_objname(bucket: &str) -> Result { } pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> { - let time_ilm = ScannerMetrics::time_ilm(lae.event.action); + let time_ilm = Metrics::time_ilm(lae.event.action); let opts = ObjectOptions { transition: TransitionOptions { @@ -842,3 +843,161 @@ pub struct RestoreObjectRequest { } const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; + +pub async fn eval_action_from_lifecycle( + lc: &BucketLifecycleConfiguration, + lr: Option, + rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, + oi: &ObjectInfo, +) -> lifecycle::Event { + let event = lc.eval(&oi.to_lifecycle_opts()).await; + //if serverDebugLog { + info!("lifecycle: Secondary scan: {}", event.action); + //} + + let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; + + match event.action { + lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + if lock_enabled { + return lifecycle::Event::default(); + } + } + lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => { + if oi.version_id.is_none() { + return lifecycle::Event::default(); + } + if lock_enabled && enforce_retention_for_deletion(oi) { + //if serverDebugLog { + if oi.version_id.is_some() { + info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting", oi.name); + } + //} + return lifecycle::Event::default(); + } + if let Some(rcfg) = rcfg { + if rep_has_active_rules(&rcfg.0, &oi.name, true) { + return lifecycle::Event::default(); + } + } + } + _ => (), + } + + event +} + +async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + if oi.delete_marker || oi.is_dir { + return false; + } + GLOBAL_TransitionState.queue_transition_task(oi, event, src).await; + true +} + +pub async fn apply_expiry_on_transitioned_object( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> bool { + // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { + return false; + } + // let _ = time_ilm(1); + + true +} + +pub async fn apply_expiry_on_non_transitioned_objects( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + _src: &LcEventSrc, +) -> bool { + let mut opts = ObjectOptions { + expiration: ExpirationOptions { expire: true }, + ..Default::default() + }; + + if lc_event.action.delete_versioned() { + opts.version_id = Some(oi.version_id.expect("err").to_string()); + } + + opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; + opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await; + + if lc_event.action.delete_all() { + opts.delete_prefix = true; + opts.delete_prefix_object = true; + } + + // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + + let mut dobj = api + .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts) + .await + .unwrap(); + if dobj.name.is_empty() { + dobj = oi.clone(); + } + + //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags(); + //tags["version-id"] = dobj.version_id; + + let mut event_name = EventName::ObjectRemovedDelete; + if oi.delete_marker { + event_name = EventName::ObjectRemovedDeleteMarkerCreated; + } + match lc_event.action { + lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions, + lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete, + _ => (), + } + send_event(EventArgs { + event_name: event_name.as_ref().to_string(), + bucket_name: dobj.bucket.clone(), + object: dobj, + user_agent: "Internal: [ILM-Expiry]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + + if lc_event.action != lifecycle::IlmAction::NoneAction { + // let mut num_versions = 1_u64; + // if lc_event.action.delete_all() { + // num_versions = oi.num_versions as u64; + // } + // let _ = time_ilm(num_versions); + } + + true +} + +async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_by_days(oi, event, src).await; + true +} + +pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut success = false; + match event.action { + lifecycle::IlmAction::DeleteVersionAction + | lifecycle::IlmAction::DeleteAction + | lifecycle::IlmAction::DeleteRestoredAction + | lifecycle::IlmAction::DeleteRestoredVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction + | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + success = apply_expiry_rule(event, src, oi).await; + } + lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + success = apply_transition_rule(event, src, oi).await; + } + _ => (), + } + success +} diff --git a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs index 2431d6ba8..3d232363c 100644 --- a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -43,49 +43,7 @@ const _ERR_XML_NOT_WELL_FORMED: &str = const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket"; -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum IlmAction { - NoneAction = 0, - DeleteAction, - DeleteVersionAction, - TransitionAction, - TransitionVersionAction, - DeleteRestoredAction, - DeleteRestoredVersionAction, - DeleteAllVersionsAction, - DelMarkerDeleteAllVersionsAction, - ActionCount, -} - -impl IlmAction { - pub fn delete_restored(&self) -> bool { - *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction - } - - pub fn delete_versioned(&self) -> bool { - *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction - } - - pub fn delete_all(&self) -> bool { - *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction - } - - pub fn delete(&self) -> bool { - if self.delete_restored() { - return true; - } - *self == Self::DeleteVersionAction - || *self == Self::DeleteAction - || *self == Self::DeleteAllVersionsAction - || *self == Self::DelMarkerDeleteAllVersionsAction - } -} - -impl Display for IlmAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} +pub use rustfs_common::metrics::IlmAction; #[async_trait::async_trait] pub trait RuleValidate { diff --git a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index 78e3e8ed0..557d6189d 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -25,7 +25,7 @@ use std::ops::Sub; use time::OffsetDateTime; use tracing::{error, warn}; -use crate::heal::data_usage_cache::TierStats; +use rustfs_common::data_usage::TierStats; pub type DailyAllTierStats = HashMap; diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 791134da9..b192cd2f3 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -18,9 +18,9 @@ use crate::bucket::utils::{deserialize, is_meta_bucketname}; use crate::cmd::bucket_targets; use crate::error::{Error, Result, is_err_bucket_not_found}; use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn}; -use crate::heal::heal_commands::HealOpts; use crate::store::ECStore; use futures::future::join_all; +use rustfs_common::heal_channel::HealOpts; use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, diff --git a/crates/ecstore/src/cache_value/mod.rs b/crates/ecstore/src/cache_value/mod.rs index 328444a05..5ab628098 100644 --- a/crates/ecstore/src/cache_value/mod.rs +++ b/crates/ecstore/src/cache_value/mod.rs @@ -12,5 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + +use lazy_static::lazy_static; +use tokio_util::sync::CancellationToken; + // pub mod cache; pub mod metacache_set; + +lazy_static! { + pub static ref LIST_PATH_RAW_CANCEL_TOKEN: Arc = Arc::new(CancellationToken::new()); +} diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs new file mode 100644 index 000000000..425081ef1 --- /dev/null +++ b/crates/ecstore/src/data_usage.rs @@ -0,0 +1,297 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{collections::HashMap, sync::Arc}; + +use crate::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore}; +use rustfs_common::data_usage::{BucketTargetUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, SizeSummary}; +use rustfs_utils::path::SLASH_SEPARATOR; +use tracing::{error, warn}; + +use crate::error::Error; + +// Data usage storage constants +pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; +const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; +const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; +pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; + +// Data usage storage paths +lazy_static::lazy_static! { + pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", + crate::disk::RUSTFS_META_BUCKET, + SLASH_SEPARATOR, + crate::disk::BUCKET_META_PREFIX + ); + pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", + crate::disk::BUCKET_META_PREFIX, + SLASH_SEPARATOR, + DATA_USAGE_OBJ_NAME + ); + pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", + crate::disk::BUCKET_META_PREFIX, + SLASH_SEPARATOR, + DATA_USAGE_BLOOM_NAME + ); +} + +/// Store data usage info to backend storage +pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc) -> Result<(), Error> { + let data = + serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; + + // Save to backend using the same mechanism as original code + crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data) + .await + .map_err(Error::other)?; + + Ok(()) +} + +/// Load data usage info from backend storage +pub async fn load_data_usage_from_backend(store: Arc) -> Result { + let buf: Vec = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await { + Ok(data) => data, + Err(e) => { + error!("Failed to read data usage info from backend: {}", e); + if e == crate::error::Error::ConfigNotFound { + return Ok(DataUsageInfo::default()); + } + return Err(Error::other(e)); + } + }; + + let mut data_usage_info: DataUsageInfo = + serde_json::from_slice(&buf).map_err(|e| Error::other(format!("Failed to deserialize data usage info: {e}")))?; + + warn!("Loaded data usage info from backend {:?}", &data_usage_info); + + // Handle backward compatibility like original code + if data_usage_info.buckets_usage.is_empty() { + data_usage_info.buckets_usage = data_usage_info + .bucket_sizes + .iter() + .map(|(bucket, &size)| { + ( + bucket.clone(), + rustfs_common::data_usage::BucketUsageInfo { + size, + ..Default::default() + }, + ) + }) + .collect(); + } + + if data_usage_info.bucket_sizes.is_empty() { + data_usage_info.bucket_sizes = data_usage_info + .buckets_usage + .iter() + .map(|(bucket, bui)| (bucket.clone(), bui.size)) + .collect(); + } + + for (bucket, bui) in &data_usage_info.buckets_usage { + if bui.replicated_size_v1 > 0 + || bui.replication_failed_count_v1 > 0 + || bui.replication_failed_size_v1 > 0 + || bui.replication_pending_count_v1 > 0 + { + if let Ok((cfg, _)) = get_replication_config(bucket).await { + if !cfg.role.is_empty() { + data_usage_info.replication_info.insert( + cfg.role.clone(), + BucketTargetUsageInfo { + replication_failed_size: bui.replication_failed_size_v1, + replication_failed_count: bui.replication_failed_count_v1, + replicated_size: bui.replicated_size_v1, + replication_pending_count: bui.replication_pending_count_v1, + replication_pending_size: bui.replication_pending_size_v1, + ..Default::default() + }, + ); + } + } + } + } + + Ok(data_usage_info) +} + +/// Create a data usage cache entry from size summary +pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry { + let mut entry = DataUsageEntry::default(); + entry.add_sizes(summary); + entry +} + +/// Convert data usage cache to DataUsageInfo +pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[crate::store_api::BucketInfo]) -> DataUsageInfo { + let e = match cache.find(path) { + Some(e) => e, + None => return DataUsageInfo::default(), + }; + let flat = cache.flatten(&e); + + let mut buckets_usage = HashMap::new(); + for bucket in buckets.iter() { + let e = match cache.find(&bucket.name) { + Some(e) => e, + None => continue, + }; + let flat = cache.flatten(&e); + let mut bui = rustfs_common::data_usage::BucketUsageInfo { + size: flat.size as u64, + versions_count: flat.versions as u64, + objects_count: flat.objects as u64, + delete_markers_count: flat.delete_markers as u64, + object_size_histogram: flat.obj_sizes.to_map(), + object_versions_histogram: flat.obj_versions.to_map(), + ..Default::default() + }; + + if let Some(rs) = &flat.replication_stats { + bui.replica_size = rs.replica_size; + bui.replica_count = rs.replica_count; + + for (arn, stat) in rs.targets.iter() { + bui.replication_info.insert( + arn.clone(), + BucketTargetUsageInfo { + replication_pending_size: stat.pending_size, + replicated_size: stat.replicated_size, + replication_failed_size: stat.failed_size, + replication_pending_count: stat.pending_count, + replication_failed_count: stat.failed_count, + replicated_count: stat.replicated_count, + ..Default::default() + }, + ); + } + } + buckets_usage.insert(bucket.name.clone(), bui); + } + + DataUsageInfo { + last_update: cache.info.last_update, + objects_total_count: flat.objects as u64, + versions_total_count: flat.versions as u64, + delete_markers_total_count: flat.delete_markers as u64, + objects_total_size: flat.size as u64, + buckets_count: e.children.len() as u64, + buckets_usage, + ..Default::default() + } +} + +// Helper functions for DataUsageCache operations +pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result { + use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; + use crate::store_api::{ObjectIO, ObjectOptions}; + use http::HeaderMap; + use rand::Rng; + use std::path::Path; + use std::time::Duration; + use tokio::time::sleep; + + let mut d = DataUsageCache::default(); + let mut retries = 0; + while retries < 5 { + let path = Path::new(BUCKET_META_PREFIX).join(name); + match store + .get_object_reader( + RUSTFS_META_BUCKET, + path.to_str().unwrap(), + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(mut reader) => { + if let Ok(info) = DataUsageCache::unmarshal(&reader.read_all().await?) { + d = info + } + break; + } + Err(err) => match err { + crate::error::Error::FileNotFound | crate::error::Error::VolumeNotFound => { + match store + .get_object_reader( + RUSTFS_META_BUCKET, + name, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(mut reader) => { + if let Ok(info) = DataUsageCache::unmarshal(&reader.read_all().await?) { + d = info + } + break; + } + Err(_) => match err { + crate::error::Error::FileNotFound | crate::error::Error::VolumeNotFound => { + break; + } + _ => {} + }, + } + } + _ => { + break; + } + }, + } + retries += 1; + let dur = { + let mut rng = rand::rng(); + rng.random_range(0..1_000) + }; + sleep(Duration::from_millis(dur)).await; + } + Ok(d) +} + +pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> { + use crate::config::com::save_config; + use crate::disk::BUCKET_META_PREFIX; + use crate::new_object_layer_fn; + use std::path::Path; + + let Some(store) = new_object_layer_fn() else { + return Err(crate::error::Error::other("errServerNotInitialized")); + }; + let buf = cache.marshal_msg().map_err(crate::error::Error::other)?; + let buf_clone = buf.clone(); + + let store_clone = store.clone(); + + let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string(); + + let name_clone = name.clone(); + tokio::spawn(async move { + let _ = save_config(store_clone, &format!("{}{}", &name_clone, ".bkp"), buf_clone).await; + }); + save_config(store, &name, buf).await?; + Ok(()) +} diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 12214f3e0..18aafecab 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -21,9 +21,6 @@ use super::{ }; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; -use crate::bucket::metadata_sys::{self}; -use crate::bucket::versioning::VersioningApi; -use crate::bucket::versioning_sys::BucketVersioningSys; use crate::disk::error::FileAccessDeniedWithContext; use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}; use crate::disk::fs::{ @@ -36,16 +33,6 @@ use crate::disk::{ }; use crate::disk::{FileWriter, STORAGE_FORMAT_FILE}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; -use crate::heal::data_scanner::{ - ScannerItem, ShouldSleepFn, SizeSummary, lc_has_active_rules, rep_has_active_rules, scan_data_folder, -}; -use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics}; -use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry}; -use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE}; -use crate::heal::heal_commands::{HealScanMode, HealingTracker}; -use crate::heal::heal_ops::HEALING_TRACKER_FILENAME; -use crate::new_object_layer_fn; -use crate::store_api::{ObjectInfo, StorageAPI}; use rustfs_utils::path::{ GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, @@ -55,19 +42,18 @@ use tokio::time::interval; use crate::erasure_coding::bitrot_verify; use bytes::Bytes; use path_absolutize::Absolutize; -use rustfs_common::defer; use rustfs_filemeta::{ Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn, get_file_info, read_xl_meta_no_data, }; use rustfs_utils::HashAlgorithm; use rustfs_utils::os::get_info; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fmt::Debug; use std::io::SeekFrom; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::{Duration, SystemTime}; +use std::time::Duration; use std::{ fs::Metadata, path::{Path, PathBuf}, @@ -76,7 +62,6 @@ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind}; use tokio::sync::RwLock; -use tokio::sync::mpsc::Sender; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -2268,184 +2253,6 @@ impl DiskAPI for LocalDisk { Ok(info) } - - #[tracing::instrument(level = "info", skip_all)] - async fn ns_scanner( - &self, - cache: &DataUsageCache, - updates: Sender, - scan_mode: HealScanMode, - we_sleep: ShouldSleepFn, - ) -> Result { - self.scanning.fetch_add(1, Ordering::SeqCst); - defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) }); - - // must before metadata_sys - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - let mut cache = cache.clone(); - // Check if the current bucket has a configured lifecycle policy - if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await { - if lc_has_active_rules(&lc, "") { - cache.info.lifecycle = Some(lc); - } - } - - // Check if the current bucket has replication configuration - if let Ok((rcfg, _)) = metadata_sys::get_replication_config(&cache.info.name).await { - if rep_has_active_rules(&rcfg, "", true) { - // TODO: globalBucketTargetSys - } - } - - let vcfg = BucketVersioningSys::get(&cache.info.name).await.ok(); - - let loc = self.get_disk_location(); - // TODO: 这里需要处理错误 - let disks = store - .get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()) - .await - .map_err(|e| Error::other(e.to_string()))?; - let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?); - let disk_clone = disk.clone(); - cache.info.updates = Some(updates.clone()); - let mut data_usage_info = scan_data_folder( - &disks, - disk, - &cache, - Box::new(move |item: &ScannerItem| { - let mut item = item.clone(); - let disk = disk_clone.clone(); - let vcfg = vcfg.clone(); - Box::pin(async move { - if !item.path.ends_with(&format!("{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}")) { - return Err(Error::other(ERR_SKIP_FILE).into()); - } - let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject); - let mut res = HashMap::new(); - let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata); - let buf = match disk.read_metadata(item.path.clone()).await { - Ok(buf) => buf, - Err(err) => { - res.insert("err".to_string(), err.to_string()); - stop_fn(&res); - return Err(Error::other(ERR_SKIP_FILE).into()); - } - }; - done_sz(buf.len() as u64); - res.insert("metasize".to_string(), buf.len().to_string()); - item.transform_meta_dir(); - let meta_cache = MetaCacheEntry { - name: item.object_path().to_string_lossy().to_string(), - metadata: buf, - ..Default::default() - }; - let fivs = match meta_cache.file_info_versions(&item.bucket) { - Ok(fivs) => fivs, - Err(err) => { - res.insert("err".to_string(), err.to_string()); - stop_fn(&res); - return Err(Error::other(ERR_SKIP_FILE).into()); - } - }; - let mut size_s = SizeSummary::default(); - let done = ScannerMetrics::time(ScannerMetric::ApplyAll); - let obj_infos = match item.apply_versions_actions(&fivs.versions).await { - Ok(obj_infos) => obj_infos, - Err(err) => { - res.insert("err".to_string(), err.to_string()); - stop_fn(&res); - return Err(Error::other(ERR_SKIP_FILE).into()); - } - }; - - let versioned = if let Some(vcfg) = vcfg.as_ref() { - vcfg.versioned(item.object_path().to_str().unwrap_or_default()) - } else { - false - }; - - let mut obj_deleted = false; - for info in obj_infos.iter() { - let done = ScannerMetrics::time(ScannerMetric::ApplyVersion); - let sz: i64; - (obj_deleted, sz) = item.apply_actions(info, &mut size_s).await; - done(); - - if obj_deleted { - break; - } - - let actual_sz = match info.get_actual_size() { - Ok(size) => size, - Err(_) => continue, - }; - - if info.delete_marker { - size_s.delete_markers += 1; - } - - if info.version_id.is_some() && sz == actual_sz { - size_s.versions += 1; - } - - size_s.total_size += sz as usize; - - if info.delete_marker { - continue; - } - } - - for free_version in fivs.free_versions.iter() { - let _obj_info = ObjectInfo::from_file_info( - free_version, - &item.bucket, - &item.object_path().to_string_lossy(), - versioned, - ); - let done = ScannerMetrics::time(ScannerMetric::TierObjSweep); - done(); - } - - // todo: global trace - if obj_deleted { - return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into()); - } - done(); - Ok(size_s) - }) - }), - scan_mode, - we_sleep, - ) - .await?; - data_usage_info.info.last_update = Some(SystemTime::now()); - debug!("ns_scanner completed: {data_usage_info:?}"); - Ok(data_usage_info) - } - - #[tracing::instrument(skip(self))] - async fn healing(&self) -> Option { - let healing_file = path_join(&[ - self.path(), - PathBuf::from(RUSTFS_META_BUCKET), - PathBuf::from(BUCKET_META_PREFIX), - PathBuf::from(HEALING_TRACKER_FILENAME), - ]); - let b = match fs::read(healing_file).await { - Ok(b) => b, - Err(_) => return None, - }; - if b.is_empty() { - return None; - } - match HealingTracker::unmarshal_msg(&b) { - Ok(h) => Some(h), - Err(_) => Some(HealingTracker::default()), - } - } } async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInfo, bool)> { diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 1f918ee36..f680a42db 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -30,11 +30,6 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; pub const STORAGE_FORMAT_FILE: &str = "xl.meta"; pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp"; -use crate::heal::{ - data_scanner::ShouldSleepFn, - data_usage_cache::{DataUsageCache, DataUsageEntry}, - heal_commands::{HealScanMode, HealingTracker}, -}; use crate::rpc::RemoteDisk; use bytes::Bytes; use endpoint::Endpoint; @@ -46,10 +41,7 @@ use rustfs_madmin::info_commands::DiskMetrics; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, path::PathBuf, sync::Arc}; use time::OffsetDateTime; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - sync::mpsc::Sender, -}; +use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; pub type DiskStore = Arc; @@ -406,28 +398,6 @@ impl DiskAPI for Disk { Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await, } } - - #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))] - async fn ns_scanner( - &self, - cache: &DataUsageCache, - updates: Sender, - scan_mode: HealScanMode, - we_sleep: ShouldSleepFn, - ) -> Result { - match self { - Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await, - Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await, - } - } - - #[tracing::instrument(skip(self))] - async fn healing(&self) -> Option { - match self { - Disk::Local(local_disk) => local_disk.healing().await, - Disk::Remote(remote_disk) => remote_disk.healing().await, - } - } } pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result { @@ -527,14 +497,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; - async fn ns_scanner( - &self, - cache: &DataUsageCache, - updates: Sender, - scan_mode: HealScanMode, - we_sleep: ShouldSleepFn, - ) -> Result; - async fn healing(&self) -> Option; } #[derive(Debug, Default, Serialize, Deserialize)] diff --git a/crates/ecstore/src/global.rs b/crates/ecstore/src/global.rs index 93c38fe0d..533512266 100644 --- a/crates/ecstore/src/global.rs +++ b/crates/ecstore/src/global.rs @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::heal::mrf::MRFState; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, disk::DiskStore, endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, event_notification::EventNotifier, - heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, store::ECStore, tier::tier::TierConfigMgr, }; @@ -51,14 +49,10 @@ pub static ref GLOBAL_LOCAL_DISK_MAP: Arc> = Arc::new(RwLock::new(Vec::new())); pub static ref GLOBAL_Endpoints: OnceLock = OnceLock::new(); pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); -pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); -pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); pub static ref GLOBAL_TierConfigMgr: Arc> = TierConfigMgr::new(); pub static ref GLOBAL_LifecycleSys: Arc = LifecycleSys::new(); pub static ref GLOBAL_EventNotifier: Arc> = EventNotifier::new(); //pub static ref GLOBAL_RemoteTargetTransport -pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); -pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); diff --git a/crates/ecstore/src/heal/background_heal_ops.rs b/crates/ecstore/src/heal/background_heal_ops.rs deleted file mode 100644 index 0bc3e2591..000000000 --- a/crates/ecstore/src/heal/background_heal_ops.rs +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use futures::future::join_all; -use rustfs_madmin::heal_commands::HealResultItem; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; -use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration}; -use tokio::{ - spawn, - sync::{ - RwLock, - mpsc::{self, Receiver, Sender}, - }, - time::interval, -}; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; -use uuid::Uuid; - -use super::{ - heal_commands::HealOpts, - heal_ops::{HealSequence, new_bg_heal_sequence}, -}; -use crate::error::{Error, Result}; -use crate::global::{GLOBAL_MRFState, get_background_services_cancel_token}; -use crate::heal::error::ERR_RETRY_HEALING; -use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode}; -use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource}; -use crate::{ - config::RUSTFS_CONFIG_PREFIX, - disk::{BUCKET_META_PREFIX, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError}, - global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP}, - heal::{ - data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, - data_usage_cache::DataUsageCache, - heal_commands::{init_healing_tracker, load_healing_tracker}, - heal_ops::NOP_HEAL, - }, - new_object_layer_fn, - store::get_disk_via_endpoint, - store_api::{BucketInfo, BucketOptions, StorageAPI}, -}; - -pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10); - -pub async fn init_auto_heal() { - info!("Initializing auto heal background task"); - - let Some(cancel_token) = get_background_services_cancel_token() else { - error!("Background services cancel token not initialized"); - return; - }; - - init_background_healing().await; - let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string()); - if v == "on" { - info!("start monitor local disks and heal"); - GLOBAL_BackgroundHealState - .push_heal_local_disks(&get_local_disks_to_heal().await) - .await; - - let cancel_clone = cancel_token.clone(); - spawn(async move { - monitor_local_disks_and_heal(cancel_clone).await; - }); - } - - let cancel_clone = cancel_token.clone(); - spawn(async move { - GLOBAL_MRFState.heal_routine_with_cancel(cancel_clone).await; - }); -} - -async fn init_background_healing() { - let bg_seq = Arc::new(new_bg_heal_sequence()); - for _ in 0..GLOBAL_BackgroundHealRoutine.workers { - let bg_seq_clone = bg_seq.clone(); - spawn(async { - GLOBAL_BackgroundHealRoutine.add_worker(bg_seq_clone).await; - }); - } - let _ = GLOBAL_BackgroundHealState.launch_new_heal_sequence(bg_seq).await; -} - -pub async fn get_local_disks_to_heal() -> Vec { - let mut disks_to_heal = Vec::new(); - for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() { - if let Some(disk) = disk { - if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { - if err == DiskError::UnformattedDisk { - info!("get_local_disks_to_heal, disk is unformatted: {}", err); - disks_to_heal.push(disk.endpoint()); - } - } - let h = disk.healing().await; - if let Some(h) = h { - if !h.finished { - info!("get_local_disks_to_heal, disk healing not finished"); - disks_to_heal.push(disk.endpoint()); - } - } - } - } - - // todo - // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n { - - // } - disks_to_heal -} - -async fn monitor_local_disks_and_heal(cancel_token: CancellationToken) { - info!("Auto heal monitor started"); - let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL); - - loop { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("Auto heal monitor received shutdown signal, exiting gracefully"); - break; - } - _ = interval.tick() => { - let heal_disks = GLOBAL_BackgroundHealState.get_heal_local_disk_endpoints().await; - if heal_disks.is_empty() { - info!("heal local disks is empty"); - interval.reset(); - continue; - } - - info!("heal local disks: {:?}", heal_disks); - - let store = new_object_layer_fn().expect("errServerNotInitialized"); - if let (_result, Some(err)) = store.heal_format(false).await.expect("heal format failed") { - error!("heal local disk format error: {}", err); - if err == Error::NoHealRequired { - } else { - info!("heal format err: {}", err.to_string()); - interval.reset(); - continue; - } - } - - let mut futures = Vec::new(); - for disk in heal_disks.into_ref().iter() { - let disk_clone = disk.clone(); - let cancel_clone = cancel_token.clone(); - futures.push(async move { - let disk_for_cancel = disk_clone.clone(); - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Disk healing task cancelled for disk: {}", disk_for_cancel); - } - _ = async { - GLOBAL_BackgroundHealState - .set_disk_healing_status(disk_clone.clone(), true) - .await; - if heal_fresh_disk(&disk_clone).await.is_err() { - info!("heal_fresh_disk is err"); - GLOBAL_BackgroundHealState - .set_disk_healing_status(disk_clone.clone(), false) - .await; - } - GLOBAL_BackgroundHealState.pop_heal_local_disks(&[disk_clone]).await; - } => {} - } - }); - } - let _ = join_all(futures).await; - interval.reset(); - } - } - } -} - -async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> { - let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.set_idx as usize); - let disk = match get_disk_via_endpoint(endpoint).await { - Some(disk) => disk, - None => { - return Err(Error::other(format!( - "Unexpected error disk must be initialized by now after formatting: {endpoint}" - ))); - } - }; - - if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { - match err { - DiskError::DriveIsRoot => { - return Ok(()); - } - DiskError::UnformattedDisk => {} - _ => { - return Err(err.into()); - } - } - } - - let mut tracker = match load_healing_tracker(&Some(disk.clone())).await { - Ok(tracker) => tracker, - Err(err) => { - match err { - DiskError::FileNotFound => { - return Ok(()); - } - _ => { - info!( - "Unable to load healing tracker on '{}': {}, re-initializing..", - disk.to_string(), - err.to_string() - ); - } - } - init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await? - } - }; - - info!( - "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.", - endpoint.to_string() - ); - - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - let mut buckets = store.list_bucket(&BucketOptions::default()).await?; - buckets.push(BucketInfo { - name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)]) - .to_string_lossy() - .to_string(), - ..Default::default() - }); - buckets.push(BucketInfo { - name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]) - .to_string_lossy() - .to_string(), - ..Default::default() - }); - - buckets.sort_by(|a, b| { - let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET); - let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET); - - match (a_has_prefix, b_has_prefix) { - (true, false) => Ordering::Less, - (false, true) => Ordering::Greater, - _ => b.created.cmp(&a.created), - } - }); - - if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await { - let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new()); - tracker.objects_total_count = data_usage_info.objects_total_count; - tracker.objects_total_size = data_usage_info.objects_total_size; - }; - - tracker.set_queue_buckets(&buckets).await; - tracker.save().await?; - - let tracker = Arc::new(RwLock::new(tracker)); - let qb = tracker.read().await.queue_buckets.clone(); - store.pools[pool_idx].disk_set[set_idx] - .clone() - .heal_erasure_set(&qb, tracker.clone()) - .await?; - let mut tracker_w = tracker.write().await; - if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 { - tracker_w.retry_attempts += 1; - tracker_w.reset_healing().await; - if let Err(err) = tracker_w.update().await { - info!("update tracker failed: {}", err.to_string()); - } - return Err(Error::other(ERR_RETRY_HEALING)); - } - - if tracker_w.items_failed > 0 { - info!( - "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).", - disk.to_string(), - tracker_w.retry_attempts, - tracker_w.items_healed, - tracker_w.item_skipped, - tracker_w.items_failed - ); - } else if tracker_w.retry_attempts > 0 { - info!( - "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).", - disk.to_string(), - tracker_w.retry_attempts, - tracker_w.items_healed, - tracker_w.item_skipped - ); - } else { - info!( - "Healing of drive '{}' is finished (healed: {}, skipped: {}).", - disk.to_string(), - tracker_w.items_healed, - tracker_w.item_skipped - ); - } - - if tracker_w.heal_id.is_empty() { - if let Err(err) = tracker_w.delete().await { - error!("delete tracker failed: {}", err.to_string()); - } - } - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let disks = store.get_disks(pool_idx, set_idx).await?; - for disk in disks.into_iter() { - if disk.is_none() { - continue; - } - let mut tracker = match load_healing_tracker(&disk).await { - Ok(tracker) => tracker, - Err(err) => { - match err { - DiskError::FileNotFound => {} - _ => { - info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string()); - } - } - continue; - } - }; - if tracker.heal_id == tracker_w.heal_id { - tracker.finished = true; - tracker.update().await?; - } - } - Ok(()) -} - -#[derive(Debug)] -pub struct HealTask { - pub bucket: String, - pub object: String, - pub version_id: String, - pub opts: HealOpts, - pub resp_tx: Option>, - pub resp_rx: Option>, -} - -impl HealTask { - pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self { - Self { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - opts: *opts, - resp_tx: None, - resp_rx: None, - } - } -} - -#[derive(Debug)] -pub struct HealResult { - pub result: HealResultItem, - pub err: Option, -} - -pub struct HealRoutine { - pub tasks_tx: Sender, - tasks_rx: RwLock>, - workers: usize, -} - -impl HealRoutine { - pub fn new() -> Arc { - let mut workers = num_cpus::get() / 2; - if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") { - if let Ok(num_healers) = env_heal_workers.parse::() { - workers = num_healers; - } - } - - if workers == 0 { - workers = 4; - } - - let (tx, rx) = mpsc::channel(100); - Arc::new(Self { - tasks_tx: tx, - tasks_rx: RwLock::new(rx), - workers, - }) - } - - pub async fn add_worker(&self, bgseq: Arc) { - loop { - let mut d_res = HealResultItem::default(); - let d_err: Option; - match self.tasks_rx.write().await.recv().await { - Some(task) => { - info!("got task: {:?}", task); - if task.bucket == NOP_HEAL { - d_err = Some(Error::other("skip file")); - } else if task.bucket == SLASH_SEPARATOR { - match heal_disk_format(task.opts).await { - Ok((res, err)) => { - d_res = res; - d_err = err; - } - Err(err) => d_err = Some(err), - } - } else { - let store = new_object_layer_fn().expect("errServerNotInitialized"); - if task.object.is_empty() { - match store.heal_bucket(&task.bucket, &task.opts).await { - Ok(res) => { - d_res = res; - d_err = None; - } - Err(err) => d_err = Some(err), - } - } else { - match store - .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts) - .await - { - Ok((res, err)) => { - d_res = res; - d_err = err; - } - Err(err) => d_err = Some(err), - } - } - } - info!("task finished, task: {:?}", task); - if let Some(resp_tx) = task.resp_tx { - let _ = resp_tx - .send(HealResult { - result: d_res, - err: d_err, - }) - .await; - } else { - // when respCh is not set caller is not waiting but we - // update the relevant metrics for them - if d_err.is_none() { - bgseq.count_healed(d_res.heal_item_type).await; - } else { - bgseq.count_failed(d_res.heal_item_type).await; - } - } - } - None => { - info!("add_worker, tasks_rx was closed, return"); - return; - } - } - } - } -} - -// pub fn active_listeners() -> Result { - -// } - -async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option)> { - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - let (res, err) = store.heal_format(opts.dry_run).await?; - // return any error, ignore error returned when disks have - // already healed. - if err.is_some() { - return Ok((HealResultItem::default(), err)); - } - Ok((res, err)) -} - -pub(crate) async fn heal_bucket(bucket: &str) -> Result<()> { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if ok { - // bg_seq must be Some when ok is true - return bg_seq - .unwrap() - .queue_heal_task( - HealSource { - bucket: bucket.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_string(), - ) - .await; - } - Ok(()) -} - -pub(crate) async fn heal_object(bucket: &str, object: &str, version_id: &str, scan_mode: HealScanMode) -> Result<()> { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if ok { - // bg_seq must be Some when ok is true - return HealSequence::heal_object(bg_seq.unwrap(), bucket, object, version_id, scan_mode).await; - } - Ok(()) -} diff --git a/crates/ecstore/src/heal/data_scanner.rs b/crates/ecstore/src/heal/data_scanner.rs deleted file mode 100644 index 4449d81a7..000000000 --- a/crates/ecstore/src/heal/data_scanner.rs +++ /dev/null @@ -1,1826 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - collections::{HashMap, HashSet}, - fs, - future::Future, - io::{Cursor, Read}, - path::{Path, PathBuf}, - pin::Pin, - sync::{ - Arc, - atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, - }, - time::{Duration, SystemTime}, -}; - -use time::{self, OffsetDateTime}; - -use super::{ - data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, - data_usage::{DATA_USAGE_BLOOM_NAME_PATH, DataUsageInfo, store_data_usage_in_backend}, - data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash}, - heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, -}; -use crate::bucket::{ - object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion}, - utils::is_meta_bucketname, -}; -use crate::cmd::bucket_replication::queue_replication_heal; -use crate::event::name::EventName; -use crate::{ - bucket::{ - lifecycle::{ - bucket_lifecycle_audit::LcEventSrc, - bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object}, - lifecycle::{self, ExpirationOptions, Lifecycle}, - }, - metadata_sys, - }, - event_notification::{EventArgs, send_event}, - global::{GLOBAL_LocalNodeName, get_background_services_cancel_token}, - store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, -}; -use crate::{ - bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, - cmd::bucket_replication::ReplicationStatusType, - disk, - heal::data_usage::DATA_USAGE_ROOT, -}; -use crate::{ - cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, - config::{ - com::{read_config, save_config}, - heal::Config, - }, - disk::{DiskInfoOptions, DiskStore}, - global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasure, GLOBAL_IsErasureSD}, - heal::{ - data_usage::BACKGROUND_HEAL_INFO_PATH, - data_usage_cache::{DataUsageHashMap, hash_path}, - error::ERR_IGNORE_FILE_CONTRIB, - heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}, - heal_ops::{BG_HEALING_UUID, HealSource}, - }, - new_object_layer_fn, - store::ECStore, - store_utils::is_reserved_or_invalid_bucket, -}; -use crate::{disk::DiskAPI, store_api::ObjectInfo}; -use crate::{ - disk::error::DiskError, - error::{Error, Result}, -}; -use crate::{disk::local::LocalDisk, heal::data_scanner_metric::current_path_updater}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use rand::Rng; -use rmp_serde::{Deserializer, Serializer}; -use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_utils::path::encode_dir_object; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{ - BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, - ReplicationRuleStatus, -}; -use serde::{Deserialize, Serialize}; -use tokio::{ - sync::{ - RwLock, broadcast, - mpsc::{self, Sender}, - }, - time::sleep, -}; -use tracing::{debug, error, info}; - -const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders. -const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles. -const DATA_SCANNER_COMPACT_LEAST_OBJECT: u64 = 500; // Compact when there are less than this many objects in a branch. -const DATA_SCANNER_COMPACT_AT_CHILDREN: u64 = 10000; // Compact when there are this many children in a branch. -const DATA_SCANNER_COMPACT_AT_FOLDERS: u64 = DATA_SCANNER_COMPACT_AT_CHILDREN / 4; // Compact when this many subfolders in a single folder. -pub const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: u64 = 250_000; // Compact when this many subfolders in a single folder (even top level). -const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(60); // Time to wait on startup and between cycles. - -pub const HEAL_DELETE_DANGLING: bool = true; -const HEAL_OBJECT_SELECT_PROB: u64 = 1024; // Overall probability of a file being scanned; one in n. - -static SCANNER_CYCLE: AtomicU64 = AtomicU64::new(DATA_SCANNER_START_DELAY.as_secs()); -static _SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle -static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100); -static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB -static SCANNER_EXCESS_FOLDERS: AtomicU64 = AtomicU64::new(50_000); - -lazy_static! { - static ref SCANNER_SLEEPER: RwLock = RwLock::new(new_dynamic_sleeper(2.0, Duration::from_secs(1), true)); - pub static ref globalHealConfig: Arc> = Arc::new(RwLock::new(Config::default())); -} - -struct DynamicSleeper { - factor: f64, - max_sleep: Duration, - min_sleep: Duration, - _is_scanner: bool, -} - -type TimerFn = Pin + Send>>; -impl DynamicSleeper { - fn timer() -> TimerFn { - let t = SystemTime::now(); - Box::pin(async move { - let done_at = SystemTime::now().duration_since(t).unwrap_or_default(); - SCANNER_SLEEPER.read().await.sleep(done_at).await; - }) - } - - async fn sleep(&self, base: Duration) { - let (min_wait, max_wait) = (self.min_sleep, self.max_sleep); - let factor = self.factor; - - let want_sleep = { - let tmp = base.mul_f64(factor); - if tmp < min_wait { - return; - } - - if max_wait > Duration::from_secs(0) && tmp > max_wait { - max_wait - } else { - tmp - } - }; - sleep(want_sleep).await; - } - - fn _update(&mut self, factor: f64, max_wait: Duration) -> Result<()> { - if (self.factor - factor).abs() < 1e-10 && self.max_sleep == max_wait { - return Ok(()); - } - - self.factor = factor; - self.max_sleep = max_wait; - - Ok(()) - } -} - -fn new_dynamic_sleeper(factor: f64, max_wait: Duration, is_scanner: bool) -> DynamicSleeper { - DynamicSleeper { - factor, - max_sleep: max_wait, - min_sleep: Duration::from_micros(100), - _is_scanner: is_scanner, - } -} - -/// Initialize and start the data scanner in the background -/// -/// This function starts a background task that continuously runs the data scanner -/// with randomized intervals between cycles to avoid resource contention. -/// -/// # Features -/// - Graceful shutdown support via cancellation token -/// - Randomized sleep intervals to prevent synchronized scanning across nodes -/// - Minimum sleep duration to avoid excessive CPU usage -/// - Proper error handling and logging -/// -/// # Architecture -/// 1. Initialize with random seed for sleep intervals -/// 2. Run scanner cycles in a loop -/// 3. Use randomized sleep between cycles to avoid thundering herd -/// 4. Ensure minimum sleep duration to prevent CPU thrashing -pub async fn init_data_scanner() { - info!("Initializing data scanner background task"); - - let Some(cancel_token) = get_background_services_cancel_token() else { - error!("Background services cancel token not initialized"); - return; - }; - - let cancel_clone = cancel_token.clone(); - tokio::spawn(async move { - info!("Data scanner background task started"); - - loop { - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Data scanner received shutdown signal, exiting gracefully"); - break; - } - _ = run_data_scanner_cycle() => { - // Calculate randomized sleep duration - let random_factor = { - let mut rng = rand::rng(); - rng.random_range(1.0..10.0) - }; - let base_cycle_duration = SCANNER_CYCLE.load(Ordering::SeqCst) as f64; - let sleep_duration_secs = random_factor * base_cycle_duration; - - let sleep_duration = Duration::from_secs_f64(sleep_duration_secs); - - debug!( - duration_secs = sleep_duration.as_secs(), - "Data scanner sleeping before next cycle" - ); - - // Interruptible sleep - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Data scanner received shutdown signal during sleep, exiting"); - break; - } - _ = sleep(sleep_duration) => { - // Continue to next cycle - } - } - } - } - } - - info!("Data scanner background task stopped gracefully"); - }); -} - -/// Run a single data scanner cycle -/// -/// This function performs one complete scan cycle, including: -/// - Loading and updating cycle information -/// - Determining scan mode based on healing configuration -/// - Running the namespace scanner -/// - Saving cycle completion state -/// -/// # Error Handling -/// - Gracefully handles missing object layer -/// - Continues operation even if individual steps fail -/// - Logs errors appropriately without terminating the scanner -async fn run_data_scanner_cycle() { - debug!("Starting data scanner cycle"); - - // Get the object layer, return early if not available - let Some(store) = new_object_layer_fn() else { - error!("Object layer not initialized, skipping scanner cycle"); - return; - }; - - // Check for cancellation before starting expensive operations - if let Some(token) = get_background_services_cancel_token() { - if token.is_cancelled() { - debug!("Scanner cancelled before starting cycle"); - return; - } - } - - // Load current cycle information from persistent storage - let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .unwrap_or_else(|err| { - info!(error = %err, "Failed to read cycle info, starting fresh"); - Vec::new() - }); - - let mut cycle_info = if buf.is_empty() { - CurrentScannerCycle::default() - } else { - let mut buf_cursor = Deserializer::new(Cursor::new(buf)); - Deserialize::deserialize(&mut buf_cursor).unwrap_or_else(|err| { - error!(error = %err, "Failed to deserialize cycle info, using default"); - CurrentScannerCycle::default() - }) - }; - - // Start metrics collection for this cycle - let stop_fn = ScannerMetrics::log(ScannerMetric::ScanCycle); - - // Update cycle information - cycle_info.current = cycle_info.next; - cycle_info.started = Utc::now(); - - // Update global scanner metrics - globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await; - - // Read background healing information and determine scan mode - let bg_heal_info = read_background_heal_info(store.clone()).await; - let scan_mode = - get_cycle_scan_mode(cycle_info.current, bg_heal_info.bitrot_start_cycle, bg_heal_info.bitrot_start_time).await; - - // Update healing info if scan mode changed - if bg_heal_info.current_scan_mode != scan_mode { - let mut new_heal_info = bg_heal_info; - new_heal_info.current_scan_mode = scan_mode; - if scan_mode == HEAL_DEEP_SCAN { - new_heal_info.bitrot_start_time = SystemTime::now(); - new_heal_info.bitrot_start_cycle = cycle_info.current; - } - save_background_heal_info(store.clone(), &new_heal_info).await; - } - - // Set up data usage storage channel - let (tx, rx) = mpsc::channel::(100); - tokio::spawn(async move { - let _ = store_data_usage_in_backend(rx).await; - }); - - // Prepare result tracking - let mut scan_result = HashMap::new(); - scan_result.insert("cycle".to_string(), cycle_info.current.to_string()); - - info!( - cycle = cycle_info.current, - scan_mode = ?scan_mode, - "Starting namespace scanner" - ); - - // Run the namespace scanner with cancellation support - match execute_namespace_scan(&store, tx, cycle_info.current, scan_mode).await { - Ok(_) => { - info!(cycle = cycle_info.current, "Namespace scanner completed successfully"); - - // Update cycle completion information - cycle_info.next += 1; - cycle_info.current = 0; - cycle_info.cycle_completed.push(Utc::now()); - - // Maintain cycle completion history (keep only recent cycles) - if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize { - let _ = cycle_info.cycle_completed.remove(0); - } - - // Update global metrics with completion info - globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await; - - // Persist updated cycle information - // ignore error, continue. - let mut serialized_data = Vec::new(); - if let Err(err) = cycle_info.serialize(&mut Serializer::new(&mut serialized_data)) { - error!(error = %err, "Failed to serialize cycle info"); - } else if let Err(err) = save_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH, serialized_data).await { - error!(error = %err, "Failed to save cycle info to storage"); - } - } - Err(err) => { - error!( - cycle = cycle_info.current, - error = %err, - "Namespace scanner failed" - ); - scan_result.insert("error".to_string(), err.to_string()); - } - } - - // Complete metrics collection for this cycle - stop_fn(&scan_result); -} - -/// Execute namespace scan with cancellation support -async fn execute_namespace_scan( - store: &Arc, - tx: Sender, - cycle: u64, - scan_mode: HealScanMode, -) -> Result<()> { - let cancel_token = - get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?; - - tokio::select! { - result = store.ns_scanner(tx, cycle as usize, scan_mode) => { - result.map_err(|e| Error::other(format!("Namespace scan failed: {e}"))) - } - _ = cancel_token.cancelled() => { - info!("Namespace scan cancelled"); - Err(Error::other("Scan cancelled")) - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct BackgroundHealInfo { - bitrot_start_time: SystemTime, - bitrot_start_cycle: u64, - current_scan_mode: HealScanMode, -} - -impl Default for BackgroundHealInfo { - fn default() -> Self { - Self { - bitrot_start_time: SystemTime::now(), - bitrot_start_cycle: Default::default(), - current_scan_mode: Default::default(), - } - } -} - -async fn read_background_heal_info(store: Arc) -> BackgroundHealInfo { - if *GLOBAL_IsErasureSD.read().await { - return BackgroundHealInfo::default(); - } - - let buf = read_config(store, &BACKGROUND_HEAL_INFO_PATH) - .await - .map_or(Vec::new(), |buf| buf); - if buf.is_empty() { - return BackgroundHealInfo::default(); - } - serde_json::from_slice::(&buf).map_or(BackgroundHealInfo::default(), |b| b) -} - -async fn save_background_heal_info(store: Arc, info: &BackgroundHealInfo) { - if *GLOBAL_IsErasureSD.read().await { - return; - } - let b = match serde_json::to_vec(info) { - Ok(info) => info, - Err(_) => return, - }; - let _ = save_config(store, &BACKGROUND_HEAL_INFO_PATH, b).await; -} - -async fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: SystemTime) -> HealScanMode { - let bitrot_cycle = globalHealConfig.read().await.bitrot_scan_cycle(); - let v = bitrot_cycle.as_secs_f64(); - if v == -1.0 { - return HEAL_NORMAL_SCAN; - } else if v == 0.0 { - return HEAL_DEEP_SCAN; - } - - if current_cycle - bitrot_start_cycle < HEAL_OBJECT_SELECT_PROB { - return HEAL_DEEP_SCAN; - } - - if SystemTime::now().duration_since(bitrot_start_time).unwrap_or_default() > bitrot_cycle { - return HEAL_DEEP_SCAN; - } - - HEAL_NORMAL_SCAN -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CurrentScannerCycle { - pub current: u64, - pub next: u64, - pub started: DateTime, - pub cycle_completed: Vec>, -} - -impl Default for CurrentScannerCycle { - fn default() -> Self { - Self { - current: Default::default(), - next: Default::default(), - started: Utc::now(), - cycle_completed: Default::default(), - } - } -} - -impl CurrentScannerCycle { - pub fn marshal_msg(&self, next_buf: &[u8]) -> Result> { - let len: u32 = 4; - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // write "current" - rmp::encode::write_str(&mut wr, "current")?; - rmp::encode::write_uint(&mut wr, self.current)?; - - // write "next" - rmp::encode::write_str(&mut wr, "next")?; - rmp::encode::write_uint(&mut wr, self.next)?; - - // write "started" - rmp::encode::write_str(&mut wr, "started")?; - rmp::encode::write_sint(&mut wr, system_time_to_timestamp(&self.started))?; - - // write "cycle_completed" - rmp::encode::write_str(&mut wr, "cycle_completed")?; - let mut buf = Vec::new(); - self.cycle_completed - .serialize(&mut Serializer::new(&mut buf)) - .expect("Serialization failed"); - rmp::encode::write_bin(&mut wr, &buf)?; - let mut result = next_buf.to_vec(); - result.extend(wr.iter()); - Ok(result) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - match field.as_str() { - "current" => { - let u: u64 = rmp::decode::read_int(&mut cur)?; - self.current = u; - } - - // "next" => { - // let u: u64 = rmp::decode::read_int(&mut cur)?; - // self.next = u; - // } - "started" => { - let u: i64 = rmp::decode::read_int(&mut cur)?; - let started = timestamp_to_system_time(u); - self.started = started; - } - "cycleCompleted" => { - let mut buf = Vec::new(); - let _ = cur.read_to_end(&mut buf)?; - let u: Vec> = - Deserialize::deserialize(&mut Deserializer::new(&buf[..])).expect("Deserialization failed"); - self.cycle_completed = u; - } - name => return Err(Error::other(format!("not support field name {name}"))), - } - } - - Ok(cur.position()) - } -} - -// Convert `SystemTime` to timestamp -fn system_time_to_timestamp(time: &DateTime) -> i64 { - time.timestamp_micros() -} - -// Convert timestamp to `SystemTime` -fn timestamp_to_system_time(timestamp: i64) -> DateTime { - DateTime::from_timestamp_micros(timestamp).unwrap_or_default() -} - -#[derive(Clone, Debug, Default)] -pub struct Heal { - enabled: bool, - bitrot: bool, -} - -#[derive(Clone)] -pub struct ScannerItem { - pub path: String, - pub bucket: String, - pub prefix: String, - pub object_name: String, - pub replication: Option, - pub lifecycle: Option, - // typ: fs::Permissions, - pub heal: Heal, - pub debug: bool, -} - -impl ScannerItem { - pub fn transform_meta_dir(&mut self) { - let split = self.prefix.split(SLASH_SEPARATOR).map(PathBuf::from).collect::>(); - if split.len() > 1 { - self.prefix = path_join(&split[0..split.len() - 1]).to_string_lossy().to_string(); - } else { - self.prefix = "".to_string(); - } - self.object_name = split.last().map_or("".to_string(), |v| v.to_string_lossy().to_string()); - } - - pub fn object_path(&self) -> PathBuf { - path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())]) - } - - async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) { - let mut size = oi.get_actual_size().expect("err!"); - if self.debug { - info!("apply_lifecycle debug"); - } - if self.lifecycle.is_none() { - return (lifecycle::IlmAction::NoneAction, size); - } - - let version_id = oi.version_id; - - let mut vc = None; - let mut lr = None; - let mut rcfg = None; - if !is_meta_bucketname(&self.bucket) { - vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap()); - lr = BucketObjectLockSys::get(&self.bucket).await; - rcfg = (metadata_sys::get_replication_config(&self.bucket).await).ok(); - } - - let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; - if self.debug { - if version_id.is_some() { - info!( - "lifecycle: {} (version-id={}), Initial scan: {}", - self.object_path().to_string_lossy().to_string(), - version_id.expect("err"), - lc_evt.action - ); - } else { - info!( - "lifecycle: {} Initial scan: {}", - self.object_path().to_string_lossy().to_string(), - lc_evt.action - ); - } - } - - match lc_evt.action { - lifecycle::IlmAction::DeleteVersionAction - | lifecycle::IlmAction::DeleteAllVersionsAction - | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - size = 0; - } - lifecycle::IlmAction::DeleteAction => { - if !vc.unwrap().prefix_enabled(&oi.name) { - size = 0 - } - } - _ => (), - } - - apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; - (lc_evt.action, size) - } - - pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result> { - let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?; - if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize { - // todo - } - - let mut cumulative_size = 0; - for obj_info in obj_infos.iter() { - cumulative_size += obj_info.size; - } - - if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst) as i64 { - //todo - } - - Ok(obj_infos) - } - - pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result> { - // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent); - - let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { - rcfg.mode.is_some() - } else { - false - }; - let _vcfg = BucketVersioningSys::get(&self.bucket).await?; - - let versioned = match BucketVersioningSys::get(&self.bucket).await { - Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()), - Err(_) => false, - }; - let mut object_infos = Vec::with_capacity(fivs.len()); - - if self.lifecycle.is_none() { - for info in fivs.iter() { - object_infos.push(ObjectInfo::from_file_info( - info, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - return Ok(object_infos); - } - - let event = self - .lifecycle - .as_ref() - .expect("lifecycle err.") - .noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { - name: self.object_path().to_string_lossy().to_string(), - ..Default::default() - }) - .await; - let lim = event.newer_noncurrent_versions; - if lim == 0 || fivs.len() <= lim + 1 { - for fi in fivs.iter() { - object_infos.push(ObjectInfo::from_file_info( - fi, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - return Ok(object_infos); - } - - let overflow_versions = &fivs[lim + 1..]; - for fi in fivs[..lim + 1].iter() { - object_infos.push(ObjectInfo::from_file_info( - fi, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - - let mut to_del = Vec::::with_capacity(overflow_versions.len()); - for fi in overflow_versions.iter() { - let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); - if lock_enabled && enforce_retention_for_deletion(&obj) { - //if enforce_retention_for_deletion(&obj) { - if self.debug { - if obj.version_id.is_some() { - info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); - } else { - info!("lifecycle: {} is locked, not deleting\n", obj.name); - } - } - object_infos.push(obj); - continue; - } - - if OffsetDateTime::now_utc().unix_timestamp() - < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32) - .unix_timestamp() - { - object_infos.push(obj); - continue; - } - - to_del.push(ObjectToDelete { - object_name: obj.name, - version_id: obj.version_id, - }); - } - - if !to_del.is_empty() { - let mut expiry_state = GLOBAL_ExpiryState.write().await; - expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await; - } - // done().await; - - Ok(object_infos) - } - - pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, i64) { - let done = ScannerMetrics::time(ScannerMetric::Ilm); - - let (action, _size) = self.apply_lifecycle(oi).await; - - info!( - "apply_actions {} {} {:?} {:?}", - oi.bucket.clone(), - oi.name.clone(), - oi.version_id.clone(), - oi.user_defined.clone() - ); - - // Create a mutable clone if you need to modify fields - let mut oi = oi.clone(); - - let versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; - if versioned { - oi.replication_status = ReplicationStatusType::from( - oi.user_defined - .get("x-amz-bucket-replication-status") - .unwrap_or(&"PENDING".to_string()), - ); - debug!("apply status is: {:?}", oi.replication_status); - self.heal_replication(&oi, _size_s).await; - } - - done(); - - if action.delete_all() { - return (true, 0); - } - - (false, oi.size) - } - - pub async fn heal_replication(&mut self, oi: &ObjectInfo, size_s: &mut SizeSummary) { - if oi.version_id.is_none() { - error!( - "heal_replication: no version_id or replication config {} {} {}", - oi.bucket, - oi.name, - oi.version_id.is_none() - ); - return; - } - - //let config = s3s::dto::ReplicationConfiguration{ role: todo!(), rules: todo!() }; - // Use the provided variable instead of borrowing self mutably. - let replication = match metadata_sys::get_replication_config(&oi.bucket).await { - Ok((replication, _)) => replication, - Err(_) => { - error!( - "heal_replication: failed to get replication config for bucket: {} and object name: {}", - oi.bucket, oi.name - ); - return; - } - }; - if replication.rules.is_empty() { - error!("heal_replication: no replication rules for bucket {} {}", oi.bucket, oi.name); - return; - } - if replication.role.is_empty() { - // error!("heal_replication: no replication role for bucket {} {}", oi.bucket, oi.name); - // return; - } - - //if oi.delete_marker || !oi.version_purge_status.is_empty() { - if oi.delete_marker { - error!( - "heal_replication: delete marker or version purge status {} {} {:?} {} {:?}", - oi.bucket, oi.name, oi.version_id, oi.delete_marker, oi.version_purge_status - ); - return; - } - - if oi.replication_status == ReplicationStatusType::Completed { - return; - } - - info!("replication status is: {:?} and user define {:?}", oi.replication_status, oi.user_defined); - - let roi = queue_replication_heal(&oi.bucket, oi, &replication, 3).await; - - if roi.is_none() { - info!("not need heal {} {} {:?}", oi.bucket, oi.name, oi.version_id); - return; - } - - for (arn, tgt_status) in &roi.unwrap().target_statuses { - let tgt_size_s = size_s.repl_target_stats.entry(arn.clone()).or_default(); - - match tgt_status { - ReplicationStatusType::Pending => { - tgt_size_s.pending_count += 1; - tgt_size_s.pending_size += oi.size as usize; - size_s.pending_count += 1; - size_s.pending_size += oi.size as usize; - } - ReplicationStatusType::Failed => { - tgt_size_s.failed_count += 1; - tgt_size_s.failed_size += oi.size as usize; - size_s.failed_count += 1; - size_s.failed_size += oi.size as usize; - } - ReplicationStatusType::Completed | ReplicationStatusType::CompletedLegacy => { - tgt_size_s.replicated_count += 1; - tgt_size_s.replicated_size += oi.size as usize; - size_s.replicated_count += 1; - size_s.replicated_size += oi.size as usize; - } - _ => {} - } - } - - if matches!(oi.replication_status, ReplicationStatusType::Replica) { - size_s.replica_count += 1; - size_s.replica_size += oi.size as usize; - } - } -} - -#[derive(Debug, Default)] -pub struct SizeSummary { - pub total_size: usize, - pub versions: usize, - pub delete_markers: usize, - pub replicated_size: usize, - pub replicated_count: usize, - pub pending_size: usize, - pub failed_size: usize, - pub replica_size: usize, - pub replica_count: usize, - pub pending_count: usize, - pub failed_count: usize, - pub repl_target_stats: HashMap, - // Todo: tires -} - -#[derive(Debug, Default)] -pub struct ReplTargetSizeSummary { - pub replicated_size: usize, - pub replicated_count: usize, - pub pending_size: usize, - pub failed_size: usize, - pub pending_count: usize, - pub failed_count: usize, -} - -#[derive(Debug, Clone)] -struct CachedFolder { - name: String, - parent: DataUsageHash, - object_heal_prob_div: u32, -} - -pub type GetSizeFn = - Box Pin> + Send>> + Send + Sync + 'static>; -pub type UpdateCurrentPathFn = Arc Pin + Send>> + Send + Sync + 'static>; -pub type ShouldSleepFn = Option bool + Send + Sync + 'static>>; - -struct FolderScanner { - root: String, - get_size: GetSizeFn, - old_cache: DataUsageCache, - new_cache: DataUsageCache, - update_cache: DataUsageCache, - data_usage_scanner_debug: bool, - heal_object_select: u32, - scan_mode: HealScanMode, - disks: Vec>, - disks_quorum: usize, - updates: Sender, - last_update: SystemTime, - update_current_path: UpdateCurrentPathFn, - skip_heal: AtomicBool, - drive: LocalDrive, - we_sleep: ShouldSleepFn, -} - -impl FolderScanner { - async fn should_heal(&self) -> bool { - if self.skip_heal.load(Ordering::SeqCst) { - return false; - } - if self.heal_object_select == 0 { - return false; - } - if let Ok(info) = self.drive.disk_info(&DiskInfoOptions::default()).await { - if info.healing { - self.skip_heal.store(true, Ordering::SeqCst); - return false; - } - } - true - } - - #[tracing::instrument(level = "info", skip_all)] - async fn scan_folder(&mut self, folder: &CachedFolder, into: &mut DataUsageEntry) -> Result<()> { - let this_hash = hash_path(&folder.name); - let was_compacted = into.compacted; - - 'outer: { - let mut abandoned_children: DataUsageHashMap = if !into.compacted { - self.old_cache.find_children_copy(this_hash.clone()) - } else { - HashSet::new() - }; - - let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name); - // Todo: lifeCycle - let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() { - if lc_has_active_rules(lc, &prefix) { - self.old_cache.info.lifecycle.clone() - } else { - None - } - } else { - None - }; - - let replication_cfg = if self.old_cache.info.replication.is_some() - && rep_has_active_rules(self.old_cache.info.replication.as_ref().unwrap(), &prefix, true) - { - self.old_cache.info.replication.clone() - } else { - None - }; - - if let Some(should_sleep) = &self.we_sleep { - if should_sleep() { - SCANNER_SLEEPER.read().await.sleep(DATA_SCANNER_SLEEP_PER_FOLDER).await; - } - } - - let mut existing_folders = Vec::new(); - let mut new_folders = Vec::new(); - let mut found_objects: bool = false; - - let path = Path::new(&self.root).join(&folder.name); - if path.is_dir() { - for entry in fs::read_dir(path)? { - let entry = entry?; - let sub_path = entry.path(); - let ent_name = Path::new(&folder.name).join(&sub_path); - let (bucket, prefix) = path_to_bucket_object_with_base_path(&self.root, ent_name.to_str().unwrap()); - if bucket.is_empty() { - continue; - } - if is_reserved_or_invalid_bucket(&bucket, false) { - continue; - } - - if sub_path.is_dir() { - let h = hash_path(ent_name.to_str().unwrap()); - if h == this_hash { - continue; - } - let this = CachedFolder { - name: ent_name.to_string_lossy().to_string(), - parent: this_hash.clone(), - object_heal_prob_div: folder.object_heal_prob_div, - }; - abandoned_children.remove(&h.key()); - if self.old_cache.cache.contains_key(&h.key()) { - existing_folders.push(this); - self.update_cache - .copy_with_children(&self.old_cache, &h, &Some(this_hash.clone())); - } else { - new_folders.push(this); - } - continue; - } - - let _wait = if let Some(should_sleep) = &self.we_sleep { - if should_sleep() { - DynamicSleeper::timer() - } else { - Box::pin(async {}) - } - } else { - Box::pin(async {}) - }; - - let mut item = ScannerItem { - path: Path::new(&self.root).join(&ent_name).to_string_lossy().to_string(), - bucket, - prefix: Path::new(&prefix) - .parent() - .unwrap_or(Path::new("")) - .to_string_lossy() - .to_string(), - object_name: ent_name - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(), - debug: self.data_usage_scanner_debug, - replication: replication_cfg.clone(), - lifecycle: active_life_cycle.clone(), - heal: Heal::default(), - }; - - item.heal.enabled = this_hash.mod_alt( - self.old_cache.info.next_cycle / folder.object_heal_prob_div, - self.heal_object_select / folder.object_heal_prob_div, - ) && self.should_heal().await; - item.heal.bitrot = self.scan_mode == HEAL_DEEP_SCAN; - - let (sz, err) = match (self.get_size)(&item).await { - Ok(sz) => (sz, None), - Err(err) => { - if err.to_string() != ERR_IGNORE_FILE_CONTRIB { - continue; - } - (SizeSummary::default(), Some(err)) - } - }; - // successfully read means we have a valid object. - found_objects = true; - // Remove filename i.e is the meta file to construct object name - item.transform_meta_dir(); - // Object already accounted for, remove from heal map, - // simply because getSize() function already heals the - // object. - abandoned_children.remove( - &path_join(&[PathBuf::from(item.bucket.clone()), item.object_path()]) - .to_string_lossy() - .to_string(), - ); - - if err.is_none() || err.unwrap().to_string() != ERR_IGNORE_FILE_CONTRIB { - into.add_sizes(&sz); - into.objects += 1; - } - } - } - // if found_objects && *GLOBAL_IsErasure.read().await { - if found_objects { - break 'outer; - } - - let should_compact = self.new_cache.info.name != folder.name - && (existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS as usize - || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS as usize); - - let total_folders = existing_folders.len() + new_folders.len(); - if total_folders > SCANNER_EXCESS_FOLDERS.load(Ordering::SeqCst) as usize { - let _prefix_name = format!("{}/", folder.name.trim_end_matches('/')); - // todo: notification - } - - if !into.compacted && should_compact { - into.compacted = true; - new_folders.extend(existing_folders.clone()); - existing_folders.clear(); - } - - // Transfer existing - if !into.compacted { - for folder in existing_folders.iter() { - let h = hash_path(&folder.name); - self.update_cache - .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); - } - } - - // Scan new... - for folder in new_folders.iter() { - let h = hash_path(&folder.name); - if !into.compacted { - let mut found_any = false; - let mut parent = this_hash.clone(); - while parent != hash_path(&self.update_cache.info.name) { - let e = self.update_cache.find(&parent.key()); - if e.is_none() || e.as_ref().unwrap().compacted { - found_any = true; - break; - } - match self.update_cache.search_parent(&parent) { - Some(next) => { - parent = next; - } - None => { - found_any = true; - break; - } - } - } - if !found_any { - self.update_cache - .replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default()); - } - } - (self.update_current_path)(&folder.name).await; - scan(folder, into, self).await; - // Add new folders if this is new and we don't have existing. - if !into.compacted { - if let Some(parent) = self.update_cache.find(&this_hash.key()) { - if !parent.compacted { - self.update_cache.delete_recursive(&h); - self.update_cache - .copy_with_children(&self.new_cache, &h, &Some(this_hash.clone())); - } - } - } - } - - // Scan existing... - for folder in existing_folders.iter() { - let h = hash_path(&folder.name); - if !into.compacted - && self.old_cache.is_compacted(&h) - && !h.mod_(self.old_cache.info.next_cycle, DATA_USAGE_UPDATE_DIR_CYCLES) - { - self.new_cache - .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); - into.add_child(&h); - continue; - } - (self.update_current_path)(&folder.name).await; - scan(folder, into, self).await; - } - - // Scan for healing - if abandoned_children.is_empty() || !self.should_heal().await { - break 'outer; - } - - if self.disks.is_empty() || self.disks_quorum == 0 { - break 'outer; - } - - let (bg_seq, found) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if !found { - break 'outer; - } - let bg_seq = bg_seq.unwrap(); - - let mut resolver = MetadataResolutionParams { - dir_quorum: self.disks_quorum, - obj_quorum: self.disks_quorum, - bucket: "".to_string(), - strict: false, - ..Default::default() - }; - - for k in abandoned_children.iter() { - if !self.should_heal().await { - break; - } - - let (bucket, prefix) = path_to_bucket_object(k); - (self.update_current_path)(k).await; - - if bucket != resolver.bucket { - bg_seq - .clone() - .queue_heal_task( - HealSource { - bucket: bucket.clone(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_owned(), - ) - .await?; - } - - resolver.bucket = bucket.clone(); - let found_objs = Arc::new(RwLock::new(false)); - let found_objs_clone = found_objs.clone(); - let (tx, rx) = broadcast::channel(1); - // let tx_partial = tx.clone(); - let tx_finished = tx.clone(); - let update_current_path_agreed = self.update_current_path.clone(); - let update_current_path_partial = self.update_current_path.clone(); - let resolver_clone = resolver.clone(); - let bg_seq_clone = bg_seq.clone(); - let lopts = ListPathRawOptions { - disks: self.disks.clone(), - bucket: bucket.clone(), - path: prefix.clone(), - recursive: true, - report_not_found: true, - min_disks: self.disks_quorum, - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - Box::pin({ - let update_current_path_agreed = update_current_path_agreed.clone(); - async move { - update_current_path_agreed(&entry.name).await; - } - }) - })), - partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - Box::pin({ - let update_current_path_partial = update_current_path_partial.clone(); - // let tx_partial = tx_partial.clone(); - let resolver_partial = resolver_clone.clone(); - let bucket_partial = bucket.clone(); - let found_objs_clone = found_objs_clone.clone(); - let bg_seq_partial = bg_seq_clone.clone(); - async move { - // Todo - // if !fs.should_heal().await { - // let _ = tx_partial.send(true); - // return; - // } - let entry = match entries.resolve(resolver_partial) { - Some(entry) => entry, - _ => match entries.first_found() { - (Some(entry), _) => entry, - _ => return, - }, - }; - - update_current_path_partial(&entry.name).await; - let mut custom = HashMap::new(); - if entry.is_dir() { - return; - } - - // We got an entry which we should be able to heal. - let fiv = match entry.file_info_versions(&bucket_partial) { - Ok(fiv) => fiv, - Err(_) => { - if let Err(err) = bg_seq_partial - .queue_heal_task( - HealSource { - bucket: bucket_partial.clone(), - object: entry.name.clone(), - version_id: "".to_string(), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await - { - match err { - Error::FileNotFound | Error::FileVersionNotFound => {} - _ => { - info!("{}", err.to_string()); - } - } - } else { - let mut w = found_objs_clone.write().await; - *w = true; - } - return; - } - }; - - custom.insert("versions", fiv.versions.len().to_string()); - let (mut success_versions, mut fail_versions) = (0, 0); - for ver in fiv.versions.iter() { - match bg_seq_partial - .queue_heal_task( - HealSource { - bucket: bucket_partial.clone(), - object: fiv.name.clone(), - version_id: ver.version_id.map_or("".to_string(), |ver_id| ver_id.to_string()), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await - { - Ok(_) => { - success_versions += 1; - - let mut w = found_objs_clone.write().await; - *w = true; - } - Err(_) => { - fail_versions += 1; - } - } - } - custom.insert("success_versions", success_versions.to_string()); - custom.insert("failed_versions", fail_versions.to_string()); - } - }) - })), - finished: Some(Box::new(move |_: &[Option]| { - Box::pin({ - let tx_finished = tx_finished.clone(); - async move { - let _ = tx_finished.send(true); - } - }) - })), - ..Default::default() - }; - let _ = list_path_raw(rx, lopts).await; - - if *found_objs.read().await { - let this: CachedFolder = CachedFolder { - name: k.clone(), - parent: this_hash.clone(), - object_heal_prob_div: 1, - }; - scan(&this, into, self).await; - } - } - } - if !was_compacted { - self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), into); - } - - if !into.compacted && self.new_cache.info.name != folder.name { - let mut flat = self.new_cache.size_recursive(&this_hash.key()).unwrap_or_default(); - flat.compacted = true; - let compact = if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT as usize { - true - } else { - // Compact if we only have objects as children... - let mut compact = true; - for k in into.children.iter() { - if let Some(v) = self.new_cache.cache.get(k) { - if !v.children.is_empty() || v.objects > 1 { - compact = false; - break; - } - } - } - compact - }; - if compact { - self.new_cache.delete_recursive(&this_hash); - self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); - let mut total: HashMap = HashMap::new(); - total.insert("objects".to_string(), flat.objects.to_string()); - total.insert("size".to_string(), flat.size.to_string()); - if flat.versions > 0 { - total.insert("versions".to_string(), flat.versions.to_string()); - } - } - } - // Compact if too many children... - if !into.compacted { - self.new_cache.reduce_children_of( - &this_hash, - DATA_SCANNER_COMPACT_AT_CHILDREN as usize, - self.new_cache.info.name != folder.name, - ); - } - if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted { - // Replace if existed before. - if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) { - self.update_cache.delete_recursive(&this_hash); - self.update_cache - .replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); - } - } - Ok(()) - } - - #[tracing::instrument(level = "info", skip_all)] - async fn send_update(&mut self) { - if SystemTime::now().duration_since(self.last_update).unwrap() < Duration::from_secs(60) { - return; - } - if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name) { - let _ = self.updates.send(flat).await; - self.last_update = SystemTime::now(); - } - } -} - -#[tracing::instrument(level = "info", skip(into, folder_scanner))] -async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) { - let mut dst = if !into.compacted { - DataUsageEntry::default() - } else { - into.clone() - }; - - if Box::pin(folder_scanner.scan_folder(folder, &mut dst)).await.is_err() { - return; - } - if !into.compacted { - let h = DataUsageHash(folder.name.clone()); - into.add_child(&h); - folder_scanner.update_cache.delete_recursive(&h); - folder_scanner - .update_cache - .copy_with_children(&folder_scanner.new_cache, &h, &Some(folder.parent.clone())); - folder_scanner.send_update().await; - } -} - -fn lc_get_prefix(rule: &LifecycleRule) -> String { - if let Some(p) = &rule.prefix { - return p.to_string(); - } else if let Some(filter) = &rule.filter { - if let Some(p) = &filter.prefix { - return p.to_string(); - } else if let Some(and) = &filter.and { - if let Some(p) = &and.prefix { - return p.to_string(); - } - } - } - - "".into() -} - -pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool { - if config.rules.is_empty() { - return false; - } - - for rule in config.rules.iter() { - if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) { - continue; - } - let rule_prefix = lc_get_prefix(rule); - if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix) - { - continue; - } - - if let Some(e) = &rule.noncurrent_version_expiration { - if let Some(true) = e.noncurrent_days.map(|d| d > 0) { - return true; - } - if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) { - return true; - } - } - - if rule.noncurrent_version_transitions.is_some() { - return true; - } - if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) { - return true; - } - - if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) { - return true; - } - - if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) { - return true; - } - - if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) { - return true; - } - - if rule.transitions.is_some() { - return true; - } - } - false -} - -pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool { - if config.rules.is_empty() { - return false; - } - - for rule in config.rules.iter() { - if rule - .status - .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)) - { - continue; - } - if !prefix.is_empty() { - if let Some(filter) = &rule.filter { - if let Some(r_prefix) = &filter.prefix { - if !r_prefix.is_empty() { - // incoming prefix must be in rule prefix - if !recursive && !prefix.starts_with(r_prefix) { - continue; - } - // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix - // does not match - if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) { - continue; - } - } - } - } - } - return true; - } - false -} - -pub type LocalDrive = Arc; -pub async fn scan_data_folder( - disks: &[Option], - drive: LocalDrive, - cache: &DataUsageCache, - get_size_fn: GetSizeFn, - heal_scan_mode: HealScanMode, - should_sleep: ShouldSleepFn, -) -> disk::error::Result { - if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT { - return Err(DiskError::other("internal error: root scan attempted")); - } - - let base_path = drive.to_string(); - let (update_path, close_disk) = current_path_updater(&base_path, &cache.info.name); - let skip_heal = if *GLOBAL_IsErasure.read().await || cache.info.skip_healing { - AtomicBool::new(true) - } else { - AtomicBool::new(false) - }; - let mut s = FolderScanner { - root: base_path, - get_size: get_size_fn, - old_cache: cache.clone(), - new_cache: DataUsageCache { - info: cache.info.clone(), - ..Default::default() - }, - update_cache: DataUsageCache { - info: cache.info.clone(), - ..Default::default() - }, - data_usage_scanner_debug: false, - heal_object_select: 0, - scan_mode: heal_scan_mode, - updates: cache.info.updates.clone().unwrap(), - last_update: SystemTime::now(), - update_current_path: update_path, - disks: disks.to_vec(), - disks_quorum: disks.len() / 2, - skip_heal, - drive: drive.clone(), - we_sleep: should_sleep, - }; - - if *GLOBAL_IsErasure.read().await || !cache.info.skip_healing { - s.heal_object_select = HEAL_OBJECT_SELECT_PROB as u32; - } - - let mut root = DataUsageEntry::default(); - let folder = CachedFolder { - name: cache.info.name.clone(), - object_heal_prob_div: 1, - parent: DataUsageHash("".to_string()), - }; - - if s.scan_folder(&folder, &mut root).await.is_err() { - close_disk().await; - } - s.new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN as usize); - s.new_cache.info.last_update = Some(SystemTime::now()); - s.new_cache.info.next_cycle = cache.info.next_cycle; - close_disk().await; - Ok(s.new_cache) -} - -pub async fn eval_action_from_lifecycle( - lc: &BucketLifecycleConfiguration, - lr: Option, - rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, - oi: &ObjectInfo, -) -> lifecycle::Event { - let event = lc.eval(&oi.to_lifecycle_opts()).await; - //if serverDebugLog { - info!("lifecycle: Secondary scan: {}", event.action); - //} - - let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; - - match event.action { - lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - if lock_enabled { - return lifecycle::Event::default(); - } - } - lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => { - if oi.version_id.is_none() { - return lifecycle::Event::default(); - } - if lock_enabled && enforce_retention_for_deletion(oi) { - //if serverDebugLog { - if oi.version_id.is_some() { - info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); - } else { - info!("lifecycle: {} is locked, not deleting", oi.name); - } - //} - return lifecycle::Event::default(); - } - if let Some(rcfg) = rcfg { - if rep_has_active_rules(&rcfg.0, &oi.name, true) { - return lifecycle::Event::default(); - } - } - } - _ => (), - } - - event -} - -async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - if oi.delete_marker || oi.is_dir { - return false; - } - GLOBAL_TransitionState.queue_transition_task(oi, event, src).await; - true -} - -pub async fn apply_expiry_on_transitioned_object( - api: Arc, - oi: &ObjectInfo, - lc_event: &lifecycle::Event, - src: &LcEventSrc, -) -> bool { - let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); - if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { - return false; - } - let _ = time_ilm(1); - - true -} - -pub async fn apply_expiry_on_non_transitioned_objects( - api: Arc, - oi: &ObjectInfo, - lc_event: &lifecycle::Event, - _src: &LcEventSrc, -) -> bool { - let mut opts = ObjectOptions { - expiration: ExpirationOptions { expire: true }, - ..Default::default() - }; - - if lc_event.action.delete_versioned() { - opts.version_id = Some(oi.version_id.expect("err").to_string()); - } - - opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; - opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await; - - if lc_event.action.delete_all() { - opts.delete_prefix = true; - opts.delete_prefix_object = true; - } - - let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); - - let mut dobj = api - .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts) - .await - .unwrap(); - if dobj.name.is_empty() { - dobj = oi.clone(); - } - - //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags(); - //tags["version-id"] = dobj.version_id; - - let mut event_name = EventName::ObjectRemovedDelete; - if oi.delete_marker { - event_name = EventName::ObjectRemovedDeleteMarkerCreated; - } - match lc_event.action { - lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions, - lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete, - _ => (), - } - send_event(EventArgs { - event_name: event_name.as_ref().to_string(), - bucket_name: dobj.bucket.clone(), - object: dobj, - user_agent: "Internal: [ILM-Expiry]".to_string(), - host: GLOBAL_LocalNodeName.to_string(), - ..Default::default() - }); - - if lc_event.action != lifecycle::IlmAction::NoneAction { - let mut num_versions = 1_u64; - if lc_event.action.delete_all() { - num_versions = oi.num_versions as u64; - } - let _ = time_ilm(num_versions); - } - - true -} - -async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - let mut expiry_state = GLOBAL_ExpiryState.write().await; - expiry_state.enqueue_by_days(oi, event, src).await; - true -} - -pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - let mut success = false; - match event.action { - lifecycle::IlmAction::DeleteVersionAction - | lifecycle::IlmAction::DeleteAction - | lifecycle::IlmAction::DeleteRestoredAction - | lifecycle::IlmAction::DeleteRestoredVersionAction - | lifecycle::IlmAction::DeleteAllVersionsAction - | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - success = apply_expiry_rule(event, src, oi).await; - } - lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { - success = apply_transition_rule(event, src, oi).await; - } - _ => (), - } - success -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use chrono::Utc; - use rmp_serde::{Deserializer, Serializer}; - use serde::{Deserialize, Serialize}; - - use super::CurrentScannerCycle; - - #[test] - fn test_current_cycle() { - let cycle_info = CurrentScannerCycle { - current: 0, - next: 1, - started: Utc::now(), - cycle_completed: vec![Utc::now(), Utc::now()], - }; - - println!("{cycle_info:?}"); - - let mut wr = Vec::new(); - cycle_info.serialize(&mut Serializer::new(&mut wr)).unwrap(); - - let mut buf_t = Deserializer::new(Cursor::new(wr)); - let c: CurrentScannerCycle = Deserialize::deserialize(&mut buf_t).unwrap(); - - println!("{c:?}"); - } -} diff --git a/crates/ecstore/src/heal/data_usage.rs b/crates/ecstore/src/heal/data_usage.rs deleted file mode 100644 index a6d569971..000000000 --- a/crates/ecstore/src/heal/data_usage.rs +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::error::{Error, Result}; -use crate::{ - bucket::metadata_sys::get_replication_config, - config::com::{read_config, save_config}, - disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, - error::to_object_err, - new_object_layer_fn, - store::ECStore, -}; -use lazy_static::lazy_static; -use rustfs_utils::path::SLASH_SEPARATOR; -use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, sync::Arc, time::SystemTime}; -use tokio::sync::mpsc::Receiver; -use tracing::{error, warn}; - -pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; -const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; -const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; -pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; -lazy_static! { - pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX); - pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME); - pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = - format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME); - pub static ref BACKGROUND_HEAL_INFO_PATH: String = - format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json"); -} - -// BucketTargetUsageInfo - bucket target usage info provides -// - replicated size for all objects sent to this target -// - replica size for all objects received from this target -// - replication pending size for all objects pending replication to this target -// - replication failed size for all objects failed replication to this target -// - replica pending count -// - replica failed count -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BucketTargetUsageInfo { - pub replication_pending_size: u64, - pub replication_failed_size: u64, - pub replicated_size: u64, - pub replica_size: u64, - pub replication_pending_count: u64, - pub replication_failed_count: u64, - pub replicated_count: u64, -} - -// BucketUsageInfo - bucket usage info provides -// - total size of the bucket -// - total objects in a bucket -// - object size histogram per bucket -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BucketUsageInfo { - pub size: u64, - // Following five fields suffixed with V1 are here for backward compatibility - // Total Size for objects that have not yet been replicated - pub replication_pending_size_v1: u64, - // Total size for objects that have witness one or more failures and will be retried - pub replication_failed_size_v1: u64, - // Total size for objects that have been replicated to destination - pub replicated_size_v1: u64, - // Total number of objects pending replication - pub replication_pending_count_v1: u64, - // Total number of objects that failed replication - pub replication_failed_count_v1: u64, - - pub objects_count: u64, - pub object_size_histogram: HashMap, - pub object_versions_histogram: HashMap, - pub versions_count: u64, - pub delete_markers_count: u64, - pub replica_size: u64, - pub replica_count: u64, - pub replication_info: HashMap, -} - -// DataUsageInfo represents data usage stats of the underlying Object API -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct DataUsageInfo { - pub total_capacity: u64, - pub total_used_capacity: u64, - pub total_free_capacity: u64, - - // LastUpdate is the timestamp of when the data usage info was last updated. - // This does not indicate a full scan. - pub last_update: Option, - - // Objects total count across all buckets - pub objects_total_count: u64, - // Versions total count across all buckets - pub versions_total_count: u64, - // Delete markers total count across all buckets - pub delete_markers_total_count: u64, - // Objects total size across all buckets - pub objects_total_size: u64, - pub replication_info: HashMap, - - // Total number of buckets in this cluster - pub buckets_count: u64, - // Buckets usage info provides following information across all buckets - // - total size of the bucket - // - total objects in a bucket - // - object size histogram per bucket - pub buckets_usage: HashMap, - // Deprecated kept here for backward compatibility reasons. - pub bucket_sizes: HashMap, - // Todo: TierStats - // TierStats contains per-tier stats of all configured remote tiers -} - -pub async fn store_data_usage_in_backend(mut rx: Receiver) { - let Some(store) = new_object_layer_fn() else { - error!("errServerNotInitialized"); - return; - }; - - let mut attempts = 1; - loop { - match rx.recv().await { - Some(data_usage_info) => { - if let Ok(data) = serde_json::to_vec(&data_usage_info) { - if attempts > 10 { - let _ = - save_config(store.clone(), &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), data.clone()).await; - attempts += 1; - } - let _ = save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await; - attempts += 1; - } else { - continue; - } - } - None => { - return; - } - } - } -} - -// TODO: cancel ctx -pub async fn load_data_usage_from_backend(store: Arc) -> Result { - let buf = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await { - Ok(data) => data, - Err(e) => { - error!("Failed to read data usage info from backend: {}", e); - if e == Error::ConfigNotFound { - return Ok(DataUsageInfo::default()); - } - - return Err(to_object_err(e, vec![RUSTFS_META_BUCKET, &DATA_USAGE_OBJ_NAME_PATH])); - } - }; - - let mut data_usage_info: DataUsageInfo = serde_json::from_slice(&buf)?; - - warn!("Loaded data usage info from backend {:?}", &data_usage_info); - - if data_usage_info.buckets_usage.is_empty() { - data_usage_info.buckets_usage = data_usage_info - .bucket_sizes - .iter() - .map(|(bucket, &size)| { - ( - bucket.clone(), - BucketUsageInfo { - size, - ..Default::default() - }, - ) - }) - .collect(); - } - - if data_usage_info.bucket_sizes.is_empty() { - data_usage_info.bucket_sizes = data_usage_info - .buckets_usage - .iter() - .map(|(bucket, bui)| (bucket.clone(), bui.size)) - .collect(); - } - - for (bucket, bui) in &data_usage_info.buckets_usage { - if bui.replicated_size_v1 > 0 - || bui.replication_failed_count_v1 > 0 - || bui.replication_failed_size_v1 > 0 - || bui.replication_pending_count_v1 > 0 - { - if let Ok((cfg, _)) = get_replication_config(bucket).await { - if !cfg.role.is_empty() { - data_usage_info.replication_info.insert( - cfg.role.clone(), - BucketTargetUsageInfo { - replication_failed_size: bui.replication_failed_size_v1, - replication_failed_count: bui.replication_failed_count_v1, - replicated_size: bui.replicated_size_v1, - replication_pending_count: bui.replication_pending_count_v1, - replication_pending_size: bui.replication_pending_size_v1, - ..Default::default() - }, - ); - } - } - } - } - - Ok(data_usage_info) -} diff --git a/crates/ecstore/src/heal/data_usage_cache.rs b/crates/ecstore/src/heal/data_usage_cache.rs deleted file mode 100644 index 70b509cf6..000000000 --- a/crates/ecstore/src/heal/data_usage_cache.rs +++ /dev/null @@ -1,928 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::config::com::save_config; -use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; -use crate::error::{Error, Result}; -use crate::new_object_layer_fn; -use crate::set_disk::SetDisks; -use crate::store_api::{BucketInfo, ObjectIO, ObjectOptions}; -use bytesize::ByteSize; -use http::HeaderMap; -use path_clean::PathClean; -use rand::Rng; -use rmp_serde::Serializer; -use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration}; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::path::Path; -use std::time::{Duration, SystemTime}; -use tokio::sync::mpsc::Sender; -use tokio::time::sleep; - -use super::data_scanner::{DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS, SizeSummary}; -use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo}; - -// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals -pub const DATA_USAGE_BUCKET_LEN: usize = 11; -pub const DATA_USAGE_VERSION_LEN: usize = 7; - -pub type DataUsageHashMap = HashSet; - -struct ObjectHistogramInterval { - name: &'static str, - start: u64, - end: u64, -} - -const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [ - ObjectHistogramInterval { - name: "LESS_THAN_1024_B", - start: 0, - end: ByteSize::kib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1024_B_AND_64_KB", - start: ByteSize::kib(1).as_u64(), - end: ByteSize::kib(64).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_64_KB_AND_256_KB", - start: ByteSize::kib(64).as_u64(), - end: ByteSize::kib(256).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_256_KB_AND_512_KB", - start: ByteSize::kib(256).as_u64(), - end: ByteSize::kib(512).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_512_KB_AND_1_MB", - start: ByteSize::kib(512).as_u64(), - end: ByteSize::mib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1024B_AND_1_MB", - start: ByteSize::kib(1).as_u64(), - end: ByteSize::mib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1_MB_AND_10_MB", - start: ByteSize::mib(1).as_u64(), - end: ByteSize::mib(10).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_10_MB_AND_64_MB", - start: ByteSize::mib(10).as_u64(), - end: ByteSize::mib(64).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_64_MB_AND_128_MB", - start: ByteSize::mib(64).as_u64(), - end: ByteSize::mib(128).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_128_MB_AND_512_MB", - start: ByteSize::mib(128).as_u64(), - end: ByteSize::mib(512).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "GREATER_THAN_512_MB", - start: ByteSize::mib(512).as_u64(), - end: u64::MAX, - }, -]; - -const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [ - ObjectHistogramInterval { - name: "UNVERSIONED", - start: 0, - end: 0, - }, - ObjectHistogramInterval { - name: "SINGLE_VERSION", - start: 1, - end: 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_2_AND_10", - start: 2, - end: 9, - }, - ObjectHistogramInterval { - name: "BETWEEN_10_AND_100", - start: 10, - end: 99, - }, - ObjectHistogramInterval { - name: "BETWEEN_100_AND_1000", - start: 100, - end: 999, - }, - ObjectHistogramInterval { - name: "BETWEEN_1000_AND_10000", - start: 1000, - end: 9999, - }, - ObjectHistogramInterval { - name: "GREATER_THAN_10000", - start: 10000, - end: u64::MAX, - }, -]; - -#[derive(Clone, Copy, Default)] -pub struct TierStats { - pub total_size: u64, - pub num_versions: i32, - pub num_objects: i32, -} - -impl TierStats { - pub fn add(&self, u: &TierStats) -> TierStats { - TierStats { - total_size: self.total_size + u.total_size, - num_versions: self.num_versions + u.num_versions, - num_objects: self.num_objects + u.num_objects, - } - } -} - -struct AllTierStats { - tiers: HashMap, -} - -impl AllTierStats { - pub fn new() -> Self { - Self { tiers: HashMap::new() } - } - - fn add_sizes(&mut self, tiers: HashMap) { - for (tier, st) in tiers { - self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); - } - } - - fn merge(&mut self, other: AllTierStats) { - for (tier, st) in other.tiers { - self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); - } - } - - fn populate_stats(&self, stats: &mut HashMap) { - for (tier, st) in &self.tiers { - stats.insert( - tier.clone(), - TierStats { - total_size: st.total_size, - num_versions: st.num_versions, - num_objects: st.num_objects, - }, - ); - } - } -} - -// sizeHistogram is a size histogram. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SizeHistogram(Vec); - -impl Default for SizeHistogram { - fn default() -> Self { - Self(vec![0; DATA_USAGE_BUCKET_LEN]) - } -} - -impl SizeHistogram { - fn add(&mut self, size: u64) { - for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() { - if size >= interval.start && size <= interval.end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let mut res = HashMap::new(); - let mut spl_count = 0; - for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) { - if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 { - res.insert(oh.name.to_string(), spl_count); - } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() { - spl_count += count; - res.insert(oh.name.to_string(), *count); - } else { - res.insert(oh.name.to_string(), *count); - } - } - res - } -} - -// versionsHistogram is a histogram of number of versions in an object. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct VersionsHistogram(Vec); - -impl Default for VersionsHistogram { - fn default() -> Self { - Self(vec![0; DATA_USAGE_VERSION_LEN]) - } -} - -impl VersionsHistogram { - fn add(&mut self, size: u64) { - for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() { - if size >= interval.start && size <= interval.end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let mut res = HashMap::new(); - for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) { - res.insert(ov.name.to_string(), *count); - } - res - } -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationStats { - pub pending_size: u64, - pub replicated_size: u64, - pub failed_size: u64, - pub failed_count: u64, - pub pending_count: u64, - pub missed_threshold_size: u64, - pub after_threshold_size: u64, - pub missed_threshold_count: u64, - pub after_threshold_count: u64, - pub replicated_count: u64, -} - -impl ReplicationStats { - pub fn empty(&self) -> bool { - self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0 - } -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationAllStats { - pub targets: HashMap, - pub replica_size: u64, - pub replica_count: u64, -} - -impl ReplicationAllStats { - pub fn empty(&self) -> bool { - if self.replica_size != 0 && self.replica_count != 0 { - return false; - } - for (_, v) in self.targets.iter() { - if !v.empty() { - return false; - } - } - - true - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageEntry { - pub children: DataUsageHashMap, - // These fields do not include any children. - pub size: usize, - pub objects: usize, - pub versions: usize, - pub delete_markers: usize, - pub obj_sizes: SizeHistogram, - pub obj_versions: VersionsHistogram, - pub replication_stats: Option, - // Todo: tier - // pub all_tier_stats: , - pub compacted: bool, -} - -impl DataUsageEntry { - pub fn add_child(&mut self, hash: &DataUsageHash) { - if self.children.contains(&hash.key()) { - return; - } - - self.children.insert(hash.key()); - } - - pub fn add_sizes(&mut self, summary: &SizeSummary) { - self.size += summary.total_size; - self.versions += summary.versions; - self.delete_markers += summary.delete_markers; - self.obj_sizes.add(summary.total_size as u64); - self.obj_versions.add(summary.versions as u64); - - let replication_stats = if self.replication_stats.is_none() { - self.replication_stats = Some(ReplicationAllStats::default()); - self.replication_stats.as_mut().unwrap() - } else { - self.replication_stats.as_mut().unwrap() - }; - replication_stats.replica_size += summary.replica_size as u64; - replication_stats.replica_count += summary.replica_count as u64; - - for (arn, st) in &summary.repl_target_stats { - let tgt_stat = replication_stats - .targets - .entry(arn.to_string()) - .or_insert(ReplicationStats::default()); - tgt_stat.pending_size += st.pending_size as u64; - tgt_stat.failed_size += st.failed_size as u64; - tgt_stat.replicated_size += st.replicated_size as u64; - tgt_stat.replicated_count += st.replicated_count as u64; - tgt_stat.failed_count += st.failed_count as u64; - tgt_stat.pending_count += st.pending_count as u64; - } - // Todo:: tiers - } - - pub fn merge(&mut self, other: &DataUsageEntry) { - self.objects += other.objects; - self.versions += other.versions; - self.delete_markers += other.delete_markers; - self.size += other.size; - if let Some(o_rep) = &other.replication_stats { - if self.replication_stats.is_none() { - self.replication_stats = Some(ReplicationAllStats::default()); - } - let s_rep = self.replication_stats.as_mut().unwrap(); - s_rep.targets.clear(); - s_rep.replica_size += o_rep.replica_size; - s_rep.replica_count += o_rep.replica_count; - for (arn, stat) in o_rep.targets.iter() { - let st = s_rep.targets.entry(arn.clone()).or_default(); - *st = ReplicationStats { - pending_size: stat.pending_size + st.pending_size, - failed_size: stat.failed_size + st.failed_size, - replicated_size: stat.replicated_size + st.replicated_size, - pending_count: stat.pending_count + st.pending_count, - failed_count: stat.failed_count + st.failed_count, - replicated_count: stat.replicated_count + st.replicated_count, - ..Default::default() - }; - } - } - - for (i, v) in other.obj_sizes.0.iter().enumerate() { - self.obj_sizes.0[i] += v; - } - - for (i, v) in other.obj_versions.0.iter().enumerate() { - self.obj_versions.0[i] += v; - } - - // todo: tiers - } -} - -#[derive(Clone)] -pub struct DataUsageEntryInfo { - pub name: String, - pub parent: String, - pub entry: DataUsageEntry, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageCacheInfo { - pub name: String, - pub next_cycle: u32, - pub last_update: Option, - pub skip_healing: bool, - #[serde(skip)] - pub lifecycle: Option, - #[serde(skip)] - pub updates: Option>, - #[serde(skip)] - pub replication: Option, -} - -// impl Default for DataUsageCacheInfo { -// fn default() -> Self { -// Self { -// name: Default::default(), -// next_cycle: Default::default(), -// last_update: SystemTime::now(), -// skip_healing: Default::default(), -// updates: Default::default(), -// replication: Default::default(), -// } -// } -// } - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageCache { - pub info: DataUsageCacheInfo, - pub cache: HashMap, -} - -impl DataUsageCache { - pub async fn load(store: &SetDisks, name: &str) -> Result { - let mut d = DataUsageCache::default(); - let mut retries = 0; - while retries < 5 { - let path = Path::new(BUCKET_META_PREFIX).join(name); - // warn!("Loading data usage cache from backend: {}", path.display()); - match store - .get_object_reader( - RUSTFS_META_BUCKET, - path.to_str().unwrap(), - None, - HeaderMap::new(), - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(mut reader) => { - if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { - d = info - } - break; - } - Err(err) => { - // warn!("Failed to load data usage cache from backend: {}", &err); - match err { - Error::FileNotFound | Error::VolumeNotFound => { - match store - .get_object_reader( - RUSTFS_META_BUCKET, - name, - None, - HeaderMap::new(), - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(mut reader) => { - if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { - d = info - } - break; - } - Err(_) => match err { - Error::FileNotFound | Error::VolumeNotFound => { - break; - } - _ => {} - }, - } - } - _ => { - break; - } - } - } - } - retries += 1; - let dur = { - let mut rng = rand::rng(); - rng.random_range(0..1_000) - }; - sleep(Duration::from_millis(dur)).await; - } - Ok(d) - } - - pub async fn save(&self, name: &str) -> Result<()> { - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let buf = self.marshal_msg()?; - let buf_clone = buf.clone(); - - let store_clone = store.clone(); - - let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string(); - - let name_clone = name.clone(); - tokio::spawn(async move { - let _ = save_config(store_clone, &format!("{}{}", &name_clone, ".bkp"), buf_clone).await; - }); - save_config(store, &name, buf).await?; - Ok(()) - } - - pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) { - let hash = hash_path(path); - self.cache.insert(hash.key(), e); - if !parent.is_empty() { - let phash = hash_path(parent); - let p = { - let p = self.cache.entry(phash.key()).or_default(); - p.add_child(&hash); - p.clone() - }; - self.cache.insert(phash.key(), p); - } - } - - pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option, e: &DataUsageEntry) { - self.cache.insert(hash.key(), e.clone()); - if let Some(parent) = parent { - self.cache.entry(parent.key()).or_default().add_child(hash); - } - } - - pub fn find(&self, path: &str) -> Option { - self.cache.get(&hash_path(path).key()).cloned() - } - - pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { - self.cache.entry(h.string()).or_default().children.clone() - } - - pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { - let mut root = root.clone(); - for id in root.children.clone().iter() { - if let Some(e) = self.cache.get(id) { - let mut e = e.clone(); - if !e.children.is_empty() { - e = self.flatten(&e); - } - root.merge(&e); - } - } - root.children.clear(); - root - } - - pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option) { - if let Some(e) = src.cache.get(&hash.string()) { - self.cache.insert(hash.key(), e.clone()); - for ch in e.children.iter() { - if *ch == hash.key() { - return; - } - self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone())); - } - if let Some(parent) = parent { - let p = self.cache.entry(parent.key()).or_default(); - p.add_child(hash); - } - } - } - - pub fn delete_recursive(&mut self, hash: &DataUsageHash) { - let mut need_remove = Vec::new(); - if let Some(v) = self.cache.get(&hash.string()) { - for child in v.children.iter() { - need_remove.push(child.clone()); - } - } - self.cache.remove(&hash.string()); - need_remove.iter().for_each(|child| { - self.delete_recursive(&DataUsageHash(child.to_string())); - }); - } - - pub fn size_recursive(&self, path: &str) -> Option { - match self.find(path) { - Some(root) => { - if root.children.is_empty() { - return Some(root); - } - let mut flat = self.flatten(&root); - if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() { - flat.replication_stats = None; - } - Some(flat) - } - None => None, - } - } - - pub fn search_parent(&self, hash: &DataUsageHash) -> Option { - let want = hash.key(); - if let Some(last_index) = want.rfind('/') { - if let Some(v) = self.find(&want[0..last_index]) { - if v.children.contains(&want) { - let found = hash_path(&want[0..last_index]); - return Some(found); - } - } - } - - for (k, v) in self.cache.iter() { - if v.children.contains(&want) { - let found = DataUsageHash(k.clone()); - return Some(found); - } - } - None - } - - pub fn is_compacted(&self, hash: &DataUsageHash) -> bool { - match self.cache.get(&hash.key()) { - Some(due) => due.compacted, - None => false, - } - } - - pub fn force_compact(&mut self, limit: usize) { - if self.cache.len() < limit { - return; - } - let top = hash_path(&self.info.name).key(); - let top_e = match self.find(&top) { - Some(e) => e, - None => return, - }; - if top_e.children.len() > >::try_into(DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS).unwrap() { - self.reduce_children_of(&hash_path(&self.info.name), limit, true); - } - if self.cache.len() <= limit { - return; - } - - let mut found = HashSet::new(); - found.insert(top); - mark(self, &top_e, &mut found); - self.cache.retain(|k, _| { - if !found.contains(k) { - return false; - } - true - }); - } - - pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) { - let e = match self.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - - if e.compacted { - return; - } - - if e.children.len() > limit && compact_self { - let mut flat = self.size_recursive(&path.key()).unwrap_or_default(); - flat.compacted = true; - self.delete_recursive(path); - self.replace_hashed(path, &None, &flat); - return; - } - let total = self.total_children_rec(&path.key()); - if total < limit { - return; - } - - let mut leaves = Vec::new(); - let mut remove = total - limit; - add(self, path, &mut leaves); - leaves.sort_by(|a, b| a.objects.cmp(&b.objects)); - - while remove > 0 && !leaves.is_empty() { - let e = leaves.first().unwrap(); - let candidate = e.path.clone(); - if candidate == *path && !compact_self { - break; - } - let removing = self.total_children_rec(&candidate.key()); - let mut flat = match self.size_recursive(&candidate.key()) { - Some(flat) => flat, - None => { - leaves.remove(0); - continue; - } - }; - - flat.compacted = true; - self.delete_recursive(&candidate); - self.replace_hashed(&candidate, &None, &flat); - - remove -= removing; - leaves.remove(0); - } - } - - pub fn total_children_rec(&self, path: &str) -> usize { - let root = self.find(path); - - if root.is_none() { - return 0; - } - let root = root.unwrap(); - if root.children.is_empty() { - return 0; - } - - let mut n = root.children.len(); - for ch in root.children.iter() { - n += self.total_children_rec(ch); - } - n - } - - pub fn merge(&mut self, o: &DataUsageCache) { - let mut existing_root = self.root(); - let other_root = o.root(); - if existing_root.is_none() && other_root.is_none() { - return; - } - if other_root.is_none() { - return; - } - if existing_root.is_none() { - *self = o.clone(); - return; - } - if o.info.last_update.gt(&self.info.last_update) { - self.info.last_update = o.info.last_update; - } - - existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap()); - self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap()); - let e_hash = self.root_hash(); - for key in other_root.as_ref().unwrap().children.iter() { - let entry = &o.cache[key]; - let flat = o.flatten(entry); - let mut existing = self.cache[key].clone(); - existing.merge(&flat); - self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing); - } - } - - pub fn root_hash(&self) -> DataUsageHash { - hash_path(&self.info.name) - } - - pub fn root(&self) -> Option { - self.find(&self.info.name) - } - - pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo { - let e = match self.find(path) { - Some(e) => e, - None => return DataUsageInfo::default(), - }; - let flat = self.flatten(&e); - DataUsageInfo { - last_update: self.info.last_update, - objects_total_count: flat.objects as u64, - versions_total_count: flat.versions as u64, - delete_markers_total_count: flat.delete_markers as u64, - objects_total_size: flat.size as u64, - buckets_count: e.children.len() as u64, - buckets_usage: self.buckets_usage_info(buckets), - ..Default::default() - } - } - - pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap { - let mut dst = HashMap::new(); - for bucket in buckets.iter() { - let e = match self.find(&bucket.name) { - Some(e) => e, - None => continue, - }; - let flat = self.flatten(&e); - let mut bui = BucketUsageInfo { - size: flat.size as u64, - versions_count: flat.versions as u64, - objects_count: flat.objects as u64, - delete_markers_count: flat.delete_markers as u64, - object_size_histogram: flat.obj_sizes.to_map(), - object_versions_histogram: flat.obj_versions.to_map(), - ..Default::default() - }; - if let Some(rs) = &flat.replication_stats { - bui.replica_size = rs.replica_size; - bui.replica_count = rs.replica_count; - - for (arn, stat) in rs.targets.iter() { - bui.replication_info.insert( - arn.clone(), - BucketTargetUsageInfo { - replication_pending_size: stat.pending_size, - replicated_size: stat.replicated_size, - replication_failed_size: stat.failed_size, - replication_pending_count: stat.pending_count, - replication_failed_count: stat.failed_count, - replicated_count: stat.replicated_count, - ..Default::default() - }, - ); - } - } - dst.insert(bucket.name.clone(), bui); - } - dst - } - - pub fn marshal_msg(&self) -> Result> { - let mut buf = Vec::new(); - - self.serialize(&mut Serializer::new(&mut buf))?; - - Ok(buf) - } - - pub fn unmarshal(buf: &[u8]) -> Result { - let t: Self = rmp_serde::from_slice(buf)?; - Ok(t) - } -} - -#[derive(Default, Clone)] -struct Inner { - objects: usize, - path: DataUsageHash, -} - -fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec) { - let e = match data_usage_cache.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - if !e.children.is_empty() { - return; - } - - let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default(); - leaves.push(Inner { - objects: sz.objects, - path: path.clone(), - }); - for ch in e.children.iter() { - add(data_usage_cache, &DataUsageHash(ch.clone()), leaves); - } -} - -fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet) { - for k in entry.children.iter() { - found.insert(k.to_string()); - if let Some(ch) = duc.cache.get(k) { - mark(duc, ch, found); - } - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct DataUsageHash(pub String); - -impl DataUsageHash { - pub fn string(&self) -> String { - self.0.clone() - } - - pub fn key(&self) -> String { - self.0.clone() - } - - pub fn mod_(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - hash as u32 % cycles == cycle % cycles - } - - pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - (hash >> 32) as u32 % cycles == cycle % cycles - } - - fn calculate_hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.0.hash(&mut hasher); - hasher.finish() - } -} - -pub fn hash_path(data: &str) -> DataUsageHash { - DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string()) -} diff --git a/crates/ecstore/src/heal/error.rs b/crates/ecstore/src/heal/error.rs deleted file mode 100644 index dd2db424e..000000000 --- a/crates/ecstore/src/heal/error.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage"; -pub const ERR_SKIP_FILE: &str = "skip this file"; -pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled"; -pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long"; -pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again"; diff --git a/crates/ecstore/src/heal/heal_commands.rs b/crates/ecstore/src/heal/heal_commands.rs deleted file mode 100644 index 3df0f3ba1..000000000 --- a/crates/ecstore/src/heal/heal_commands.rs +++ /dev/null @@ -1,544 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - collections::{HashMap, HashSet}, - path::Path, - time::SystemTime, -}; - -use crate::{ - config::storageclass::{RRS, STANDARD}, - disk::{BUCKET_META_PREFIX, DeleteOptions, DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError, fs::read_file}, - global::GLOBAL_BackgroundHealState, - heal::heal_ops::HEALING_TRACKER_FILENAME, - new_object_layer_fn, - store_api::{BucketInfo, StorageAPI}, -}; -use crate::{disk, error::Result}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; -use tokio::sync::RwLock; - -use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID}; - -pub type HealScanMode = usize; - -pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0; -pub const HEAL_NORMAL_SCAN: HealScanMode = 1; -pub const HEAL_DEEP_SCAN: HealScanMode = 2; - -pub const HEAL_ITEM_METADATA: &str = "metadata"; -pub const HEAL_ITEM_BUCKET: &str = "bucket"; -pub const HEAL_ITEM_BUCKET_METADATA: &str = "bucket-metadata"; -pub const HEAL_ITEM_OBJECT: &str = "object"; - -pub const DRIVE_STATE_OK: &str = "ok"; -pub const DRIVE_STATE_OFFLINE: &str = "offline"; -pub const DRIVE_STATE_CORRUPT: &str = "corrupt"; -pub const DRIVE_STATE_MISSING: &str = "missing"; -pub const DRIVE_STATE_PERMISSION: &str = "permission-denied"; -pub const DRIVE_STATE_FAULTY: &str = "faulty"; -pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount"; -pub const DRIVE_STATE_UNKNOWN: &str = "unknown"; -pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk - -lazy_static! { - pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap(); -} - -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] -pub struct HealOpts { - pub recursive: bool, - #[serde(rename = "dryRun")] - pub dry_run: bool, - pub remove: bool, - pub recreate: bool, - #[serde(rename = "scanMode")] - pub scan_mode: HealScanMode, - #[serde(rename = "updateParity")] - pub update_parity: bool, - #[serde(rename = "nolock")] - pub no_lock: bool, - pub pool: Option, - pub set: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct HealStartSuccess { - #[serde(rename = "clientToken")] - pub client_token: String, - #[serde(rename = "clientAddress")] - pub client_address: String, - #[serde(rename = "startTime")] - pub start_time: DateTime, -} - -impl Default for HealStartSuccess { - fn default() -> Self { - Self { - client_token: Default::default(), - client_address: Default::default(), - start_time: Utc::now(), - } - } -} - -pub type HealStopSuccess = HealStartSuccess; - -#[derive(Debug, Default, Deserialize, Serialize)] -pub struct HealingTracker { - #[serde(skip_serializing, skip_deserializing)] - pub disk: Option, - pub id: String, - pub pool_index: Option, - pub set_index: Option, - pub disk_index: Option, - pub path: String, - pub endpoint: String, - pub started: Option, - pub last_update: Option, - pub objects_total_count: u64, - pub objects_total_size: u64, - pub items_healed: u64, - pub items_failed: u64, - pub item_skipped: u64, - pub bytes_done: u64, - pub bytes_failed: u64, - pub bytes_skipped: u64, - pub bucket: String, - pub object: String, - pub resume_items_healed: u64, - pub resume_items_failed: u64, - pub resume_items_skipped: u64, - pub resume_bytes_done: u64, - pub resume_bytes_failed: u64, - pub resume_bytes_skipped: u64, - pub queue_buckets: Vec, - pub healed_buckets: Vec, - pub heal_id: String, - pub retry_attempts: u64, - pub finished: bool, - #[serde(skip_serializing, skip_deserializing)] - pub mu: RwLock, -} - -impl HealingTracker { - pub fn marshal_msg(&self) -> disk::error::Result> { - Ok(serde_json::to_vec(self)?) - } - - pub fn unmarshal_msg(data: &[u8]) -> disk::error::Result { - Ok(serde_json::from_slice::(data)?) - } - - pub async fn reset_healing(&mut self) { - let _ = self.mu.write().await; - self.items_healed = 0; - self.items_failed = 0; - self.bytes_done = 0; - self.bytes_failed = 0; - self.resume_items_healed = 0; - self.resume_items_failed = 0; - self.resume_bytes_done = 0; - self.resume_bytes_failed = 0; - self.item_skipped = 0; - self.bytes_skipped = 0; - - self.healed_buckets = Vec::new(); - self.bucket = String::new(); - self.object = String::new(); - } - - pub async fn get_last_update(&self) -> Option { - let _ = self.mu.read().await; - - self.last_update - } - - pub async fn get_bucket(&self) -> String { - let _ = self.mu.read().await; - - self.bucket.clone() - } - - pub async fn set_bucket(&mut self, bucket: &str) { - let _ = self.mu.write().await; - - self.bucket = bucket.to_string(); - } - - pub async fn get_object(&self) -> String { - let _ = self.mu.read().await; - - self.object.clone() - } - - pub async fn set_object(&mut self, object: &str) { - let _ = self.mu.write().await; - - self.object = object.to_string(); - } - - pub async fn update_progress(&mut self, success: bool, skipped: bool, by: u64) { - let _ = self.mu.write().await; - - if success { - self.items_healed += 1; - self.bytes_done += by; - } else if skipped { - self.item_skipped += 1; - self.bytes_skipped += by; - } else { - self.items_failed += 1; - self.bytes_failed += by; - } - } - - pub async fn update(&mut self) -> disk::error::Result<()> { - if let Some(disk) = &self.disk { - if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() { - return Err(DiskError::other(format!("healingTracker: drive {} is not marked as healing", self.id))); - } - let _ = self.mu.write().await; - if self.id.is_empty() || self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { - self.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); - let disk_location = disk.get_disk_location(); - self.pool_index = disk_location.pool_idx; - self.set_index = disk_location.set_idx; - self.disk_index = disk_location.disk_idx; - } - } - - self.save().await - } - - pub async fn save(&mut self) -> disk::error::Result<()> { - let _ = self.mu.write().await; - if self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { - let Some(store) = new_object_layer_fn() else { - return Err(DiskError::other("errServerNotInitialized")); - }; - - // TODO: check error type - (self.pool_index, self.set_index, self.disk_index) = - store.get_pool_and_set(&self.id).await.map_err(|_| DiskError::DiskNotFound)?; - } - - self.last_update = Some(SystemTime::now()); - - let htracker_bytes = self.marshal_msg()?; - - GLOBAL_BackgroundHealState.update_heal_status(self).await; - - if let Some(disk) = &self.disk { - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - disk.write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes.into()) - .await?; - } - Ok(()) - } - - pub async fn delete(&self) -> Result<()> { - if let Some(disk) = &self.disk { - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - disk.delete( - RUSTFS_META_BUCKET, - file_path.to_str().unwrap(), - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await?; - } - - Ok(()) - } - - pub async fn is_healed(&self, bucket: &str) -> bool { - let _ = self.mu.read().await; - for v in self.healed_buckets.iter() { - if v == bucket { - return true; - } - } - - false - } - - pub async fn resume(&mut self) { - let _ = self.mu.write().await; - - self.items_healed = self.resume_items_healed; - self.items_failed = self.resume_items_failed; - self.item_skipped = self.resume_items_skipped; - self.bytes_done = self.resume_bytes_done; - self.bytes_failed = self.resume_bytes_failed; - self.bytes_skipped = self.resume_bytes_skipped; - } - - pub async fn bucket_done(&mut self, bucket: &str) { - let _ = self.mu.write().await; - - self.resume_items_healed = self.items_healed; - self.resume_items_failed = self.items_failed; - self.resume_items_skipped = self.item_skipped; - self.resume_bytes_done = self.bytes_done; - self.resume_bytes_failed = self.bytes_failed; - self.resume_bytes_skipped = self.bytes_skipped; - self.healed_buckets.push(bucket.to_string()); - - self.queue_buckets.retain(|x| x != bucket); - } - - pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { - let _ = self.mu.write().await; - - buckets.iter().for_each(|bucket| { - if !self.healed_buckets.contains(&bucket.name) { - self.queue_buckets.push(bucket.name.clone()); - } - }); - } - - pub async fn to_healing_disk(&self) -> rustfs_madmin::HealingDisk { - let _ = self.mu.read().await; - - rustfs_madmin::HealingDisk { - id: self.id.clone(), - heal_id: self.heal_id.clone(), - pool_index: self.pool_index, - set_index: self.set_index, - disk_index: self.disk_index, - endpoint: self.endpoint.clone(), - path: self.path.clone(), - started: self.started, - last_update: self.last_update, - retry_attempts: self.retry_attempts, - objects_total_count: self.objects_total_count, - objects_total_size: self.objects_total_size, - items_healed: self.items_healed, - items_failed: self.items_failed, - item_skipped: self.item_skipped, - bytes_done: self.bytes_done, - bytes_failed: self.bytes_failed, - bytes_skipped: self.bytes_skipped, - objects_healed: self.items_healed, - objects_failed: self.items_failed, - bucket: self.bucket.clone(), - object: self.object.clone(), - queue_buckets: self.queue_buckets.clone(), - healed_buckets: self.healed_buckets.clone(), - finished: self.finished, - } - } -} - -impl Clone for HealingTracker { - fn clone(&self) -> Self { - Self { - disk: self.disk.clone(), - id: self.id.clone(), - pool_index: self.pool_index, - set_index: self.set_index, - disk_index: self.disk_index, - path: self.path.clone(), - endpoint: self.endpoint.clone(), - started: self.started, - last_update: self.last_update, - objects_total_count: self.objects_total_count, - objects_total_size: self.objects_total_size, - items_healed: self.items_healed, - items_failed: self.items_failed, - item_skipped: self.item_skipped, - bytes_done: self.bytes_done, - bytes_failed: self.bytes_failed, - bytes_skipped: self.bytes_skipped, - bucket: self.bucket.clone(), - object: self.object.clone(), - resume_items_healed: self.resume_items_healed, - resume_items_failed: self.resume_items_failed, - resume_items_skipped: self.resume_items_skipped, - resume_bytes_done: self.resume_bytes_done, - resume_bytes_failed: self.resume_bytes_failed, - resume_bytes_skipped: self.resume_bytes_skipped, - queue_buckets: self.queue_buckets.clone(), - healed_buckets: self.healed_buckets.clone(), - heal_id: self.heal_id.clone(), - retry_attempts: self.retry_attempts, - finished: self.finished, - mu: RwLock::new(false), - } - } -} - -pub async fn load_healing_tracker(disk: &Option) -> disk::error::Result { - if let Some(disk) = disk { - let disk_id = disk.get_disk_id().await?; - if let Some(disk_id) = disk_id { - let disk_id = disk_id.to_string(); - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - let data = disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await?; - let mut healing_tracker = HealingTracker::unmarshal_msg(&data)?; - if healing_tracker.id != disk_id && !healing_tracker.id.is_empty() { - return Err(DiskError::other(format!( - "loadHealingTracker: drive id mismatch expected {}, got {}", - healing_tracker.id, disk_id - ))); - } - healing_tracker.id = disk_id; - healing_tracker.disk = Some(disk.clone()); - Ok(healing_tracker) - } else { - Err(DiskError::other("loadHealingTracker: disk not have id")) - } - } else { - Err(DiskError::other("loadHealingTracker: nil drive given")) - } -} - -pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> disk::error::Result { - let disk_location = disk.get_disk_location(); - Ok(HealingTracker { - id: disk - .get_disk_id() - .await - .map_or("".to_string(), |id| id.map_or("".to_string(), |id| id.to_string())), - heal_id: heal_id.to_string(), - path: disk.to_string(), - endpoint: disk.endpoint().to_string(), - started: Some(OffsetDateTime::now_utc()), - pool_index: disk_location.pool_idx, - set_index: disk_location.set_idx, - disk_index: disk_location.disk_idx, - disk: Some(disk), - ..Default::default() - }) -} - -pub async fn healing(derive_path: &str) -> disk::error::Result> { - let healing_file = Path::new(derive_path) - .join(RUSTFS_META_BUCKET) - .join(BUCKET_META_PREFIX) - .join(HEALING_TRACKER_FILENAME); - - let b = read_file(healing_file).await?; - if b.is_empty() { - return Ok(None); - } - - let healing_tracker = HealingTracker::unmarshal_msg(&b)?; - - Ok(Some(healing_tracker)) -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct MRFStatus { - bytes_healed: u64, - items_healed: u64, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct SetStatus { - pub id: String, - pub pool_index: i32, - pub set_index: i32, - pub heal_status: String, - pub heal_priority: String, - pub total_objects: usize, - pub disks: Vec, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BgHealState { - offline_endpoints: Vec, - scanned_items_count: u64, - heal_disks: Vec, - sets: Vec, - mrf: HashMap, - scparity: HashMap, -} - -pub async fn get_local_background_heal_status() -> (BgHealState, bool) { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if !ok { - return (BgHealState::default(), false); - } - let bg_seq = bg_seq.unwrap(); - let mut status = BgHealState { - scanned_items_count: bg_seq.get_scanned_items_count().await as u64, - ..Default::default() - }; - let mut heal_disks_map = HashSet::new(); - for ep in get_local_disks_to_heal().await.iter() { - heal_disks_map.insert(ep.to_string()); - } - - let Some(store) = new_object_layer_fn() else { - let healing = GLOBAL_BackgroundHealState.get_local_healing_disks().await; - for disk in healing.values() { - status.heal_disks.push(disk.endpoint.clone()); - } - return (status, true); - }; - - let si = store.local_storage_info().await; - let mut indexed = HashMap::new(); - for disk in si.disks.iter() { - let set_idx = format!("{}-{}", disk.pool_index, disk.set_index); - // indexed.insert(set_idx, disk); - indexed.entry(set_idx).or_insert(Vec::new()).push(disk); - } - - for (id, disks) in indexed { - let mut ss = SetStatus { - id, - set_index: disks[0].set_index, - pool_index: disks[0].pool_index, - ..Default::default() - }; - for disk in disks { - ss.disks.push(disk.clone()); - if disk.healing { - ss.heal_status = "healing".to_string(); - ss.heal_priority = "high".to_string(); - status.heal_disks.push(disk.endpoint.clone()); - } - } - ss.disks.sort_by(|a, b| { - if a.pool_index != b.pool_index { - return a.pool_index.cmp(&b.pool_index); - } - if a.set_index != b.set_index { - return a.set_index.cmp(&b.set_index); - } - a.disk_index.cmp(&b.disk_index) - }); - status.sets.push(ss); - } - status.sets.sort_by(|a, b| a.id.cmp(&b.id)); - let backend_info = store.backend_info().await; - status - .scparity - .insert(STANDARD.to_string(), backend_info.standard_sc_parity.unwrap_or_default()); - status - .scparity - .insert(RRS.to_string(), backend_info.rr_sc_parity.unwrap_or_default()); - - (status, true) -} diff --git a/crates/ecstore/src/heal/heal_ops.rs b/crates/ecstore/src/heal/heal_ops.rs deleted file mode 100644 index 2ee4aee10..000000000 --- a/crates/ecstore/src/heal/heal_ops.rs +++ /dev/null @@ -1,842 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - background_heal_ops::HealTask, - data_scanner::HEAL_DELETE_DANGLING, - error::ERR_SKIP_FILE, - heal_commands::{HEAL_ITEM_BUCKET_METADATA, HealOpts, HealScanMode, HealStopSuccess, HealingTracker}, -}; -use crate::error::{Error, Result}; -use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}; -use crate::store_api::StorageAPI; -use crate::{ - config::com::CONFIG_PREFIX, - disk::RUSTFS_META_BUCKET, - global::GLOBAL_BackgroundHealRoutine, - heal::{error::ERR_HEAL_STOP_SIGNALLED, heal_commands::DRIVE_STATE_OK}, -}; -use crate::{ - disk::endpoint::Endpoint, - endpoints::Endpoints, - global::GLOBAL_IsDistErasure, - heal::heal_commands::{HEAL_UNKNOWN_SCAN, HealStartSuccess}, - new_object_layer_fn, -}; -use chrono::Utc; -use futures::join; -use lazy_static::lazy_static; -use rustfs_filemeta::MetaCacheEntry; -use rustfs_madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem}; -use rustfs_utils::path::has_prefix; -use rustfs_utils::path::path_join; -use serde::{Deserialize, Serialize}; -use std::{ - collections::HashMap, - future::Future, - path::PathBuf, - pin::Pin, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; -use tokio::{ - select, spawn, - sync::{ - RwLock, broadcast, - mpsc::{self, Receiver as M_Receiver, Sender as M_Sender}, - watch::{self, Receiver as W_Receiver, Sender as W_Sender}, - }, - time::{interval, sleep}, -}; -use tracing::{error, info}; -use uuid::Uuid; - -type HealStatusSummary = String; -type ItemsMap = HashMap; -pub type HealEntryFn = - Arc Pin> + Send>> + Send + Sync + 'static>; - -pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000"; -pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin"; -const KEEP_HEAL_SEQ_STATE_DURATION: Duration = Duration::from_secs(10 * 60); -const HEAL_NOT_STARTED_STATUS: &str = "not started"; -const HEAL_RUNNING_STATUS: &str = "running"; -const HEAL_STOPPED_STATUS: &str = "stopped"; -const HEAL_FINISHED_STATUS: &str = "finished"; - -pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs"; -pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs"; -pub const LOGIN_PATH_PREFIX: &str = "/login"; - -const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000; -const HEAL_UNCONSUMED_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); -pub const NOP_HEAL: &str = ""; - -lazy_static! {} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct HealSequenceStatus { - pub summary: HealStatusSummary, - pub failure_detail: String, - pub start_time: u64, - pub heal_setting: HealOpts, - pub items: Vec, -} - -#[derive(Debug, Default)] -pub struct HealSource { - pub bucket: String, - pub object: String, - pub version_id: String, - pub no_wait: bool, - pub opts: Option, -} - -#[derive(Debug)] -pub struct HealSequence { - pub bucket: String, - pub object: String, - pub report_progress: bool, - pub start_time: SystemTime, - pub end_time: Arc>, - pub client_token: String, - pub client_address: String, - pub force_started: bool, - pub setting: HealOpts, - pub current_status: Arc>, - pub last_sent_result_index: RwLock, - pub scanned_items_map: RwLock, - pub healed_items_map: RwLock, - pub heal_failed_items_map: RwLock, - pub last_heal_activity: RwLock, - - traverse_and_heal_done_tx: Arc>>>, - traverse_and_heal_done_rx: Arc>>>, - - tx: W_Sender, - rx: W_Receiver, -} - -pub fn new_bg_heal_sequence() -> HealSequence { - let hs = HealOpts { - remove: HEAL_DELETE_DANGLING, - ..Default::default() - }; - - HealSequence { - start_time: SystemTime::now(), - client_token: BG_HEALING_UUID.to_string(), - bucket: RUSTFS_RESERVED_BUCKET.to_string(), - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - report_progress: false, - scanned_items_map: HashMap::new().into(), - healed_items_map: HashMap::new().into(), - heal_failed_items_map: HashMap::new().into(), - ..Default::default() - } -} - -pub fn new_heal_sequence(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> HealSequence { - let client_token = Uuid::new_v4().to_string(); - let (tx, rx) = mpsc::channel(10); - HealSequence { - bucket: bucket.to_string(), - object: obj_prefix.to_string(), - report_progress: true, - start_time: SystemTime::now(), - client_token, - client_address: client_addr.to_string(), - force_started: force_start, - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - traverse_and_heal_done_tx: Arc::new(RwLock::new(tx)), - traverse_and_heal_done_rx: Arc::new(RwLock::new(rx)), - scanned_items_map: HashMap::new().into(), - healed_items_map: HashMap::new().into(), - heal_failed_items_map: HashMap::new().into(), - ..Default::default() - } -} - -impl Default for HealSequence { - fn default() -> Self { - let (h_tx, h_rx) = mpsc::channel(1); - let (tx, rx) = watch::channel(false); - Self { - bucket: Default::default(), - object: Default::default(), - report_progress: Default::default(), - start_time: SystemTime::now(), - end_time: Arc::new(RwLock::new(SystemTime::now())), - client_token: Default::default(), - client_address: Default::default(), - force_started: Default::default(), - setting: Default::default(), - current_status: Default::default(), - last_sent_result_index: Default::default(), - scanned_items_map: Default::default(), - healed_items_map: Default::default(), - heal_failed_items_map: Default::default(), - last_heal_activity: RwLock::new(SystemTime::now()), - traverse_and_heal_done_tx: Arc::new(RwLock::new(h_tx)), - traverse_and_heal_done_rx: Arc::new(RwLock::new(h_rx)), - tx, - rx, - } - } -} - -impl HealSequence { - pub fn new(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self { - let client_token = Uuid::new_v4().to_string(); - - Self { - bucket: bucket.to_string(), - object: obj_prefix.to_string(), - report_progress: true, - client_token, - client_address: client_addr.to_string(), - force_started: force_start, - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - ..Default::default() - } - } -} - -impl HealSequence { - pub async fn get_scanned_items_count(&self) -> usize { - self.scanned_items_map.read().await.values().sum() - } - - async fn _get_scanned_items_map(&self) -> ItemsMap { - self.scanned_items_map.read().await.clone() - } - - async fn _get_healed_items_map(&self) -> ItemsMap { - self.healed_items_map.read().await.clone() - } - - async fn _get_heal_failed_items_map(&self) -> ItemsMap { - self.heal_failed_items_map.read().await.clone() - } - - pub async fn count_failed(&self, heal_type: HealItemType) { - *self.heal_failed_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - pub async fn count_scanned(&self, heal_type: HealItemType) { - *self.scanned_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - pub async fn count_healed(&self, heal_type: HealItemType) { - *self.healed_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - async fn is_quitting(&self) -> bool { - if let Ok(true) = self.rx.has_changed() { - info!("quited"); - return true; - } - false - } - - async fn has_ended(&self) -> bool { - if self.client_token == *BG_HEALING_UUID { - return false; - } - - *(self.end_time.read().await) != self.start_time - } - - async fn stop(&self) { - let _ = self.tx.send(true); - } - - async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> { - let mut r = r.clone(); - let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT); - #[allow(unused_assignments)] - let mut items_len = 0; - loop { - { - let current_status_r = self.current_status.read().await; - items_len = current_status_r.items.len(); - } - - if items_len == MAX_UNCONSUMED_HEAL_RESULT_ITEMS { - select! { - _ = sleep(Duration::from_secs(1)) => { - - } - _ = self.is_done() => { - return Err(Error::other("stopped")); - } - _ = interval_timer.tick() => { - return Err(Error::other("timeout")); - } - } - } else { - break; - } - } - - let mut current_status_w = self.current_status.write().await; - if items_len > 0 { - r.result_index = 1 + current_status_w.items[items_len - 1].result_index; - } else { - r.result_index = 1 + *self.last_sent_result_index.read().await; - } - - current_status_w.items.push(r); - - Ok(()) - } - - pub async fn queue_heal_task(&self, source: HealSource, heal_type: HealItemType) -> Result<()> { - let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting); - info!("queue_heal_task, {:?}", task); - if let Some(opts) = source.opts { - task.opts = opts; - } else { - task.opts.scan_mode = HEAL_UNKNOWN_SCAN; - } - - self.count_scanned(heal_type.clone()).await; - - if source.no_wait { - let task_str = format!("{task:?}"); - if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() { - info!("Task in the queue: {:?}", task_str); - } - return Ok(()); - } - - let (resp_tx, mut resp_rx) = mpsc::channel(1); - task.resp_tx = Some(resp_tx); - - let task_str = format!("{task:?}"); - if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() { - info!("Task in the queue: {:?}", task_str); - } else { - error!("push task to queue failed"); - } - let count_ok_drives = |drivers: &[HealDriveInfo]| { - let mut count = 0; - for drive in drivers.iter() { - if drive.state == DRIVE_STATE_OK { - count += 1; - } - } - count - }; - - match resp_rx.recv().await { - Some(mut res) => { - if res.err.is_none() { - self.count_healed(heal_type.clone()).await; - } else { - self.count_failed(heal_type.clone()).await; - } - if !self.report_progress { - return if let Some(err) = res.err { - if err.to_string() == ERR_SKIP_FILE { - return Ok(()); - } - Err(err) - } else { - Ok(()) - }; - } - res.result.heal_item_type = heal_type.clone(); - if let Some(err) = res.err.as_ref() { - res.result.detail = err.to_string(); - } - if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks - { - let got = count_ok_drives(&res.result.after.drives); - if got < res.result.parity_blocks { - res.result.detail = format!( - "quorum loss - expected {} minimum, got drive states in OK {}", - res.result.parity_blocks, got - ); - } - } - - info!("queue_heal_task, HealResult: {:?}", res); - self.push_heal_result_item(&res.result).await - } - None => Ok(()), - } - } - - async fn heal_disk_meta(h: Arc) -> Result<()> { - HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await - } - - async fn heal_items(h: Arc, buckets_only: bool) -> Result<()> { - if h.client_token == *BG_HEALING_UUID { - return Ok(()); - } - - let bucket = h.bucket.clone(); - let task1 = Self::heal_disk_meta(h.clone()); - let task2 = Self::heal_bucket(h.clone(), &bucket, buckets_only); - let results = join!(task1, task2); - results.0?; - results.1?; - - Ok(()) - } - - async fn traverse_and_heal(h: Arc) { - let buckets_only = false; - let result = Self::heal_items(h.clone(), buckets_only).await.err(); - let _ = h.traverse_and_heal_done_tx.read().await.send(result).await; - } - - async fn heal_rustfs_sys_meta(h: Arc, meta_prefix: &str) -> Result<()> { - info!("heal_rustfs_sys_meta, h: {:?}", h); - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let setting = h.setting; - store - .heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true) - .await - } - - async fn is_done(&self) -> bool { - if let Ok(true) = self.rx.has_changed() { - return true; - } - false - } - - pub async fn heal_bucket(hs: Arc, bucket: &str, bucket_only: bool) -> Result<()> { - info!("heal_bucket, hs: {:?}", hs); - let (object, setting) = { - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_string(), - ) - .await?; - - if bucket_only { - return Ok(()); - } - - if !hs.setting.recursive { - if !hs.object.is_empty() { - HealSequence::heal_object(hs.clone(), bucket, &hs.object, "", hs.setting.scan_mode).await?; - } - return Ok(()); - } - (hs.object.clone(), hs.setting) - }; - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - store.heal_objects(bucket, &object, &setting, hs.clone(), false).await - } - - pub async fn heal_object( - hs: Arc, - bucket: &str, - object: &str, - version_id: &str, - _scan_mode: HealScanMode, - ) -> Result<()> { - info!("heal_object"); - if hs.is_quitting().await { - info!("heal_object hs is quitting"); - return Err(Error::other(ERR_HEAL_STOP_SIGNALLED)); - } - - info!("will queue task"); - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - opts: Some(hs.setting), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await?; - - Ok(()) - } - - pub async fn heal_meta_object( - hs: Arc, - bucket: &str, - object: &str, - version_id: &str, - _scan_mode: HealScanMode, - ) -> Result<()> { - if hs.is_quitting().await { - return Err(Error::other(ERR_HEAL_STOP_SIGNALLED)); - } - - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET_METADATA.to_string(), - ) - .await?; - - Ok(()) - } -} - -pub async fn heal_sequence_start(h: Arc) { - { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_RUNNING_STATUS.to_string(); - current_status_w.start_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - } - - let h_clone = h.clone(); - spawn(async move { - HealSequence::traverse_and_heal(h_clone).await; - }); - - let h_clone_1 = h.clone(); - let mut x = h.traverse_and_heal_done_rx.write().await; - select! { - _ = h.is_done() => { - *(h.end_time.write().await) = SystemTime::now(); - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_FINISHED_STATUS.to_string(); - - spawn(async move { - let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await; - rx_w.recv().await; - }); - } - result = x.recv() => { - if let Some(err) = result { - match err { - Some(err) => { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_STOPPED_STATUS.to_string(); - current_status_w.failure_detail = err.to_string(); - }, - None => { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_FINISHED_STATUS.to_string(); - } - } - } - - } - } -} - -#[derive(Debug, Default)] -pub struct AllHealState { - mu: RwLock, - - heal_seq_map: RwLock>>, - heal_local_disks: RwLock>, - heal_status: RwLock>, -} - -impl AllHealState { - pub fn new(cleanup: bool) -> Arc { - let state = Arc::new(AllHealState::default()); - let (_, mut rx) = broadcast::channel(1); - if cleanup { - let state_clone = state.clone(); - spawn(async move { - loop { - select! { - result = rx.recv() =>{ - if let Ok(true) = result { - return; - } - } - _ = sleep(Duration::from_secs(5 * 60)) => { - state_clone.periodic_heal_seqs_clean().await; - } - } - } - }); - } - - state - } - - pub async fn pop_heal_local_disks(&self, heal_local_disks: &[Endpoint]) { - let _ = self.mu.write().await; - - self.heal_local_disks.write().await.retain(|k, _| { - if heal_local_disks.contains(k) { - return false; - } - true - }); - - let heal_local_disks = heal_local_disks.iter().map(|s| s.to_string()).collect::>(); - self.heal_status.write().await.retain(|_, v| { - if heal_local_disks.contains(&v.endpoint) { - return false; - } - - true - }); - } - - pub async fn pop_heal_status_json(&self, heal_path: &str, client_token: &str) -> Result> { - match self.get_heal_sequence(heal_path).await { - Some(h) => { - if client_token != h.client_token { - info!("err heal invalid client token"); - return Err(Error::other("err heal invalid client token")); - } - let num_items = h.current_status.read().await.items.len(); - let mut last_result_index = *h.last_sent_result_index.read().await; - if num_items > 0 { - if let Some(item) = h.current_status.read().await.items.last() { - last_result_index = item.result_index; - } - } - *h.last_sent_result_index.write().await = last_result_index; - let data = h.current_status.read().await.clone(); - match serde_json::to_vec(&data) { - Ok(b) => { - h.current_status.write().await.items.clear(); - Ok(b) - } - Err(e) => { - h.current_status.write().await.items.clear(); - info!("json encode err, e: {}", e); - Err(Error::other(e.to_string())) - } - } - } - None => serde_json::to_vec(&HealSequenceStatus { - summary: HEAL_FINISHED_STATUS.to_string(), - ..Default::default() - }) - .map_err(|e| { - info!("json encode err, e: {}", e); - Error::other(e.to_string()) - }), - } - } - - pub async fn update_heal_status(&self, tracker: &HealingTracker) { - let _ = self.mu.write().await; - let _ = tracker.mu.read().await; - - self.heal_status.write().await.insert(tracker.id.clone(), tracker.clone()); - } - - pub async fn get_local_healing_disks(&self) -> HashMap { - let _ = self.mu.read().await; - - let mut dst = HashMap::new(); - for v in self.heal_status.read().await.values() { - dst.insert(v.endpoint.clone(), v.to_healing_disk().await); - } - - dst - } - - pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints { - let _ = self.mu.read().await; - - let mut endpoints = Vec::new(); - self.heal_local_disks.read().await.iter().for_each(|(k, v)| { - if !v { - endpoints.push(k.clone()); - } - }); - - Endpoints::from(endpoints) - } - - pub async fn set_disk_healing_status(&self, ep: Endpoint, healing: bool) { - let _ = self.mu.write().await; - - self.heal_local_disks.write().await.insert(ep, healing); - } - - pub async fn push_heal_local_disks(&self, heal_local_disks: &[Endpoint]) { - let _ = self.mu.write().await; - - for heal_local_disk in heal_local_disks.iter() { - self.heal_local_disks.write().await.insert(heal_local_disk.clone(), false); - } - } - - pub async fn periodic_heal_seqs_clean(&self) { - let _ = self.mu.write().await; - let now = SystemTime::now(); - - let mut keys_to_remove = Vec::new(); - for (k, v) in self.heal_seq_map.read().await.iter() { - if v.has_ended().await && now.duration_since(*(v.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION { - keys_to_remove.push(k.clone()) - } - } - for key in keys_to_remove.iter() { - self.heal_seq_map.write().await.remove(key); - } - } - - pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option>, bool) { - let _ = self.mu.read().await; - - for v in self.heal_seq_map.read().await.values() { - if v.client_token == token { - return (Some(v.clone()), true); - } - } - - (None, false) - } - - pub async fn get_heal_sequence(&self, path: &str) -> Option> { - let _ = self.mu.read().await; - - self.heal_seq_map.read().await.get(path).cloned() - } - - pub async fn stop_heal_sequence(&self, path: &str) -> Result> { - let mut hsp = HealStopSuccess::default(); - if let Some(he) = self.get_heal_sequence(path).await { - let client_token = he.client_token.clone(); - if *GLOBAL_IsDistErasure.read().await { - // TODO: proxy - } - - hsp.client_token = client_token; - hsp.client_address = he.client_address.clone(); - hsp.start_time = Utc::now(); - - he.stop().await; - - loop { - if he.has_ended().await { - break; - } - - sleep(Duration::from_secs(1)).await; - } - - let _ = self.mu.write().await; - self.heal_seq_map.write().await.remove(path); - } else { - hsp.client_token = "unknown".to_string(); - } - - let b = serde_json::to_string(&hsp)?; - Ok(b.as_bytes().to_vec()) - } - - // LaunchNewHealSequence - launches a background routine that performs - // healing according to the healSequence argument. For each heal - // sequence, state is stored in the `globalAllHealState`, which is a - // map of the heal path to `healSequence` which holds state about the - // heal sequence. - // - // Heal results are persisted in server memory for - // `keepHealSeqStateDuration`. This function also launches a - // background routine to clean up heal results after the - // aforementioned duration. - pub async fn launch_new_heal_sequence(&self, heal_sequence: Arc) -> Result> { - let path = path_join(&[ - PathBuf::from(heal_sequence.bucket.clone()), - PathBuf::from(heal_sequence.object.clone()), - ]); - let path_s = path.to_str().unwrap(); - if heal_sequence.force_started { - self.stop_heal_sequence(path_s).await?; - } else if let Some(hs) = self.get_heal_sequence(path_s).await { - if !hs.has_ended().await { - return Err(Error::other(format!( - "Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", - heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token - ))); - } - } - - let _ = self.mu.write().await; - - for (k, v) in self.heal_seq_map.read().await.iter() { - if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await { - return Err(Error::other(format!( - "The provided heal sequence path overlaps with an existing heal path: {k}" - ))); - } - } - - self.heal_seq_map - .write() - .await - .insert(path_s.to_string(), heal_sequence.clone()); - - let client_token = heal_sequence.client_token.clone(); - if *GLOBAL_IsDistErasure.read().await { - // TODO: proxy - } - - if heal_sequence.client_token == BG_HEALING_UUID { - // For background heal do nothing, do not spawn an unnecessary goroutine. - } else { - let heal_sequence_clone = heal_sequence.clone(); - spawn(async { - heal_sequence_start(heal_sequence_clone).await; - }); - } - - let b = serde_json::to_vec(&HealStartSuccess { - client_token, - client_address: heal_sequence.client_address.clone(), - // start_time: Utc::now(), - start_time: heal_sequence.start_time.into(), - })?; - Ok(b) - } -} diff --git a/crates/ecstore/src/heal/mrf.rs b/crates/ecstore/src/heal/mrf.rs deleted file mode 100644 index 3f42fa202..000000000 --- a/crates/ecstore/src/heal/mrf.rs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; -use crate::heal::background_heal_ops::{heal_bucket, heal_object}; -use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use regex::Regex; -use rustfs_utils::path::SLASH_SEPARATOR; -use std::ops::Sub; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; -use tokio::sync::RwLock; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::time::sleep; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; -use uuid::Uuid; - -pub const MRF_OPS_QUEUE_SIZE: u64 = 100000; -pub const HEAL_DIR: &str = ".heal"; -pub const HEAL_MRFMETA_FORMAT: u64 = 1; -pub const HEAL_MRFMETA_VERSION_V1: u64 = 1; - -lazy_static! { - pub static ref HEAL_MRF_DIR: String = - format!("{}{}{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, HEAL_DIR, SLASH_SEPARATOR, "mrf"); - static ref PATTERNS: Vec = vec![ - Regex::new(r"^buckets/.*/.metacache/.*").unwrap(), - Regex::new(r"^tmp/.*").unwrap(), - Regex::new(r"^multipart/.*").unwrap(), - Regex::new(r"^tmp-old/.*").unwrap(), - ]; -} - -#[derive(Default)] -pub struct PartialOperation { - pub bucket: String, - pub object: String, - pub version_id: Option, - pub versions: Vec, - pub set_index: usize, - pub pool_index: usize, - pub queued: DateTime, - pub bitrot_scan: bool, -} - -pub struct MRFState { - tx: Sender, - rx: RwLock>, - closed: AtomicBool, - closing: AtomicBool, -} - -impl Default for MRFState { - fn default() -> Self { - Self::new() - } -} - -impl MRFState { - pub fn new() -> MRFState { - let (tx, rx) = tokio::sync::mpsc::channel(MRF_OPS_QUEUE_SIZE as usize); - MRFState { - tx, - rx: RwLock::new(rx), - closed: Default::default(), - closing: Default::default(), - } - } - - pub async fn add_partial(&self, op: PartialOperation) { - if self.closed.load(Ordering::SeqCst) || self.closing.load(Ordering::SeqCst) { - return; - } - let _ = self.tx.send(op).await; - } - - /// Enhanced heal routine with cancellation support - /// - /// This method implements the same healing logic as the original heal_routine, - /// but adds proper cancellation support via CancellationToken. - /// The core logic remains identical to maintain compatibility. - pub async fn heal_routine_with_cancel(&self, cancel_token: CancellationToken) { - info!("MRF heal routine started with cancellation support"); - - loop { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("MRF heal routine received shutdown signal, exiting gracefully"); - break; - } - op_result = async { - let mut rx_guard = self.rx.write().await; - rx_guard.recv().await - } => { - if let Some(op) = op_result { - // Special path filtering (original logic) - if op.bucket == RUSTFS_META_BUCKET { - for pattern in &*PATTERNS { - if pattern.is_match(&op.object) { - continue; // Skip this operation, continue with next - } - } - } - - // Network reconnection delay (original logic) - let now = Utc::now(); - if now.sub(op.queued).num_seconds() < 1 { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("MRF heal routine cancelled during reconnection delay"); - break; - } - _ = sleep(Duration::from_secs(1)) => {} - } - } - - // Core healing logic (original logic preserved) - let scan_mode = if op.bitrot_scan { HEAL_DEEP_SCAN } else { HEAL_NORMAL_SCAN }; - - if op.object.is_empty() { - // Heal bucket (original logic) - if let Err(err) = heal_bucket(&op.bucket).await { - error!("heal bucket failed, bucket: {}, err: {:?}", op.bucket, err); - } - } else if op.versions.is_empty() { - // Heal single object (original logic) - if let Err(err) = heal_object( - &op.bucket, - &op.object, - &op.version_id.clone().unwrap_or_default(), - scan_mode - ).await { - error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err); - } - } else { - // Heal multiple versions (original logic) - let vers = op.versions.len() / 16; - if vers > 0 { - for i in 0..vers { - // Check for cancellation before each version - if cancel_token.is_cancelled() { - info!("MRF heal routine cancelled during version processing"); - return; - } - - let start = i * 16; - let end = start + 16; - if let Err(err) = heal_object( - &op.bucket, - &op.object, - &Uuid::from_slice(&op.versions[start..end]).expect("").to_string(), - scan_mode, - ).await { - error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err); - } - } - } - } - } else { - info!("MRF heal routine channel closed, exiting"); - break; - } - } - } - } - - info!("MRF heal routine stopped gracefully"); - } -} diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index daf032595..770e04027 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -23,13 +23,13 @@ mod chunk_stream; pub mod cmd; pub mod compress; pub mod config; +pub mod data_usage; pub mod disk; pub mod disks_layout; pub mod endpoints; pub mod erasure_coding; pub mod error; pub mod global; -pub mod heal; pub mod lock_utils; pub mod metrics_realtime; pub mod notification_sys; diff --git a/crates/ecstore/src/metrics_realtime.rs b/crates/ecstore/src/metrics_realtime.rs index 298ff846f..a5f5f3a3a 100644 --- a/crates/ecstore/src/metrics_realtime.rs +++ b/crates/ecstore/src/metrics_realtime.rs @@ -15,7 +15,11 @@ use std::collections::{HashMap, HashSet}; use chrono::Utc; -use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr}; +use rustfs_common::{ + globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr}, + heal_channel::DriveState, + metrics::globalMetrics, +}; use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics}; use rustfs_utils::os::get_drive_stats; use serde::{Deserialize, Serialize}; @@ -23,10 +27,6 @@ use tracing::info; use crate::{ admin_server_info::get_local_server_property, - heal::{ - data_scanner_metric::globalScannerMetrics, - heal_commands::{DRIVE_STATE_OK, DRIVE_STATE_UNFORMATTED}, - }, new_object_layer_fn, store_api::StorageAPI, // utils::os::get_drive_stats, @@ -108,7 +108,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) if types.contains(&MetricType::SCANNER) { info!("start get scanner metrics"); - let metrics = globalScannerMetrics.report().await; + let metrics = globalMetrics.report().await; real_time_metrics.aggregated.scanner = Some(metrics); } @@ -147,7 +147,7 @@ async fn collect_local_disks_metrics(disks: &HashSet) -> HashMap Result { - let mut client = node_service_time_out_client(&self.grid_host) - .await - .map_err(|err| Error::other(err.to_string()))?; - let request = Request::new(BackgroundHealStatusRequest {}); - - let response = client.background_heal_status(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); - } - return Err(Error::other("")); - } - let data = response.bg_heal_state; - - let mut buf = Deserializer::new(Cursor::new(data)); - let bg_heal_state: BgHealState = Deserialize::deserialize(&mut buf)?; - - Ok(bg_heal_state) - } - pub async fn get_metacache_listing(&self) -> Result<()> { let _client = node_service_time_out_client(&self.grid_host) .await diff --git a/crates/ecstore/src/rpc/peer_s3_client.rs b/crates/ecstore/src/rpc/peer_s3_client.rs index a7789aabc..10a00e279 100644 --- a/crates/ecstore/src/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/rpc/peer_s3_client.rs @@ -17,10 +17,6 @@ use crate::disk::error::{Error, Result}; use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs}; use crate::disk::{DiskAPI, DiskStore}; use crate::global::GLOBAL_LOCAL_DISK_MAP; -use crate::heal::heal_commands::{ - DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_BUCKET, HealOpts, -}; -use crate::heal::heal_ops::RUSTFS_RESERVED_BUCKET; use crate::store::all_local_disk; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{ @@ -30,6 +26,7 @@ use crate::{ }; use async_trait::async_trait; use futures::future::join_all; +use rustfs_common::heal_channel::{DriveState, HealItemType, HealOpts, RUSTFS_RESERVED_BUCKET}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_protos::node_service_time_out_client; use rustfs_protos::proto_gen::node_service::{ @@ -542,7 +539,7 @@ impl PeerS3Client for RemotePeerS3Client { } Ok(HealResultItem { - heal_item_type: HEAL_ITEM_BUCKET.to_string(), + heal_item_type: HealItemType::Bucket.to_string(), bucket: bucket.to_string(), set_count: 0, ..Default::default() @@ -651,13 +648,13 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result disk, None => { - bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); - as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + bs_clone.write().await[index] = DriveState::Offline.to_string(); + as_clone.write().await[index] = DriveState::Offline.to_string(); return Some(Error::DiskNotFound); } }; - bs_clone.write().await[index] = DRIVE_STATE_OK.to_string(); - as_clone.write().await[index] = DRIVE_STATE_OK.to_string(); + bs_clone.write().await[index] = DriveState::Ok.to_string(); + as_clone.write().await[index] = DriveState::Ok.to_string(); if bucket == RUSTFS_RESERVED_BUCKET { return None; @@ -667,18 +664,18 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result None, Err(err) => match err { Error::DiskNotFound => { - bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); - as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + bs_clone.write().await[index] = DriveState::Offline.to_string(); + as_clone.write().await[index] = DriveState::Offline.to_string(); Some(err) } Error::VolumeNotFound => { - bs_clone.write().await[index] = DRIVE_STATE_MISSING.to_string(); - as_clone.write().await[index] = DRIVE_STATE_MISSING.to_string(); + bs_clone.write().await[index] = DriveState::Missing.to_string(); + as_clone.write().await[index] = DriveState::Missing.to_string(); Some(err) } _ => { - bs_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string(); - as_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string(); + bs_clone.write().await[index] = DriveState::Corrupt.to_string(); + as_clone.write().await[index] = DriveState::Corrupt.to_string(); Some(err) } }, @@ -687,7 +684,7 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result Result { - as_clone.write().await[idx] = DRIVE_STATE_OK.to_string(); + as_clone.write().await[idx] = DriveState::Ok.to_string(); return None; } Err(err) => { diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index e45c9e63a..5f2f078a7 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -21,9 +21,9 @@ use rustfs_protos::{ node_service_time_out_client, proto_gen::node_service::{ CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, - DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest, - ReadAllRequest, ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, - RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, + DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, + ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, + StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, }, }; @@ -32,26 +32,15 @@ use crate::disk::{ ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, endpoint::Endpoint, }; +use crate::disk::{FileReader, FileWriter}; use crate::{ disk::error::{Error, Result}, rpc::build_auth_headers, }; -use crate::{ - disk::{FileReader, FileWriter}, - heal::{ - data_scanner::ShouldSleepFn, - data_usage_cache::{DataUsageCache, DataUsageEntry}, - heal_commands::{HealScanMode, HealingTracker}, - }, -}; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_rio::{HttpReader, HttpWriter}; -use tokio::{ - io::AsyncWrite, - sync::mpsc::{self, Sender}, -}; -use tokio_stream::{StreamExt, wrappers::ReceiverStream}; +use tokio::io::AsyncWrite; use tonic::Request; use tracing::info; use uuid::Uuid; @@ -927,55 +916,6 @@ impl DiskAPI for RemoteDisk { Ok(disk_info) } - - #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))] - async fn ns_scanner( - &self, - cache: &DataUsageCache, - updates: Sender, - scan_mode: HealScanMode, - _we_sleep: ShouldSleepFn, - ) -> Result { - info!("ns_scanner"); - let cache = serde_json::to_string(cache)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - - let (tx, rx) = mpsc::channel(10); - let in_stream = ReceiverStream::new(rx); - let mut response = client.ns_scanner(in_stream).await?.into_inner(); - let request = NsScannerRequest { - disk: self.endpoint.to_string(), - cache, - scan_mode: scan_mode as u64, - }; - tx.send(request) - .await - .map_err(|err| Error::other(format!("can not send request, err: {err}")))?; - - loop { - match response.next().await { - Some(Ok(resp)) => { - if !resp.update.is_empty() { - let data_usage_cache = serde_json::from_str::(&resp.update)?; - let _ = updates.send(data_usage_cache).await; - } else if !resp.data_usage_cache.is_empty() { - let data_usage_cache = serde_json::from_str::(&resp.data_usage_cache)?; - return Ok(data_usage_cache); - } else { - return Err(Error::other("scan was interrupted")); - } - } - _ => return Err(Error::other("scan was interrupted")), - } - } - } - - #[tracing::instrument(skip(self))] - async fn healing(&self) -> Option { - None - } } #[cfg(test)] diff --git a/crates/ecstore/src/rpc/tonic_service.rs b/crates/ecstore/src/rpc/tonic_service.rs index 3607d0daa..cbf30dcc2 100644 --- a/crates/ecstore/src/rpc/tonic_service.rs +++ b/crates/ecstore/src/rpc/tonic_service.rs @@ -22,25 +22,21 @@ use crate::{ DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, error::DiskError, }, - heal::{ - data_usage_cache::DataUsageCache, - heal_commands::{HealOpts, get_local_background_heal_status}, - }, metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}, new_object_layer_fn, rpc::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, store_api::{BucketOptions, DeleteBucketOptions, MakeBucketOptions, StorageAPI}, }; -use futures::{Stream, StreamExt}; +use futures::Stream; use futures_util::future::join_all; -use rustfs_common::globals::GLOBAL_Local_Node_Name; -use rustfs_lock::{LockClient, LockRequest}; +use rustfs_common::{globals::GLOBAL_Local_Node_Name, heal_channel::HealOpts}; use bytes::Bytes; use rmp_serde::{Deserializer, Serializer}; use rustfs_filemeta::{FileInfo, MetacacheReader}; +use rustfs_lock::{LockClient, LockRequest}; use rustfs_madmin::health::{ get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services, }; @@ -1439,96 +1435,6 @@ impl Node for NodeService { } } - type NsScannerStream = ResponseStream; - async fn ns_scanner(&self, request: Request>) -> Result, Status> { - info!("ns_scanner"); - - let mut in_stream = request.into_inner(); - let (tx, rx) = mpsc::channel(10); - - tokio::spawn(async move { - match in_stream.next().await { - Some(Ok(request)) => { - if let Some(disk) = find_local_disk(&request.disk).await { - let cache = match serde_json::from_str::(&request.cache) { - Ok(cache) => cache, - Err(err) => { - tx.send(Ok(NsScannerResponse { - success: false, - update: "".to_string(), - data_usage_cache: "".to_string(), - error: Some(DiskError::other(format!("decode DataUsageCache failed: {err}")).into()), - })) - .await - .expect("working rx"); - return; - } - }; - let (updates_tx, mut updates_rx) = mpsc::channel(100); - let tx_clone = tx.clone(); - let task = tokio::spawn(async move { - loop { - match updates_rx.recv().await { - Some(update) => { - let update = serde_json::to_string(&update).expect("encode failed"); - tx_clone - .send(Ok(NsScannerResponse { - success: true, - update, - data_usage_cache: "".to_string(), - error: None, - })) - .await - .expect("working rx"); - } - None => return, - } - } - }); - let data_usage_cache = disk.ns_scanner(&cache, updates_tx, request.scan_mode as usize, None).await; - let _ = task.await; - match data_usage_cache { - Ok(data_usage_cache) => { - let data_usage_cache = serde_json::to_string(&data_usage_cache).expect("encode failed"); - tx.send(Ok(NsScannerResponse { - success: true, - update: "".to_string(), - data_usage_cache, - error: None, - })) - .await - .expect("working rx"); - } - Err(err) => { - tx.send(Ok(NsScannerResponse { - success: false, - update: "".to_string(), - data_usage_cache: "".to_string(), - error: Some(err.into()), - })) - .await - .expect("working rx"); - } - } - } else { - tx.send(Ok(NsScannerResponse { - success: false, - update: "".to_string(), - data_usage_cache: "".to_string(), - error: Some(DiskError::other("can not find disk".to_string()).into()), - })) - .await - .expect("working rx"); - } - } - _ => todo!(), - } - }); - - let out_stream = ReceiverStream::new(rx); - Ok(tonic::Response::new(Box::pin(out_stream))) - } - async fn lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); // Parse the request to extract resource and owner @@ -2196,28 +2102,7 @@ impl Node for NodeService { &self, _request: Request, ) -> Result, Status> { - let (state, ok) = get_local_background_heal_status().await; - if !ok { - return Ok(tonic::Response::new(BackgroundHealStatusResponse { - success: false, - bg_heal_state: Bytes::new(), - error_info: Some("errServerNotInitialized".to_string()), - })); - } - - let mut buf = Vec::new(); - if let Err(err) = state.serialize(&mut Serializer::new(&mut buf)) { - return Ok(tonic::Response::new(BackgroundHealStatusResponse { - success: false, - bg_heal_state: Bytes::new(), - error_info: Some(err.to_string()), - })); - } - Ok(tonic::Response::new(BackgroundHealStatusResponse { - success: true, - bg_heal_state: buf.into(), - error_info: None, - })) + todo!() } async fn get_metacache_listing( @@ -3412,20 +3297,6 @@ mod tests { assert!(!proc_response.proc_info.is_empty()); } - #[tokio::test] - async fn test_background_heal_status() { - let service = create_test_node_service(); - - let request = Request::new(BackgroundHealStatusRequest {}); - - let response = service.background_heal_status(request).await; - assert!(response.is_ok()); - - let heal_response = response.unwrap().into_inner(); - // May fail if heal status is not available - assert!(heal_response.success || heal_response.error_info.is_some()); - } - #[tokio::test] async fn test_reload_pool_meta() { let service = create_test_node_service(); @@ -3685,14 +3556,6 @@ mod tests { assert!(format!("{service2:?}").contains("NodeService")); } - #[tokio::test] - async fn test_all_disk_method() { - let service = create_test_node_service(); - let disks = service.all_disk().await; - // Should return empty vector in test environment - assert!(disks.is_empty()); - } - #[tokio::test] async fn test_find_disk_method() { let service = create_test_node_service(); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 87b79ecae..4519d5205 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -17,6 +17,7 @@ use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl}; +use crate::disk::STORAGE_FORMAT_FILE; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::{ self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, @@ -26,10 +27,8 @@ use crate::erasure_coding; use crate::erasure_coding::bitrot_verify; use crate::error::{Error, Result}; use crate::error::{ObjectApiError, is_err_object_not_found}; -use crate::global::GLOBAL_MRFState; use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr}; -use crate::heal::data_usage_cache::DataUsageCache; -use crate::heal::heal_ops::{HealEntryFn, HealSequence}; +use crate::store_api::ListObjectVersionsInfo; use crate::store_api::{ListPartsInfo, ObjectToDelete}; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts}, @@ -43,19 +42,7 @@ use crate::{ error::{StorageError, to_object_err}, event::name::EventName, event_notification::{EventArgs, send_event}, - global::{ - GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, - is_dist_erasure, - }, - heal::{ - data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, - data_usage_cache::{DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo}, - heal_commands::{ - DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_DEEP_SCAN, HEAL_ITEM_OBJECT, - HEAL_NORMAL_SCAN, HealOpts, HealScanMode, HealingTracker, - }, - heal_ops::BG_HEALING_UUID, - }, + global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, store_api::{ BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, ListMultipartsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectIO, ObjectInfo, @@ -63,11 +50,6 @@ use crate::{ }, store_init::load_format_erasure, }; -use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; -use crate::{ - heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, - store_api::ListObjectVersionsInfo, -}; use bytes::Bytes; use bytesize::ByteSize; use chrono::Utc; @@ -76,6 +58,7 @@ use glob::Pattern; use http::HeaderMap; use md5::{Digest as Md5Digest, Md5}; use rand::{Rng, seq::SliceRandom}; +use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType, HealOpts, HealScanMode, send_heal_disk}; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; use rustfs_filemeta::{ FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo, @@ -1653,21 +1636,14 @@ impl SetDisks { Err(e) => { warn!("connect_endpoint err {:?}", &e); if ep.is_local && e == DiskError::UnformattedDisk { - info!("unformatteddisk will push_heal_local_disks, {:?}", ep); - GLOBAL_BackgroundHealState.push_heal_local_disks(&[ep.clone()]).await; + info!("unformatteddisk will trigger heal_disk, {:?}", ep); + let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx); + let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await; } return; } }; - if new_disk.is_local() { - if let Some(h) = new_disk.healing().await { - if !h.finished { - GLOBAL_BackgroundHealState.push_heal_local_disks(&[new_disk.endpoint()]).await; - } - } - } - let (set_idx, disk_idx) = match self.find_disk_index(&fm) { Ok(res) => res, Err(e) => { @@ -2033,16 +2009,15 @@ impl SetDisks { let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?; if errs.iter().any(|err| err.is_some()) { - GLOBAL_MRFState - .add_partial(PartialOperation { - bucket: fi.volume.to_string(), - object: fi.name.to_string(), - queued: Utc::now(), - version_id: fi.version_id.map(|v| v.to_string()), - set_index: self.set_index, - pool_index: self.pool_index, - ..Default::default() - }) + let _ = + rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( + fi.volume.to_string(), // bucket + Some(fi.name.to_string()), // object_prefix + false, // force_start + Some(rustfs_common::heal_channel::HealChannelPriority::Normal), // priority + Some(self.pool_index), // pool_index + Some(self.set_index), // set_index + )) .await; } // debug!("get_object_fileinfo pick fi {:?}", &fi); @@ -2174,18 +2149,17 @@ impl SetDisks { match de_err { DiskError::FileNotFound | DiskError::FileCorrupt => { error!("erasure.decode err 111 {:?}", &de_err); - GLOBAL_MRFState - .add_partial(PartialOperation { - bucket: bucket.to_string(), - object: object.to_string(), - queued: Utc::now(), - version_id: fi.version_id.map(|v| v.to_string()), - set_index, - pool_index, - bitrot_scan: de_err == DiskError::FileCorrupt, - ..Default::default() - }) - .await; + let _ = rustfs_common::heal_channel::send_heal_request( + rustfs_common::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(rustfs_common::heal_channel::HealChannelPriority::Normal), + Some(pool_index), + Some(set_index), + ), + ) + .await; has_err = false; } _ => {} @@ -2264,99 +2238,6 @@ impl SetDisks { Ok(()) } - pub async fn list_and_heal(&self, bucket: &str, prefix: &str, opts: &HealOpts, heal_entry: HealEntryFn) -> Result<()> { - let bucket = bucket.to_string(); - let (disks, _) = self.get_online_disk_with_healing(false).await?; - if disks.is_empty() { - return Err(Error::other("listAndHeal: No non-healing drives found")); - } - - let expected_disks = disks.len() / 2 + 1; - let fallback_disks = &disks[expected_disks..]; - let disks = &disks[..expected_disks]; - let resolver = MetadataResolutionParams { - dir_quorum: 1, - obj_quorum: 1, - bucket: bucket.clone(), - strict: false, - ..Default::default() - }; - let path = Path::new(prefix).parent().map_or("", |p| p.to_str().unwrap()); - let filter_prefix = prefix.trim_start_matches(path).trim_matches('/'); - let opts_clone = *opts; - let bucket_agreed = bucket.clone(); - let bucket_partial = bucket.to_string(); - let (tx, rx) = broadcast::channel(1); - let tx_agreed = tx.clone(); - let tx_partial = tx.clone(); - let func_agreed = heal_entry.clone(); - let func_partial = heal_entry.clone(); - let lopts = ListPathRawOptions { - disks: disks.to_vec(), - fallback_disks: fallback_disks.to_vec(), - bucket: bucket.to_string(), - path: path.to_string(), - filter_prefix: { - if filter_prefix.is_empty() { - None - } else { - Some(filter_prefix.to_string()) - } - }, - recursive: true, - forward_to: None, - min_disks: 1, - report_not_found: false, - per_disk_limit: 0, - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let heal_entry = func_agreed.clone(); - let tx_agreed = tx_agreed.clone(); - - Box::pin({ - let bucket_agreed = bucket_agreed.clone(); - async move { - if heal_entry(bucket_agreed.clone(), entry.clone(), opts_clone.scan_mode) - .await - .is_err() - { - let _ = tx_agreed.send(true); - } - } - }) - })), - partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - let heal_entry = func_partial.clone(); - let tx_partial = tx_partial.clone(); - - Box::pin({ - let resolver_partial = resolver.clone(); - let bucket_partial = bucket_partial.clone(); - async move { - let entry = match entries.resolve(resolver_partial) { - Some(entry) => entry, - _ => match entries.first_found() { - (Some(entry), _) => entry, - _ => return, - }, - }; - - if heal_entry(bucket_partial.clone(), entry.clone(), opts_clone.scan_mode) - .await - .is_err() - { - let _ = tx_partial.send(true); - } - } - }) - })), - finished: None, - }; - - _ = list_path_raw(rx, lopts) - .await - .map_err(|err| Error::other(format!("listPathRaw returned {err}: bucket: {bucket}, path: {path}"))); - Ok(()) - } async fn get_online_disk_with_healing(&self, incl_healing: bool) -> Result<(Vec>, bool)> { let (new_disks, _, healing) = self.get_online_disk_with_healing_and_info(incl_healing).await?; @@ -2425,7 +2306,7 @@ impl SetDisks { ) -> disk::error::Result<(HealResultItem, Option)> { info!("SetDisks heal_object"); let mut result = HealResultItem { - heal_item_type: HEAL_ITEM_OBJECT.to_string(), + heal_item_type: HealItemType::Object.to_string(), bucket: bucket.to_string(), object: object.to_string(), version_id: version_id.to_string(), @@ -2531,15 +2412,15 @@ impl SetDisks { let drive_state = match reason { Some(err) => match err { - DiskError::DiskNotFound => DRIVE_STATE_OFFLINE, + DiskError::DiskNotFound => DriveState::Offline.to_string(), DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound | DiskError::PartMissingOrCorrupt - | DiskError::OutdatedXLMeta => DRIVE_STATE_MISSING, - _ => DRIVE_STATE_CORRUPT, + | DiskError::OutdatedXLMeta => DriveState::Missing.to_string(), + _ => DriveState::Corrupt.to_string(), }, - None => DRIVE_STATE_OK, + None => DriveState::Ok.to_string(), }; result.before.drives.push(HealDriveInfo { uuid: "".to_string(), @@ -2757,8 +2638,8 @@ impl SetDisks { false } }; - - for disk in out_dated_disks.iter() { + // write to all disks + for disk in self.disks.read().await.iter() { let writer = create_bitrot_writer( is_inline_buffer, disk.as_ref(), @@ -2821,7 +2702,6 @@ impl SetDisks { // writers.push(None); // } } - // Heal each part. erasure.Heal() will write the healed // part to .rustfs/tmp/uuid/ which needs to be renamed // later to the final location. @@ -2872,7 +2752,6 @@ impl SetDisks { } } } - // Rename from tmp location to the actual location. for (index, disk) in out_dated_disks.iter().enumerate() { if let Some(disk) = disk { @@ -2885,37 +2764,82 @@ impl SetDisks { "rename temp data, src_volume: {}, src_path: {}, dst_volume: {}, dst_path: {}", RUSTFS_META_TMP_BUCKET, tmp_id, bucket, object ); - if let Err(err) = disk + let rename_result = disk .rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object) - .await - { - info!("rename temp data err: {}", err.to_string()); - // self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await?; - return Ok((result, Some(err))); - } + .await; - info!("remove temp object, volume: {}, path: {}", RUSTFS_META_TMP_BUCKET, tmp_id); - self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) - .await - .map_err(DiskError::other)?; - if parts_metadata[index].is_remote() { - let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); - let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); - disk.delete( - bucket, - d_path.to_str().unwrap(), - DeleteOptions { - immediate: true, - recursive: true, - ..Default::default() - }, - ) - .await?; - } + if let Err(err) = &rename_result { + info!( + "rename temp data err: {}. Try fallback to direct xl.meta overwrite...", + err.to_string() + ); - for (i, v) in result.before.drives.iter().enumerate() { - if v.endpoint == disk.endpoint().to_string() { - result.after.drives[i].state = DRIVE_STATE_OK.to_string(); + let healthy_index = latest_disks.iter().position(|d| d.is_some()).unwrap_or(0); + + if let Some(healthy_disk) = &latest_disks[healthy_index] { + let xlmeta_path = format!("{object}/xl.meta"); + + match healthy_disk.read_all(bucket, &xlmeta_path).await { + Ok(xlmeta_bytes) => { + if let Err(e) = disk.write_all(bucket, &xlmeta_path, xlmeta_bytes).await { + info!("fallback xl.meta overwrite failed: {}", e.to_string()); + + return Ok(( + result, + Some(DiskError::other(format!("fallback xl.meta overwrite failed: {e}"))), + )); + } else { + info!("fallback xl.meta overwrite succeeded for disk {}", disk.to_string()); + } + } + + Err(e) => { + info!("read healthy xl.meta failed: {}", e.to_string()); + + return Ok(( + result, + Some(DiskError::other(format!("read healthy xl.meta failed: {e}"))), + )); + } + } + } else { + info!("no healthy disk found for xl.meta fallback overwrite"); + + return Ok(( + result, + Some(DiskError::other("no healthy disk found for xl.meta fallback overwrite")), + )); + } + } else { + info!("remove temp object, volume: {}, path: {}", RUSTFS_META_TMP_BUCKET, tmp_id); + + self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) + .await + .map_err(DiskError::other)?; + + if parts_metadata[index].is_remote() { + let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); + + let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); + + disk.delete( + bucket, + d_path.to_str().unwrap(), + DeleteOptions { + immediate: true, + + recursive: true, + + ..Default::default() + }, + ) + .await?; + } + + for (i, v) in result.before.drives.iter().enumerate() { + if v.endpoint == disk.endpoint().to_string() { + result.after.drives[i].state = DriveState::Ok.to_string(); + } } } } @@ -2975,7 +2899,7 @@ impl SetDisks { disks.clone() }; let mut result = HealResultItem { - heal_item_type: HEAL_ITEM_OBJECT.to_string(), + heal_item_type: HealItemType::Object.to_string(), bucket: bucket.to_string(), object: object.to_string(), disk_count: self.disks.read().await.len(), @@ -3019,11 +2943,11 @@ impl SetDisks { let endpoint = drive.to_string(); let drive_state = match err { Some(err) => match err { - DiskError::DiskNotFound => DRIVE_STATE_OFFLINE, - DiskError::FileNotFound | DiskError::VolumeNotFound => DRIVE_STATE_MISSING, - _ => DRIVE_STATE_CORRUPT, + DiskError::DiskNotFound => DriveState::Offline.to_string(), + DiskError::FileNotFound | DiskError::VolumeNotFound => DriveState::Missing.to_string(), + _ => DriveState::Corrupt.to_string(), }, - None => DRIVE_STATE_OK, + None => DriveState::Ok.to_string(), }; result.before.drives.push(HealDriveInfo { uuid: "".to_string(), @@ -3050,11 +2974,11 @@ impl SetDisks { if let (Some(DiskError::VolumeNotFound | DiskError::FileNotFound), Some(disk)) = (err, disk) { let vol_path = Path::new(bucket).join(object); let drive_state = match disk.make_volume(vol_path.to_str().unwrap()).await { - Ok(_) => DRIVE_STATE_OK, + Ok(_) => DriveState::Ok.to_string(), Err(merr) => match merr { - DiskError::VolumeExists => DRIVE_STATE_OK, - DiskError::DiskNotFound => DRIVE_STATE_OFFLINE, - _ => DRIVE_STATE_CORRUPT, + DiskError::VolumeExists => DriveState::Ok.to_string(), + DiskError::DiskNotFound => DriveState::Offline.to_string(), + _ => DriveState::Corrupt.to_string(), }, }; result.after.drives[index].state = drive_state.to_string(); @@ -3074,7 +2998,7 @@ impl SetDisks { ) -> HealResultItem { let disk_len = { self.disks.read().await.len() }; let mut result = HealResultItem { - heal_item_type: HEAL_ITEM_OBJECT.to_string(), + heal_item_type: HealItemType::Object.to_string(), bucket: bucket.to_string(), object: object.to_string(), object_size: lfi.size as usize, @@ -3096,23 +3020,23 @@ impl SetDisks { result.before.drives.push(HealDriveInfo { uuid: "".to_string(), endpoint: self.set_endpoints[index].to_string(), - state: DRIVE_STATE_OFFLINE.to_string(), + state: DriveState::Offline.to_string(), }); result.after.drives.push(HealDriveInfo { uuid: "".to_string(), endpoint: self.set_endpoints[index].to_string(), - state: DRIVE_STATE_OFFLINE.to_string(), + state: DriveState::Offline.to_string(), }); } - let mut drive_state = DRIVE_STATE_CORRUPT; + let mut drive_state = DriveState::Corrupt; if let Some(err) = &errs[index] { if err == &DiskError::FileNotFound || err == &DiskError::VolumeNotFound { - drive_state = DRIVE_STATE_MISSING; + drive_state = DriveState::Missing; } } else { - drive_state = DRIVE_STATE_OK; + drive_state = DriveState::Ok; } result.before.drives.push(HealDriveInfo { @@ -3204,710 +3128,6 @@ impl SetDisks { } } - pub async fn ns_scanner( - self: Arc, - buckets: &[BucketInfo], - want_cycle: u32, - updates: Sender, - heal_scan_mode: HealScanMode, - ) -> Result<()> { - info!("ns_scanner"); - if buckets.is_empty() { - info!("data-scanner: no buckets to scan, skipping scanner cycle"); - return Ok(()); - } - - let (mut disks, healing) = self.get_online_disk_with_healing(false).await?; - if disks.is_empty() { - info!("data-scanner: all drives are offline or being healed, skipping scanner cycle"); - return Ok(()); - } - - let old_cache = DataUsageCache::load(&self, DATA_USAGE_CACHE_NAME).await?; - let mut cache = DataUsageCache { - info: DataUsageCacheInfo { - name: DATA_USAGE_ROOT.to_string(), - next_cycle: old_cache.info.next_cycle, - ..Default::default() - }, - cache: HashMap::new(), - }; - - // Put all buckets into channel. - let (bucket_tx, bucket_rx) = mpsc::channel(buckets.len()); - // Shuffle buckets to ensure total randomness of buckets, being scanned. - // Otherwise, same set of buckets get scanned across erasure sets always. - // at any given point in time. This allows different buckets to be scanned - // in different order per erasure set, this wider spread is needed when - // there are lots of buckets with different order of objects in them. - let permutes = { - let mut rng = rand::rng(); - let mut permutes: Vec = (0..buckets.len()).collect(); - permutes.shuffle(&mut rng); - permutes - }; - - // Add new buckets first - for idx in permutes.iter() { - let b = buckets[*idx].clone(); - match old_cache.find(&b.name) { - Some(e) => { - cache.replace(&b.name, DATA_USAGE_ROOT, e); - let _ = bucket_tx.send(b).await; - } - None => { - let _ = bucket_tx.send(b).await; - } - } - } - - let (buckets_results_tx, mut buckets_results_rx) = mpsc::channel::(disks.len()); - // 新增：从环境变量读取基础间隔，默认 30 秒 - let set_disk_update_interval_secs = std::env::var("RUSTFS_NS_SCANNER_INTERVAL") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(30); - let update_time = { - let mut rng = rand::rng(); - Duration::from_secs(set_disk_update_interval_secs) + Duration::from_secs_f64(10.0 * rng.random_range(0.0..1.0)) - }; - let mut ticker = interval(update_time); - - // 检查是否需要运行后台任务 - let skip_background_task = std::env::var("RUSTFS_SKIP_BACKGROUND_TASK") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(false); - - let task = if !skip_background_task { - Some(tokio::spawn(async move { - let last_save = Some(SystemTime::now()); - let mut need_loop = true; - while need_loop { - select! { - _ = ticker.tick() => { - if !cache.info.last_update.eq(&last_save) { - let _ = cache.save(DATA_USAGE_CACHE_NAME).await; - let _ = updates.send(cache.clone()).await; - } - } - result = buckets_results_rx.recv() => { - match result { - Some(result) => { - cache.replace(&result.name, &result.parent, result.entry); - cache.info.last_update = Some(SystemTime::now()); - }, - None => { - need_loop = false; - cache.info.next_cycle = want_cycle; - cache.info.last_update = Some(SystemTime::now()); - let _ = cache.save(DATA_USAGE_CACHE_NAME).await; - let _ = updates.send(cache.clone()).await; - } - } - } - } - } - })) - } else { - None - }; - - // Restrict parallelism for disk usage scanner - let max_procs = num_cpus::get(); - if max_procs < disks.len() { - disks = disks[0..max_procs].to_vec(); - } - - let mut futures = Vec::new(); - let bucket_rx = Arc::new(RwLock::new(bucket_rx)); - for disk in disks.iter() { - let disk = match disk { - Some(disk) => disk.clone(), - None => continue, - }; - let self_clone = Arc::clone(&self); - let bucket_rx_clone = bucket_rx.clone(); - let buckets_results_tx_clone = buckets_results_tx.clone(); - futures.push(async move { - loop { - match bucket_rx_clone.write().await.try_recv() { - Err(_) => return, - Ok(bucket_info) => { - let cache_name = Path::new(&bucket_info.name).join(DATA_USAGE_CACHE_NAME); - let mut cache = match DataUsageCache::load(&self_clone, &cache_name.to_string_lossy()).await { - Ok(cache) => cache, - Err(_) => continue, - }; - if cache.info.name.is_empty() { - cache.info.name = bucket_info.name.clone(); - } - cache.info.skip_healing = healing; - cache.info.next_cycle = want_cycle; - if cache.info.name != bucket_info.name { - cache.info = DataUsageCacheInfo { - name: bucket_info.name, - last_update: Some(SystemTime::now()), - next_cycle: want_cycle, - ..Default::default() - }; - } - - // Collect updates. - let (tx, mut rx) = mpsc::channel(1); - let buckets_results_tx_inner_clone = buckets_results_tx_clone.clone(); - let name = cache.info.name.clone(); - let task = tokio::spawn(async move { - loop { - match rx.recv().await { - Some(entry) => { - let _ = buckets_results_tx_inner_clone - .send(DataUsageEntryInfo { - name: name.clone(), - parent: DATA_USAGE_ROOT.to_string(), - entry, - }) - .await; - } - None => return, - } - } - }); - - // Calc usage - let before = cache.info.last_update; - let mut cache = match disk.ns_scanner(&cache, tx, heal_scan_mode, None).await { - Ok(cache) => cache, - Err(_) => { - if cache.info.last_update > before { - let _ = cache.save(&cache_name.to_string_lossy()).await; - } - let _ = task.await; - continue; - } - }; - - cache.info.updates = None; - let _ = task.await; - let mut root = DataUsageEntry::default(); - if let Some(r) = cache.root() { - root = cache.flatten(&r); - if let Some(r) = &root.replication_stats { - if r.empty() { - root.replication_stats = None; - } - } - } - let _ = buckets_results_tx_clone - .send(DataUsageEntryInfo { - name: cache.info.name.clone(), - parent: DATA_USAGE_ROOT.to_string(), - entry: root, - }) - .await; - let _ = cache.save(&cache_name.to_string_lossy()).await; - } - } - info!("continue scanner"); - } - }); - } - - info!("ns_scanner start"); - let _ = join_all(futures).await; - if let Some(task) = task { - let _ = task.await; - } - info!("ns_scanner completed"); - Ok(()) - } - - pub async fn heal_erasure_set(self: Arc, buckets: &[String], tracker: Arc>) -> Result<()> { - let (bg_seq, found) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if !found { - return Err(Error::other("no local healing sequence initialized, unable to heal the drive")); - } - let bg_seq = bg_seq.unwrap(); - let scan_mode = HEAL_NORMAL_SCAN; - - let tracker_defer = tracker.clone(); - let defer = async move { - let mut w = tracker_defer.write().await; - w.set_object("").await; - w.set_bucket("").await; - let _ = w.update().await; - }; - - for bucket in buckets.iter() { - if let Err(err) = HealSequence::heal_bucket(bg_seq.clone(), bucket, true).await { - info!("{}", err.to_string()); - } - } - - let info = match tracker - .read() - .await - .disk - .as_ref() - .unwrap() - .disk_info(&DiskInfoOptions::default()) - .await - { - Ok(info) => info, - Err(err) => { - defer.await; - return Err(Error::other(format!("unable to get disk information before healing it: {err}"))); - } - }; - let num_cores = num_cpus::get(); // 使用 num_cpus crate 获取核心数 - let mut num_healers: usize; - - if info.nr_requests as usize > num_cores { - num_healers = num_cores / 4; - } else { - num_healers = (info.nr_requests / 4) as usize; - } - - if num_healers < 4 { - num_healers = 4; - } - - let v = globalHealConfig.read().await.get_workers(); - if v > 0 { - num_healers = v; - } - info!( - "Healing drive '{}' - use {} parallel workers.", - tracker.read().await.disk.as_ref().unwrap().to_string(), - num_healers - ); - - let jt = rustfs_workers::workers::Workers::new(num_healers).map_err(|err| Error::other(err.to_string()))?; - - let heal_entry_done = |name: String| HealEntryResult { - entry_done: true, - name, - ..Default::default() - }; - - let heal_entry_success = |sz: usize| HealEntryResult { - bytes: sz, - success: true, - ..Default::default() - }; - - let heal_entry_failure = |sz: usize| HealEntryResult { - bytes: sz, - ..Default::default() - }; - - let heal_entry_skipped = |sz: usize| HealEntryResult { - bytes: sz, - skipped: true, - ..Default::default() - }; - - let (result_tx, mut result_rx) = mpsc::channel::(1000); - let tracker_task = tracker.clone(); - let task = tokio::spawn(async move { - loop { - match result_rx.recv().await { - Some(entry) => { - if entry.entry_done { - tracker_task.write().await.set_object(entry.name.as_str()).await; - if let Some(last_update) = tracker_task.read().await.get_last_update().await { - if SystemTime::now().duration_since(last_update).unwrap() > Duration::from_secs(60) { - if let Err(err) = tracker_task.write().await.update().await { - info!("tracker update failed, err: {}", err.to_string()); - } - } - } - continue; - } - - tracker_task - .write() - .await - .update_progress(entry.success, entry.skipped, entry.bytes as u64) - .await; - } - None => { - if let Err(err) = tracker_task.write().await.update().await { - info!("tracker update failed, err: {}", err.to_string()); - } - return; - } - } - } - }); - - let started = tracker.read().await.started; - let mut ret_err = None; - for bucket in buckets.iter() { - if tracker.read().await.is_healed(bucket).await { - info!("bucket{} was healed", bucket); - continue; - } - - let mut forward_to = None; - let b = tracker.read().await.get_bucket().await; - if b == *bucket { - forward_to = Some(tracker.read().await.get_object().await); - } - - if !b.is_empty() { - tracker.write().await.resume().await; - } - - tracker.write().await.set_object("").await; - tracker.write().await.set_bucket("").await; - - if let Err(err) = HealSequence::heal_bucket(bg_seq.clone(), bucket, true).await { - info!("heal bucket failed: {}", err.to_string()); - ret_err = Some(err); - continue; - } - - // let vc: VersioningConfiguration; - // let lc: BucketLifecycleConfiguration; - // let lr: ObjectLockConfiguration; - // let rcfg: ReplicationConfiguration; - // if !is_rustfs_meta_bucket_name(bucket) { - // vc = match get_versioning_config(bucket).await { - // Ok((r, _)) => r, - // Err(err) => { - // ret_err = Some(err); - // info!("get versioning config failed, err: {}", err.to_string()); - // continue; - // } - // }; - // lc = match get_lifecycle_config(bucket).await { - // Ok((r, _)) => r, - // Err(err) => { - // ret_err = Some(err); - // info!("get lifecycle config failed, err: {}", err.to_string()); - // continue; - // } - // }; - // lr = match get_object_lock_config(bucket).await { - // Ok((r, _)) => r, - // Err(err) => { - // ret_err = Some(err); - // info!("get object lock config failed, err: {}", err.to_string()); - // continue; - // } - // }; - // rcfg = match get_replication_config(bucket).await { - // Ok((r, _)) => r, - // Err(err) => { - // ret_err = Some(err); - // info!("get replication config failed, err: {}", err.to_string()); - // continue; - // } - // }; - // } - let (mut disks, _, healing) = self.get_online_disk_with_healing_and_info(true).await?; - if disks.len() == healing { - info!("all drives are in healing state, aborting.."); - defer.await; - return Ok(()); - } - - disks = disks[0..disks.len() - healing].to_vec(); - if disks.len() < self.set_drive_count / 2 { - defer.await; - return Err(Error::other(format!( - "not enough drives (found={}, healing={}, total={}) are available to heal `{}`", - disks.len(), - healing, - self.set_drive_count, - tracker.read().await.disk.as_ref().unwrap().to_string() - ))); - } - - { - let mut rng = rand::rng(); - - // 随机洗牌 - disks.shuffle(&mut rng); - } - - let expected_disk = disks.len() / 2 + 1; - let fallback_disks = disks[expected_disk..].to_vec(); - disks = disks[..expected_disk].to_vec(); - - //todo - // let filter_life_cycle = |bucket: &str, object: &str, fi: FileInfo| { - // if lc.rules.is_empty() { - // return false; - // } - // // todo: versioning - // let versioned = false; - // let obj_info = fi.to_object_info(bucket, object, versioned); - // - // }; - - let result_tx_send = result_tx.clone(); - let bg_seq_send = bg_seq.clone(); - let send = Box::new(move |result: HealEntryResult| { - let result_tx_send = result_tx_send.clone(); - let bg_seq_send = bg_seq_send.clone(); - Box::pin(async move { - let _ = result_tx_send.send(result).await; - bg_seq_send.count_scanned(HEAL_ITEM_OBJECT.to_string()).await; - true - }) - }); - - let jt_clone = jt.clone(); - let self_clone = self.clone(); - let started_clone = started; - let tracker_heal = tracker.clone(); - let bg_seq_clone = bg_seq.clone(); - let send_clone = send.clone(); - let heal_entry = Arc::new(move |bucket: String, entry: MetaCacheEntry| { - info!("heal entry, bucket: {}, entry: {:?}", bucket, entry); - let jt_clone = jt_clone.clone(); - let self_clone = self_clone.clone(); - let started = started_clone; - let tracker_heal = tracker_heal.clone(); - let bg_seq = bg_seq_clone.clone(); - let send = send_clone.clone(); - Box::pin(async move { - let defer = async { - jt_clone.give().await; - }; - if entry.name.is_empty() && entry.metadata.is_empty() { - defer.await; - return; - } - if entry.is_dir() { - defer.await; - return; - } - if bucket == RUSTFS_META_BUCKET - && (Pattern::new("buckets/*/.metacache/*") - .map(|p| p.matches(&entry.name)) - .unwrap_or(false) - || Pattern::new("tmp/.trash/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false)) - { - defer.await; - return; - } - let encoded_entry_name = encode_dir_object(entry.name.as_str()); - let mut result: HealEntryResult; - let fivs = match entry.file_info_versions(bucket.as_str()) { - Ok(fivs) => fivs, - Err(err) => { - match self_clone - .heal_object( - &bucket, - &encoded_entry_name, - "", - &HealOpts { - scan_mode, - remove: HEAL_DELETE_DANGLING, - ..Default::default() - }, - ) - .await - { - Ok((res, None)) => { - bg_seq.count_healed(HEAL_ITEM_OBJECT.to_string()).await; - result = heal_entry_success(res.object_size); - } - Ok((_, Some(err))) => { - if DiskError::is_err_object_not_found(&err) || DiskError::is_err_version_not_found(&err) { - defer.await; - return; - } - - result = heal_entry_failure(0); - bg_seq.count_failed(HEAL_ITEM_OBJECT.to_string()).await; - info!("unable to heal object {}/{}: {}", bucket, entry.name, err.to_string()); - } - Err(_) => { - result = heal_entry_failure(0); - bg_seq.count_failed(HEAL_ITEM_OBJECT.to_string()).await; - info!("unable to heal object {}/{}: {}", bucket, entry.name, err.to_string()); - } - } - send(result.clone()).await; - defer.await; - return; - } - }; - let mut version_not_found = 0; - for version in fivs.versions.iter() { - if let (Some(started), Some(mod_time)) = (started, version.mod_time) { - if mod_time > started { - version_not_found += 1; - if send(heal_entry_skipped(version.size as usize)).await { - defer.await; - return; - } - continue; - } - } - - let mut version_healed = false; - match self_clone - .heal_object( - &bucket, - &encoded_entry_name, - version - .version_id - .as_ref() - .map(|v| v.to_string()) - .unwrap_or("".to_string()) - .as_str(), - &HealOpts { - scan_mode, - remove: HEAL_DELETE_DANGLING, - ..Default::default() - }, - ) - .await - { - Ok((res, None)) => { - if res.after.drives[tracker_heal.read().await.disk_index.unwrap()].state == DRIVE_STATE_OK { - version_healed = true; - } - } - Ok((_, Some(err))) => match err { - DiskError::FileNotFound | DiskError::FileVersionNotFound => { - version_not_found += 1; - continue; - } - _ => {} - }, - Err(_) => {} - } - - if version_healed { - bg_seq.count_healed(HEAL_ITEM_OBJECT.to_string()).await; - result = heal_entry_success(version.size as usize); - } else { - bg_seq.count_failed(HEAL_ITEM_OBJECT.to_string()).await; - result = heal_entry_failure(version.size as usize); - match version.version_id { - Some(version_id) => { - info!("unable to heal object {}/{}-v({})", bucket, version.name, version_id); - } - None => { - info!("unable to heal object {}/{}", bucket, version.name); - } - } - } - - if !send(result).await { - defer.await; - return; - } - } - if version_not_found == fivs.versions.len() { - defer.await; - return; - } - send(heal_entry_done(entry.name.clone())).await; - defer.await; - }) - }); - let resolver = MetadataResolutionParams { - dir_quorum: 1, - obj_quorum: 1, - bucket: bucket.clone(), - ..Default::default() - }; - let (_, rx) = broadcast::channel(1); - let jt_agree = jt.clone(); - let jt_partial = jt.clone(); - let bucket_agree = bucket.clone(); - let bucket_partial = bucket.clone(); - let heal_entry_agree = heal_entry.clone(); - let heal_entry_partial = heal_entry.clone(); - if let Err(err) = list_path_raw( - rx, - ListPathRawOptions { - disks, - fallback_disks, - bucket: bucket.clone(), - recursive: true, - forward_to, - min_disks: 1, - report_not_found: false, - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let jt = jt_agree.clone(); - let bucket = bucket_agree.clone(); - let heal_entry = heal_entry_agree.clone(); - Box::pin(async move { - jt.take().await; - let bucket = bucket.clone(); - tokio::spawn(async move { - heal_entry(bucket, entry).await; - }); - }) - })), - partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - let jt = jt_partial.clone(); - let bucket = bucket_partial.clone(); - let heal_entry = heal_entry_partial.clone(); - Box::pin({ - let heal_entry = heal_entry.clone(); - let resolver = resolver.clone(); - async move { - let entry = if let Some(entry) = entries.resolve(resolver) { - entry - } else if let (Some(entry), _) = entries.first_found() { - entry - } else { - return; - }; - jt.take().await; - let bucket = bucket.clone(); - let heal_entry = heal_entry.clone(); - tokio::spawn(async move { - heal_entry(bucket, entry).await; - }); - } - }) - })), - finished: None, - ..Default::default() - }, - ) - .await - { - ret_err = Some(err.into()); - } - - jt.wait().await; - if let Some(err) = ret_err.as_ref() { - info!("listing failed with: {} on bucket: {}", err.to_string(), bucket); - continue; - } - tracker.write().await.bucket_done(bucket).await; - if let Err(err) = tracker.write().await.update().await { - info!("tracker update failed, err: {}", err.to_string()); - } - } - - if let Some(err) = ret_err.as_ref() { - return Err(err.clone()); - } - if !tracker.read().await.queue_buckets.is_empty() { - return Err(Error::other(format!( - "not all buckets were healed: {:?}", - tracker.read().await.queue_buckets - ))); - } - drop(result_tx); - let _ = task.await; - defer.await; - Ok(()) - } - async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> { let disks = self.get_disks_internal().await; let write_quorum = disks.len() / 2 + 1; @@ -4695,17 +3915,15 @@ impl StorageAPI for SetDisks { #[tracing::instrument(skip(self))] async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { - GLOBAL_MRFState - .add_partial(PartialOperation { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: Some(version_id.to_string()), - queued: Utc::now(), - set_index: self.set_index, - pool_index: self.pool_index, - ..Default::default() - }) - .await; + let _ = rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(rustfs_common::heal_channel::HealChannelPriority::Normal), + Some(self.pool_index), + Some(self.set_index), + )) + .await; Ok(()) } @@ -5847,16 +5065,15 @@ impl StorageAPI for SetDisks { .await?; } if let Some(versions) = versions { - GLOBAL_MRFState - .add_partial(PartialOperation { - bucket: bucket.to_string(), - object: object.to_string(), - queued: Utc::now(), - versions, - set_index: self.set_index, - pool_index: self.pool_index, - ..Default::default() - }) + let _ = + rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(rustfs_common::heal_channel::HealChannelPriority::Normal), + Some(self.pool_index), + Some(self.set_index), + )) .await; } @@ -5943,11 +5160,11 @@ impl StorageAPI for SetDisks { let (result, err) = self.heal_object(bucket, object, version_id, opts).await?; if let Some(err) = err.as_ref() { match err { - &DiskError::FileCorrupt if opts.scan_mode != HEAL_DEEP_SCAN => { + &DiskError::FileCorrupt if opts.scan_mode != HealScanMode::Deep => { // Instead of returning an error when a bitrot error is detected // during a normal heal scan, heal again with bitrot flag enabled. let mut opts = *opts; - opts.scan_mode = HEAL_DEEP_SCAN; + opts.scan_mode = HealScanMode::Deep; let (result, err) = self.heal_object(bucket, object, version_id, &opts).await?; return Ok((result, err.map(|e| e.into()))); } @@ -5957,18 +5174,6 @@ impl StorageAPI for SetDisks { Ok((result, err.map(|e| e.into()))) } - #[tracing::instrument(skip(self))] - async fn heal_objects( - &self, - _bucket: &str, - _prefix: &str, - _opts: &HealOpts, - _hs: Arc, - _is_meta: bool, - ) -> Result<()> { - unimplemented!() - } - #[tracing::instrument(skip(self))] async fn get_pool_and_set(&self, _id: &str) -> Result<(Option, Option, Option)> { unimplemented!() @@ -5978,6 +5183,14 @@ impl StorageAPI for SetDisks { async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { unimplemented!() } + + #[tracing::instrument(skip(self))] + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let mut get_object_reader = + ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; + let _ = get_object_reader.read_all().await?; + Ok(()) + } } #[derive(Debug, PartialEq, Eq)] @@ -6256,7 +5469,7 @@ async fn disks_with_all_parts( let mut verify_resp = CheckPartsResp::default(); let mut verify_err = None; meta.data_dir = latest_meta.data_dir; - if scan_mode == HEAL_DEEP_SCAN { + if scan_mode == HealScanMode::Deep { // disk has a valid xl.meta but may not have all the // parts. This is considered an outdated disk, since // it needs healing too. diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 79a35684c..40e692ae9 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -28,9 +28,6 @@ use crate::{ endpoints::{Endpoints, PoolEndpoints}, error::StorageError, global::{GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure}, - heal::heal_commands::{ - DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_METADATA, HealOpts, - }, set_disk::SetDisks, store_api::{ BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, @@ -41,7 +38,11 @@ use crate::{ }; use futures::future::join_all; use http::HeaderMap; -use rustfs_common::globals::GLOBAL_Local_Node_Name; +use rustfs_common::heal_channel::HealOpts; +use rustfs_common::{ + globals::GLOBAL_Local_Node_Name, + heal_channel::{DriveState, HealItemType}, +}; use rustfs_filemeta::FileInfo; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; @@ -49,7 +50,6 @@ use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use tokio::sync::RwLock; use uuid::Uuid; -use crate::heal::heal_ops::HealSequence; use tokio::sync::broadcast::{Receiver, Sender}; use tokio::time::Duration; use tracing::warn; @@ -787,7 +787,7 @@ impl StorageAPI for Sets { Err(err) => return Ok((HealResultItem::default(), Some(err))), }; let mut res = HealResultItem { - heal_item_type: HEAL_ITEM_METADATA.to_string(), + heal_item_type: HealItemType::Metadata.to_string(), detail: "disk-format".to_string(), disk_count: self.set_count * self.set_drive_count, set_count: self.set_count, @@ -811,7 +811,6 @@ impl StorageAPI for Sets { // return Ok((res, Some(Error::new(DiskError::CorruptedFormat)))); // } - let format_op_id = Uuid::new_v4().to_string(); let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs); if !dry_run { let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count]; @@ -819,14 +818,14 @@ impl StorageAPI for Sets { for (j, fm) in set.iter().enumerate() { if let Some(fm) = fm { res.after.drives[i * self.set_drive_count + j].uuid = fm.erasure.this.to_string(); - res.after.drives[i * self.set_drive_count + j].state = DRIVE_STATE_OK.to_string(); + res.after.drives[i * self.set_drive_count + j].state = DriveState::Ok.to_string(); tmp_new_formats[i * self.set_drive_count + j] = Some(fm.clone()); } } } // Save new formats `format.json` on unformatted disks. for (fm, disk) in tmp_new_formats.iter_mut().zip(disks.iter()) { - if fm.is_some() && disk.is_some() && save_format_file(disk, fm, &format_op_id).await.is_err() { + if fm.is_some() && disk.is_some() && save_format_file(disk, fm).await.is_err() { let _ = disk.as_ref().unwrap().close().await; *fm = None; } @@ -869,17 +868,6 @@ impl StorageAPI for Sets { .await } #[tracing::instrument(skip(self))] - async fn heal_objects( - &self, - _bucket: &str, - _prefix: &str, - _opts: &HealOpts, - _hs: Arc, - _is_meta: bool, - ) -> Result<()> { - unimplemented!() - } - #[tracing::instrument(skip(self))] async fn get_pool_and_set(&self, _id: &str) -> Result<(Option, Option, Option)> { unimplemented!() } @@ -887,6 +875,13 @@ impl StorageAPI for Sets { async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { unimplemented!() } + + #[tracing::instrument(skip(self))] + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object) + .verify_object_integrity(bucket, object, opts) + .await + } } async fn _close_storage_disks(disks: &[Option]) { @@ -957,17 +952,17 @@ fn formats_to_drives_info(endpoints: &Endpoints, formats: &[Option], e for (index, format) in formats.iter().enumerate() { let drive = endpoints.get_string(index); let state = if format.is_some() { - DRIVE_STATE_OK + DriveState::Ok.to_string() } else if let Some(Some(err)) = errs.get(index) { if *err == DiskError::UnformattedDisk { - DRIVE_STATE_MISSING + DriveState::Missing.to_string() } else if *err == DiskError::DiskNotFound { - DRIVE_STATE_OFFLINE + DriveState::Offline.to_string() } else { - DRIVE_STATE_CORRUPT + DriveState::Corrupt.to_string() } } else { - DRIVE_STATE_CORRUPT + DriveState::Corrupt.to_string() }; let uuid = if let Some(format) = format { diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 193d3e061..88cffaeca 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -30,11 +30,6 @@ use crate::global::{ GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, }; -use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; -use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; -use crate::heal::heal_commands::{HEAL_ITEM_METADATA, HealOpts, HealScanMode}; -use crate::heal::heal_ops::{HealEntryFn, HealSequence}; -use crate::new_object_layer_fn; use crate::notification_sys::get_global_notification_sys; use crate::pools::PoolMeta; use crate::rebalance::RebalanceMeta; @@ -54,13 +49,12 @@ use crate::{ store_init, }; use futures::future::join_all; -use glob::Pattern; use http::HeaderMap; use lazy_static::lazy_static; use rand::Rng as _; use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; +use rustfs_common::heal_channel::{HealItemType, HealOpts}; use rustfs_filemeta::FileInfo; -use rustfs_filemeta::MetaCacheEntry; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::crypto::base64_decode; use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; @@ -73,9 +67,8 @@ use std::time::SystemTime; use std::{collections::HashMap, sync::Arc, time::Duration}; use time::OffsetDateTime; use tokio::select; -use tokio::sync::mpsc::Sender; -use tokio::sync::{RwLock, broadcast, mpsc}; -use tokio::time::{interval, sleep}; +use tokio::sync::{RwLock, broadcast}; +use tokio::time::sleep; use tracing::{debug, info}; use tracing::{error, warn}; use uuid::Uuid; @@ -811,123 +804,6 @@ impl ECStore { errs } - pub async fn ns_scanner( - &self, - updates: Sender, - want_cycle: usize, - heal_scan_mode: HealScanMode, - ) -> Result<()> { - info!("ns_scanner updates - {}", want_cycle); - let all_buckets = self.list_bucket(&BucketOptions::default()).await?; - if all_buckets.is_empty() { - info!("No buckets found"); - let _ = updates.send(DataUsageInfo::default()).await; - return Ok(()); - } - - let mut total_results = 0; - let mut result_index = 0; - self.pools.iter().for_each(|pool| { - total_results += pool.disk_set.len(); - }); - let results = Arc::new(RwLock::new(vec![DataUsageCache::default(); total_results])); - let (cancel, _) = broadcast::channel(100); - let first_err = Arc::new(RwLock::new(None)); - let mut futures = Vec::new(); - for pool in self.pools.iter() { - for set in pool.disk_set.iter() { - let index = result_index; - let results_clone = results.clone(); - let first_err_clone = first_err.clone(); - let cancel_clone = cancel.clone(); - let all_buckets_clone = all_buckets.clone(); - futures.push(async move { - let (tx, mut rx) = mpsc::channel(1); - let task = tokio::spawn(async move { - loop { - match rx.recv().await { - Some(info) => { - results_clone.write().await[index] = info; - } - None => { - return; - } - } - } - }); - if let Err(err) = set - .clone() - .ns_scanner(&all_buckets_clone, want_cycle as u32, tx, heal_scan_mode) - .await - { - let mut f_w = first_err_clone.write().await; - if f_w.is_none() { - *f_w = Some(err); - } - let _ = cancel_clone.send(true); - return; - } - let _ = task.await; - }); - result_index += 1; - } - } - let (update_closer_tx, mut update_close_rx) = mpsc::channel(10); - let mut ctx_clone = cancel.subscribe(); - let all_buckets_clone = all_buckets.clone(); - // 新增：从环境变量读取 interval，默认 30 秒 - let ns_scanner_interval_secs = std::env::var("RUSTFS_NS_SCANNER_INTERVAL") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(30); - - // 检查是否跳过后台任务 - let skip_background_task = std::env::var("RUSTFS_SKIP_BACKGROUND_TASK") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(false); - - if skip_background_task { - info!("跳过后台任务执行：RUSTFS_SKIP_BACKGROUND_TASK=true"); - return Ok(()); - } - - let task = tokio::spawn(async move { - let mut last_update: Option = None; - let mut interval = interval(Duration::from_secs(ns_scanner_interval_secs)); - let all_merged = Arc::new(RwLock::new(DataUsageCache::default())); - loop { - select! { - _ = ctx_clone.recv() => { - return; - } - _ = update_close_rx.recv() => { - update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await; - return; - } - _ = interval.tick() => { - update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await; - } - } - } - }); - let _ = join_all(futures).await; - let mut ctx_closer = cancel.subscribe(); - select! { - _ = update_closer_tx.send(true) => { - - } - _ = ctx_closer.recv() => { - - } - } - let _ = task.await; - if let Some(err) = first_err.read().await.as_ref() { - return Err(err.clone()); - } - Ok(()) - } - async fn get_latest_object_info_with_idx( &self, bucket: &str, @@ -1068,34 +944,6 @@ impl ECStore { } } -#[tracing::instrument(level = "info", skip(all_buckets, updates))] -async fn update_scan( - all_merged: Arc>, - results: Arc>>, - last_update: &mut Option, - all_buckets: Vec, - updates: Sender, -) { - let mut w = all_merged.write().await; - *w = DataUsageCache { - info: DataUsageCacheInfo { - name: DATA_USAGE_ROOT.to_string(), - ..Default::default() - }, - ..Default::default() - }; - for info in results.read().await.iter() { - if info.info.last_update.is_none() { - return; - } - w.merge(info); - } - if (last_update.is_none() || w.info.last_update > *last_update) && w.root().is_some() { - let _ = updates.send(w.dui(&w.info.name, &all_buckets)).await; - *last_update = w.info.last_update; - } -} - pub async fn find_local_disk(disk_path: &String) -> Option { let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; @@ -2237,7 +2085,7 @@ impl StorageAPI for ECStore { async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { info!("heal_format"); let mut r = HealResultItem { - heal_item_type: HEAL_ITEM_METADATA.to_string(), + heal_item_type: HealItemType::Metadata.to_string(), detail: "disk-format".to_string(), ..Default::default() }; @@ -2351,120 +2199,6 @@ impl StorageAPI for ECStore { Ok((HealResultItem::default(), Some(Error::FileNotFound))) } - #[tracing::instrument(skip(self))] - async fn heal_objects( - &self, - bucket: &str, - prefix: &str, - opts: &HealOpts, - hs: Arc, - is_meta: bool, - ) -> Result<()> { - info!("heal objects"); - let opts_clone = *opts; - let heal_entry: HealEntryFn = Arc::new(move |bucket: String, entry: MetaCacheEntry, scan_mode: HealScanMode| { - let opts_clone = opts_clone; - let hs_clone = hs.clone(); - Box::pin(async move { - if entry.is_dir() { - return Ok(()); - } - - if bucket == RUSTFS_META_BUCKET - && Pattern::new("buckets/*/.metacache/*") - .map(|p| p.matches(&entry.name)) - .unwrap_or(false) - || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - { - return Ok(()); - } - let fivs = match entry.file_info_versions(&bucket) { - Ok(fivs) => fivs, - Err(_) => { - return if is_meta { - HealSequence::heal_meta_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await - } else { - HealSequence::heal_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await - }; - } - }; - - if opts_clone.remove && !opts_clone.dry_run { - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts_clone).await { - info!("unable to check object {}/{} for abandoned data: {}", bucket, entry.name, err.to_string()); - } - } - for version in fivs.versions.iter() { - if is_meta { - if let Err(err) = HealSequence::heal_meta_object( - hs_clone.clone(), - &bucket, - &version.name, - &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()), - scan_mode, - ) - .await - { - match err { - Error::FileNotFound | Error::FileVersionNotFound => {} - _ => { - return Err(err); - } - } - } - } else if let Err(err) = HealSequence::heal_object( - hs_clone.clone(), - &bucket, - &version.name, - &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()), - scan_mode, - ) - .await - { - match err { - Error::FileNotFound | Error::FileVersionNotFound => {} - _ => { - return Err(err); - } - } - } - } - Ok(()) - }) - }); - let mut first_err = None; - for (idx, pool) in self.pools.iter().enumerate() { - if opts.pool.is_some() && opts.pool.unwrap() != idx { - continue; - } - //TODO: IsSuspended - - for (idx, set) in pool.disk_set.iter().enumerate() { - if opts.set.is_some() && opts.set.unwrap() != idx { - continue; - } - - if let Err(err) = set.list_and_heal(bucket, prefix, opts, heal_entry.clone()).await { - if first_err.is_none() { - first_err = Some(err) - } - } - } - } - - if first_err.is_some() { - return Err(first_err.unwrap()); - } - - Ok(()) - } - #[tracing::instrument(skip(self))] async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)> { for (pool_idx, pool) in self.pools.iter().enumerate() { @@ -2501,6 +2235,13 @@ impl StorageAPI for ECStore { Ok(()) } + + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let mut get_object_reader = + ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; + let _ = get_object_reader.read_all().await?; + Ok(()) + } } async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) { diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index 7ca9cdb8f..5a582a528 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -15,16 +15,16 @@ use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::versioning::VersioningApi as _; use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType}; +use crate::disk::DiskStore; use crate::error::{Error, Result}; -use crate::heal::heal_ops::HealSequence; use crate::store_utils::clean_metadata; use crate::{ bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::ExpirationOptions, bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions}, }; -use crate::{disk::DiskStore, heal::heal_commands::HealOpts}; use http::{HeaderMap, HeaderValue}; +use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; use rustfs_madmin::heal_commands::HealResultItem; @@ -970,6 +970,7 @@ pub trait StorageAPI: ObjectIO { // Walk TODO: async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; async fn copy_object( &self, src_bucket: &str, @@ -1072,8 +1073,8 @@ pub trait StorageAPI: ObjectIO { version_id: &str, opts: &HealOpts, ) -> Result<(HealResultItem, Option)>; - async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, hs: Arc, is_meta: bool) - -> Result<()>; + // async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, hs: Arc, is_meta: bool) + // -> Result<()>; async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)>; async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; } diff --git a/crates/ecstore/src/store_init.rs b/crates/ecstore/src/store_init.rs index 85ef58988..36bc0f33c 100644 --- a/crates/ecstore/src/store_init.rs +++ b/crates/ecstore/src/store_init.rs @@ -24,7 +24,6 @@ use crate::{ new_disk, }, endpoints::Endpoints, - heal::heal_commands::init_healing_tracker, }; use futures::future::join_all; use std::collections::{HashMap, hash_map::Entry}; @@ -288,7 +287,7 @@ async fn save_format_file_all(disks: &[Option], formats: &[Option], formats: &[Option, format: &Option, heal_id: &str) -> disk::error::Result<()> { +pub async fn save_format_file(disk: &Option, format: &Option) -> disk::error::Result<()> { if disk.is_none() { return Err(DiskError::DiskNotFound); } @@ -331,10 +330,6 @@ pub async fn save_format_file(disk: &Option, format: &Option bool { - // TODO: when use inlinedata - true + !self.inlinedata() } pub fn inlinedata(&self) -> bool { diff --git a/crates/protos/README.md b/crates/protos/README.md index 73059598f..1dca836e0 100644 --- a/crates/protos/README.md +++ b/crates/protos/README.md @@ -28,6 +28,12 @@ - Type-safe message definitions - Code generation for multiple programming languages +## generate code + +``` +cargo run --bin gproto +``` + ## 📚 Documentation For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs). diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 44f133792..b48cc3807 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -604,26 +604,6 @@ pub struct DiskInfoResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct NsScannerRequest { - #[prost(string, tag = "1")] - pub disk: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub cache: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub scan_mode: u64, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct NsScannerResponse { - #[prost(bool, tag = "1")] - pub success: bool, - #[prost(string, tag = "2")] - pub update: ::prost::alloc::string::String, - #[prost(string, tag = "3")] - pub data_usage_cache: ::prost::alloc::string::String, - #[prost(message, optional, tag = "4")] - pub error: ::core::option::Option, -} /// lock api have same argument type #[derive(Clone, PartialEq, ::prost::Message)] pub struct GenerallyLockRequest { @@ -1660,21 +1640,6 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "DiskInfo")); self.inner.unary(req, path, codec).await } - pub async fn ns_scanner( - &mut self, - request: impl tonic::IntoStreamingRequest, - ) -> std::result::Result>, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NsScanner"); - let mut req = request.into_streaming_request(); - req.extensions_mut() - .insert(GrpcMethod::new("node_service.NodeService", "NsScanner")); - self.inner.streaming(req, path, codec).await - } pub async fn lock( &mut self, request: impl tonic::IntoRequest, @@ -2466,14 +2431,6 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; - /// Server streaming response type for the NsScanner method. - type NsScannerStream: tonic::codegen::tokio_stream::Stream> - + std::marker::Send - + 'static; - async fn ns_scanner( - &self, - request: tonic::Request>, - ) -> std::result::Result, tonic::Status>; async fn lock( &self, request: tonic::Request, @@ -3670,35 +3627,6 @@ pub mod node_service_server { }; Box::pin(fut) } - "/node_service.NodeService/NsScanner" => { - #[allow(non_camel_case_types)] - struct NsScannerSvc(pub Arc); - impl tonic::server::StreamingService for NsScannerSvc { - type Response = super::NsScannerResponse; - type ResponseStream = T::NsScannerStream; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request>) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { ::ns_scanner(&inner, request).await }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = NsScannerSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); - let res = grpc.streaming(method, req).await; - Ok(res) - }; - Box::pin(fut) - } "/node_service.NodeService/Lock" => { #[allow(non_camel_case_types)] struct LockSvc(pub Arc); diff --git a/crates/protos/src/main.rs b/crates/protos/src/main.rs index 889f28626..1a752bee4 100644 --- a/crates/protos/src/main.rs +++ b/crates/protos/src/main.rs @@ -16,7 +16,7 @@ use std::{cmp, env, fs, io::Write, path::Path, process::Command}; type AnyError = Box; -const VERSION_PROTOBUF: Version = Version(30, 2, 0); // 30.2.0 +const VERSION_PROTOBUF: Version = Version(27, 2, 0); // 27.2.0 const VERSION_FLATBUFFERS: Version = Version(24, 3, 25); // 24.3.25 /// Build protos if the major version of `flatc` or `protoc` is greater /// or lesser than the expected version. @@ -27,7 +27,7 @@ const ENV_FLATC_PATH: &str = "FLATC_PATH"; fn main() -> Result<(), AnyError> { let version = protobuf_compiler_version()?; let need_compile = match version.compare_ext(&VERSION_PROTOBUF) { - Ok(cmp::Ordering::Equal) => true, + Ok(cmp::Ordering::Greater) => true, Ok(_) => { let version_err = Version::build_error_message(&version, &VERSION_PROTOBUF).unwrap(); println!("cargo:warning=Tool `protoc` {version_err}, skip compiling."); @@ -47,6 +47,7 @@ fn main() -> Result<(), AnyError> { // path of proto file let project_root_dir = env::current_dir()?.join("crates/protos/src"); let proto_dir = project_root_dir.clone(); + println!("proto_dir: {proto_dir:?}"); let proto_files = &["node.proto"]; let proto_out_dir = project_root_dir.join("generated").join("proto_gen"); let flatbuffer_out_dir = project_root_dir.join("generated").join("flatbuffers_generated"); @@ -67,12 +68,44 @@ fn main() -> Result<(), AnyError> { let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "pub mod node_service;")?; + writeln!( + &mut generated_mod_rs, + r#"// Copyright 2024 RustFS Team + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License."# + )?; generated_mod_rs.flush()?; let generated_mod_rs_path = project_root_dir.join("generated").join("mod.rs"); let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "#![allow(unused_imports)]")?; + writeln!( + &mut generated_mod_rs, + r#"// Copyright 2024 RustFS Team + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License."# + )?; writeln!(&mut generated_mod_rs, "#![allow(clippy::all)]")?; writeln!(&mut generated_mod_rs, "pub mod proto_gen;")?; generated_mod_rs.flush()?; @@ -107,7 +140,7 @@ fn compile_flatbuffers_models, S: AsRef>( ) -> Result<(), AnyError> { let version = flatbuffers_compiler_version(flatc_path)?; let need_compile = match version.compare_ext(&VERSION_FLATBUFFERS) { - Ok(cmp::Ordering::Equal) => true, + Ok(cmp::Ordering::Greater) => true, Ok(_) => { let version_err = Version::build_error_message(&version, &VERSION_FLATBUFFERS).unwrap(); println!("cargo:warning=Tool `{flatc_path}` {version_err}, skip compiling."); @@ -217,7 +250,7 @@ impl Version { Ok(self.compare_major_version(expected_version)) } else { match self.compare_major_version(expected_version) { - cmp::Ordering::Equal => Ok(cmp::Ordering::Equal), + cmp::Ordering::Greater => Ok(cmp::Ordering::Greater), _ => Err(Self::build_error_message(self, expected_version).unwrap()), } } diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index b375f37bc..547e0e6be 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -423,19 +423,6 @@ message DiskInfoResponse { optional Error error = 3; } -message NsScannerRequest { - string disk = 1; - string cache = 2; - uint64 scan_mode = 3; -} - -message NsScannerResponse { - bool success = 1; - string update = 2; - string data_usage_cache = 3; - optional Error error = 4; -} - // lock api have same argument type message GenerallyLockRequest { string args = 1; @@ -805,7 +792,6 @@ service NodeService { rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {}; - rpc NsScanner(stream NsScannerRequest) returns (stream NsScannerResponse) {}; /* -------------------------------lock service-------------------------- */ diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index 5061a6097..3923fc636 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -23,18 +23,15 @@ use http::{HeaderMap, Uri}; use hyper::StatusCode; use matchit::Params; use percent_encoding::{AsciiSet, CONTROLS, percent_encode}; +use rustfs_common::heal_channel::HealOpts; use rustfs_ecstore::admin_server_info::get_server_info; use rustfs_ecstore::bucket::metadata_sys::{self, get_replication_config}; use rustfs_ecstore::bucket::target::BucketTarget; use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; use rustfs_ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys}; +use rustfs_ecstore::data_usage::load_data_usage_from_backend; use rustfs_ecstore::error::StorageError; -use rustfs_ecstore::global::GLOBAL_ALlHealState; use rustfs_ecstore::global::get_global_action_cred; -// use rustfs_ecstore::heal::data_usage::load_data_usage_from_backend; -use rustfs_ecstore::heal::data_usage::load_data_usage_from_backend; -use rustfs_ecstore::heal::heal_commands::HealOpts; -use rustfs_ecstore::heal::heal_ops::new_heal_sequence; use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; @@ -689,33 +686,20 @@ impl Operation for HealHandler { } let heal_path = path_join(&[PathBuf::from(hip.bucket.clone()), PathBuf::from(hip.obj_prefix.clone())]); - if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop { - match GLOBAL_ALlHealState - .pop_heal_status_json(heal_path.to_str().unwrap_or_default(), &hip.client_token) - .await - { - Ok(b) => { - info!("pop_heal_status_json success"); - return Ok(S3Response::new((StatusCode::OK, Body::from(b)))); - } - Err(_e) => { - info!("pop_heal_status_json failed"); - return Ok(S3Response::new((StatusCode::INTERNAL_SERVER_ERROR, Body::from(vec![])))); - } - } - } let (tx, mut rx) = mpsc::channel(1); - if hip.force_stop { + + if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop { + // Query heal status let tx_clone = tx.clone(); + let heal_path_str = heal_path.to_str().unwrap_or_default().to_string(); + let client_token = hip.client_token.clone(); spawn(async move { - match GLOBAL_ALlHealState - .stop_heal_sequence(heal_path.to_str().unwrap_or_default()) - .await - { - Ok(b) => { + match rustfs_common::heal_channel::query_heal_status(heal_path_str, client_token).await { + Ok(_) => { + // TODO: Get actual response from channel let _ = tx_clone .send(HealResp { - resp_bytes: b, + resp_bytes: vec![], ..Default::default() }) .await; @@ -723,7 +707,32 @@ impl Operation for HealHandler { Err(e) => { let _ = tx_clone .send(HealResp { - _api_err: Some(e), + _api_err: Some(StorageError::other(e)), + ..Default::default() + }) + .await; + } + } + }); + } else if hip.force_stop { + // Cancel heal task + let tx_clone = tx.clone(); + let heal_path_str = heal_path.to_str().unwrap_or_default().to_string(); + spawn(async move { + match rustfs_common::heal_channel::cancel_heal_task(heal_path_str).await { + Ok(_) => { + // TODO: Get actual response from channel + let _ = tx_clone + .send(HealResp { + resp_bytes: vec![], + ..Default::default() + }) + .await; + } + Err(e) => { + let _ = tx_clone + .send(HealResp { + _api_err: Some(StorageError::other(e)), ..Default::default() }) .await; @@ -731,22 +740,36 @@ impl Operation for HealHandler { } }); } else if hip.client_token.is_empty() { - let nh = Arc::new(new_heal_sequence(&hip.bucket, &hip.obj_prefix, "", hip.hs, hip.force_start)); + // Use new heal channel mechanism let tx_clone = tx.clone(); spawn(async move { - match GLOBAL_ALlHealState.launch_new_heal_sequence(nh).await { - Ok(b) => { + // Create heal request through channel + let heal_request = rustfs_common::heal_channel::create_heal_request( + hip.bucket.clone(), + if hip.obj_prefix.is_empty() { + None + } else { + Some(hip.obj_prefix.clone()) + }, + hip.force_start, + Some(rustfs_common::heal_channel::HealChannelPriority::Normal), + ); + + match rustfs_common::heal_channel::send_heal_request(heal_request).await { + Ok(_) => { + // Success - send empty response for now let _ = tx_clone .send(HealResp { - resp_bytes: b, + resp_bytes: vec![], ..Default::default() }) .await; } Err(e) => { + // Error - send error response let _ = tx_clone .send(HealResp { - _api_err: Some(e), + _api_err: Some(StorageError::other(e)), ..Default::default() }) .await; @@ -1072,7 +1095,7 @@ impl Operation for RemoveRemoteTargetHandler { #[cfg(test)] mod test { - use rustfs_ecstore::heal::heal_commands::HealOpts; + use rustfs_common::heal_channel::HealOpts; #[ignore] // FIXME: failed in github actions #[test] diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index f8cbd0ed5..4e24d7dc1 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -32,6 +32,7 @@ use rpc::register_rpc_route; use s3s::route::S3Route; const ADMIN_PREFIX: &str = "/rustfs/admin"; +// const ADMIN_PREFIX: &str = "/minio/admin"; pub fn make_admin_route(console_enabled: bool) -> std::io::Result { let mut r: S3Router = S3Router::new(console_enabled); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 799fe5d31..0aa2b5736 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -29,7 +29,9 @@ use chrono::Datelike; use clap::Parser; use license::init_license; use rustfs_ahm::scanner::data_scanner::ScannerConfig; -use rustfs_ahm::{Scanner, create_ahm_services_cancel_token, shutdown_ahm_services}; +use rustfs_ahm::{ + Scanner, create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services, +}; use rustfs_common::globals::set_global_addr; use rustfs_config::DEFAULT_DELIMITER; use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys; @@ -52,6 +54,7 @@ use rustfs_iam::init_iam_sys; use rustfs_obs::{init_obs, set_global_guard}; use rustfs_utils::net::parse_and_resolve_address; use std::io::{Error, Result}; +use std::sync::Arc; use tracing::{debug, error, info, instrument, warn}; #[cfg(all(target_os = "linux", target_env = "gnu"))] @@ -185,7 +188,12 @@ async fn run(opt: config::Opt) -> Result<()> { // init_data_scanner().await; // init_auto_heal().await; let _ = create_ahm_services_cancel_token(); - let scanner = Scanner::new(Some(ScannerConfig::default())); + + // Initialize heal manager with channel processor + let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone())); + let heal_manager = init_heal_manager(heal_storage, None).await?; + + let scanner = Scanner::new(Some(ScannerConfig::default()), Some(heal_manager)); scanner.start().await?; print_server_info(); init_bucket_replication_pool().await;