From 2ce7e01f55731a5f755053156823adf21b907f70 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 24 Jul 2025 11:15:00 +0800 Subject: [PATCH] Chore: remove dirty file(heal) Signed-off-by: junxiang Mu <1948535941@qq.com> --- .../ecstore/src/heal/background_heal_ops.rs | 512 ----- crates/ecstore/src/heal/data_scanner.rs | 1826 ----------------- .../ecstore/src/heal/data_scanner_metric.rs | 486 ----- crates/ecstore/src/heal/data_usage.rs | 221 -- crates/ecstore/src/heal/data_usage_cache.rs | 928 --------- crates/ecstore/src/heal/error.rs | 19 - crates/ecstore/src/heal/heal_commands.rs | 544 ----- crates/ecstore/src/heal/heal_ops.rs | 842 -------- crates/ecstore/src/heal/mod.rs | 23 - crates/ecstore/src/heal/mrf.rs | 183 -- crates/ecstore/src/lib.rs | 1 - 11 files changed, 5585 deletions(-) delete mode 100644 crates/ecstore/src/heal/background_heal_ops.rs delete mode 100644 crates/ecstore/src/heal/data_scanner.rs delete mode 100644 crates/ecstore/src/heal/data_scanner_metric.rs delete mode 100644 crates/ecstore/src/heal/data_usage.rs delete mode 100644 crates/ecstore/src/heal/data_usage_cache.rs delete mode 100644 crates/ecstore/src/heal/error.rs delete mode 100644 crates/ecstore/src/heal/heal_commands.rs delete mode 100644 crates/ecstore/src/heal/heal_ops.rs delete mode 100644 crates/ecstore/src/heal/mod.rs delete mode 100644 crates/ecstore/src/heal/mrf.rs diff --git a/crates/ecstore/src/heal/background_heal_ops.rs b/crates/ecstore/src/heal/background_heal_ops.rs deleted file mode 100644 index 59abc3db9..000000000 --- a/crates/ecstore/src/heal/background_heal_ops.rs +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use futures::future::join_all; -use rustfs_madmin::heal_commands::HealResultItem; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; -use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration}; -use tokio::{ - spawn, - sync::{ - RwLock, - mpsc::{self, Receiver, Sender}, - }, - time::interval, -}; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; -use uuid::Uuid; - -use super::{ - heal_commands::HealOpts, - heal_ops::{HealSequence, new_bg_heal_sequence}, -}; -use crate::error::{Error, Result}; -use crate::global::get_background_services_cancel_token; -use crate::heal::error::ERR_RETRY_HEALING; -use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode}; -use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource}; -use crate::{ - config::RUSTFS_CONFIG_PREFIX, - disk::{BUCKET_META_PREFIX, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError}, - global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP}, - heal::{ - data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, - data_usage_cache::DataUsageCache, - heal_commands::{init_healing_tracker, load_healing_tracker}, - heal_ops::NOP_HEAL, - }, - new_object_layer_fn, - store::get_disk_via_endpoint, - store_api::{BucketInfo, BucketOptions, StorageAPI}, -}; - -pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10); - -pub async fn init_auto_heal() { - info!("Initializing auto heal background task"); - - let Some(cancel_token) = get_background_services_cancel_token() else { - error!("Background services cancel token not initialized"); - return; - }; - - init_background_healing().await; - let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string()); - if v == "on" { - info!("start monitor local disks and heal"); - GLOBAL_BackgroundHealState - .push_heal_local_disks(&get_local_disks_to_heal().await) - .await; - - let cancel_clone = cancel_token.clone(); - spawn(async move { - monitor_local_disks_and_heal(cancel_clone).await; - }); - } - - // let cancel_clone = cancel_token.clone(); - // spawn(async move { - // GLOBAL_MRFState.heal_routine_with_cancel(cancel_clone).await; - // }); -} - -async fn init_background_healing() { - let bg_seq = Arc::new(new_bg_heal_sequence()); - for _ in 0..GLOBAL_BackgroundHealRoutine.workers { - let bg_seq_clone = bg_seq.clone(); - spawn(async { - GLOBAL_BackgroundHealRoutine.add_worker(bg_seq_clone).await; - }); - } - let _ = GLOBAL_BackgroundHealState.launch_new_heal_sequence(bg_seq).await; -} - -pub async fn get_local_disks_to_heal() -> Vec { - let mut disks_to_heal = Vec::new(); - for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() { - if let Some(disk) = disk { - if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { - if err == DiskError::UnformattedDisk { - info!("get_local_disks_to_heal, disk is unformatted: {}", err); - disks_to_heal.push(disk.endpoint()); - } - } - let h = disk.healing().await; - if let Some(h) = h { - if !h.finished { - info!("get_local_disks_to_heal, disk healing not finished"); - disks_to_heal.push(disk.endpoint()); - } - } - } - } - - // todo - // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n { - - // } - disks_to_heal -} - -async fn monitor_local_disks_and_heal(cancel_token: CancellationToken) { - info!("Auto heal monitor started"); - let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL); - - loop { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("Auto heal monitor received shutdown signal, exiting gracefully"); - break; - } - _ = interval.tick() => { - let heal_disks = GLOBAL_BackgroundHealState.get_heal_local_disk_endpoints().await; - if heal_disks.is_empty() { - info!("heal local disks is empty"); - interval.reset(); - continue; - } - - info!("heal local disks: {:?}", heal_disks); - - let store = new_object_layer_fn().expect("errServerNotInitialized"); - if let (_result, Some(err)) = store.heal_format(false).await.expect("heal format failed") { - error!("heal local disk format error: {}", err); - if err == Error::NoHealRequired { - } else { - info!("heal format err: {}", err.to_string()); - interval.reset(); - continue; - } - } - - let mut futures = Vec::new(); - for disk in heal_disks.into_ref().iter() { - let disk_clone = disk.clone(); - let cancel_clone = cancel_token.clone(); - futures.push(async move { - let disk_for_cancel = disk_clone.clone(); - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Disk healing task cancelled for disk: {}", disk_for_cancel); - } - _ = async { - GLOBAL_BackgroundHealState - .set_disk_healing_status(disk_clone.clone(), true) - .await; - if heal_fresh_disk(&disk_clone).await.is_err() { - info!("heal_fresh_disk is err"); - GLOBAL_BackgroundHealState - .set_disk_healing_status(disk_clone.clone(), false) - .await; - } - GLOBAL_BackgroundHealState.pop_heal_local_disks(&[disk_clone]).await; - } => {} - } - }); - } - let _ = join_all(futures).await; - interval.reset(); - } - } - } -} - -async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> { - let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.set_idx as usize); - let disk = match get_disk_via_endpoint(endpoint).await { - Some(disk) => disk, - None => { - return Err(Error::other(format!( - "Unexpected error disk must be initialized by now after formatting: {endpoint}" - ))); - } - }; - - if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { - match err { - DiskError::DriveIsRoot => { - return Ok(()); - } - DiskError::UnformattedDisk => {} - _ => { - return Err(err.into()); - } - } - } - - let mut tracker = match load_healing_tracker(&Some(disk.clone())).await { - Ok(tracker) => tracker, - Err(err) => { - match err { - DiskError::FileNotFound => { - return Ok(()); - } - _ => { - info!( - "Unable to load healing tracker on '{}': {}, re-initializing..", - disk.to_string(), - err.to_string() - ); - } - } - init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await? - } - }; - - info!( - "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.", - endpoint.to_string() - ); - - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - let mut buckets = store.list_bucket(&BucketOptions::default()).await?; - buckets.push(BucketInfo { - name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)]) - .to_string_lossy() - .to_string(), - ..Default::default() - }); - buckets.push(BucketInfo { - name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]) - .to_string_lossy() - .to_string(), - ..Default::default() - }); - - buckets.sort_by(|a, b| { - let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET); - let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET); - - match (a_has_prefix, b_has_prefix) { - (true, false) => Ordering::Less, - (false, true) => Ordering::Greater, - _ => b.created.cmp(&a.created), - } - }); - - if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await { - let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new()); - tracker.objects_total_count = data_usage_info.objects_total_count; - tracker.objects_total_size = data_usage_info.objects_total_size; - }; - - tracker.set_queue_buckets(&buckets).await; - tracker.save().await?; - - let tracker = Arc::new(RwLock::new(tracker)); - let qb = tracker.read().await.queue_buckets.clone(); - store.pools[pool_idx].disk_set[set_idx] - .clone() - .heal_erasure_set(&qb, tracker.clone()) - .await?; - let mut tracker_w = tracker.write().await; - if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 { - tracker_w.retry_attempts += 1; - tracker_w.reset_healing().await; - if let Err(err) = tracker_w.update().await { - info!("update tracker failed: {}", err.to_string()); - } - return Err(Error::other(ERR_RETRY_HEALING)); - } - - if tracker_w.items_failed > 0 { - info!( - "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).", - disk.to_string(), - tracker_w.retry_attempts, - tracker_w.items_healed, - tracker_w.item_skipped, - tracker_w.items_failed - ); - } else if tracker_w.retry_attempts > 0 { - info!( - "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).", - disk.to_string(), - tracker_w.retry_attempts, - tracker_w.items_healed, - tracker_w.item_skipped - ); - } else { - info!( - "Healing of drive '{}' is finished (healed: {}, skipped: {}).", - disk.to_string(), - tracker_w.items_healed, - tracker_w.item_skipped - ); - } - - if tracker_w.heal_id.is_empty() { - if let Err(err) = tracker_w.delete().await { - error!("delete tracker failed: {}", err.to_string()); - } - } - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let disks = store.get_disks(pool_idx, set_idx).await?; - for disk in disks.into_iter() { - if disk.is_none() { - continue; - } - let mut tracker = match load_healing_tracker(&disk).await { - Ok(tracker) => tracker, - Err(err) => { - match err { - DiskError::FileNotFound => {} - _ => { - info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string()); - } - } - continue; - } - }; - if tracker.heal_id == tracker_w.heal_id { - tracker.finished = true; - tracker.update().await?; - } - } - Ok(()) -} - -#[derive(Debug)] -pub struct HealTask { - pub bucket: String, - pub object: String, - pub version_id: String, - pub opts: HealOpts, - pub resp_tx: Option>, - pub resp_rx: Option>, -} - -impl HealTask { - pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self { - Self { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - opts: *opts, - resp_tx: None, - resp_rx: None, - } - } -} - -#[derive(Debug)] -pub struct HealResult { - pub result: HealResultItem, - pub err: Option, -} - -pub struct HealRoutine { - pub tasks_tx: Sender, - tasks_rx: RwLock>, - workers: usize, -} - -impl HealRoutine { - pub fn new() -> Arc { - let mut workers = num_cpus::get() / 2; - if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") { - if let Ok(num_healers) = env_heal_workers.parse::() { - workers = num_healers; - } - } - - if workers == 0 { - workers = 4; - } - - let (tx, rx) = mpsc::channel(100); - Arc::new(Self { - tasks_tx: tx, - tasks_rx: RwLock::new(rx), - workers, - }) - } - - pub async fn add_worker(&self, bgseq: Arc) { - loop { - let mut d_res = HealResultItem::default(); - let d_err: Option; - match self.tasks_rx.write().await.recv().await { - Some(task) => { - info!("got task: {:?}", task); - if task.bucket == NOP_HEAL { - d_err = Some(Error::other("skip file")); - } else if task.bucket == SLASH_SEPARATOR { - match heal_disk_format(task.opts).await { - Ok((res, err)) => { - d_res = res; - d_err = err; - } - Err(err) => d_err = Some(err), - } - } else { - let store = new_object_layer_fn().expect("errServerNotInitialized"); - if task.object.is_empty() { - match store.heal_bucket(&task.bucket, &task.opts).await { - Ok(res) => { - d_res = res; - d_err = None; - } - Err(err) => d_err = Some(err), - } - } else { - match store - .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts) - .await - { - Ok((res, err)) => { - d_res = res; - d_err = err; - } - Err(err) => d_err = Some(err), - } - } - } - info!("task finished, task: {:?}", task); - if let Some(resp_tx) = task.resp_tx { - let _ = resp_tx - .send(HealResult { - result: d_res, - err: d_err, - }) - .await; - } else { - // when respCh is not set caller is not waiting but we - // update the relevant metrics for them - if d_err.is_none() { - bgseq.count_healed(d_res.heal_item_type).await; - } else { - bgseq.count_failed(d_res.heal_item_type).await; - } - } - } - None => { - info!("add_worker, tasks_rx was closed, return"); - return; - } - } - } - } -} - -// pub fn active_listeners() -> Result { - -// } - -async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option)> { - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - - let (res, err) = store.heal_format(opts.dry_run).await?; - // return any error, ignore error returned when disks have - // already healed. - if err.is_some() { - return Ok((HealResultItem::default(), err)); - } - Ok((res, err)) -} - -pub(crate) async fn heal_bucket(bucket: &str) -> Result<()> { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if ok { - // bg_seq must be Some when ok is true - return bg_seq - .unwrap() - .queue_heal_task( - HealSource { - bucket: bucket.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_string(), - ) - .await; - } - Ok(()) -} - -pub(crate) async fn heal_object(bucket: &str, object: &str, version_id: &str, scan_mode: HealScanMode) -> Result<()> { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if ok { - // bg_seq must be Some when ok is true - return HealSequence::heal_object(bg_seq.unwrap(), bucket, object, version_id, scan_mode).await; - } - Ok(()) -} diff --git a/crates/ecstore/src/heal/data_scanner.rs b/crates/ecstore/src/heal/data_scanner.rs deleted file mode 100644 index 98a0924fd..000000000 --- a/crates/ecstore/src/heal/data_scanner.rs +++ /dev/null @@ -1,1826 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - collections::{HashMap, HashSet}, - fs, - future::Future, - io::{Cursor, Read}, - path::{Path, PathBuf}, - pin::Pin, - sync::{ - Arc, - atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, - }, - time::{Duration, SystemTime}, -}; - -use time::{self, OffsetDateTime}; - -use super::{ - data_usage::{DATA_USAGE_BLOOM_NAME_PATH, DataUsageInfo, store_data_usage_in_backend}, - data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash}, - heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, -}; -use crate::bucket::{ - object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion}, - utils::is_meta_bucketname, -}; -use crate::cmd::bucket_replication::queue_replication_heal; -use crate::disk::local::LocalDisk; -use crate::event::name::EventName; -use crate::{ - bucket::{ - lifecycle::{ - bucket_lifecycle_audit::LcEventSrc, - bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object}, - lifecycle::{self, ExpirationOptions, Lifecycle}, - }, - metadata_sys, - }, - event_notification::{EventArgs, send_event}, - global::{GLOBAL_LocalNodeName, get_background_services_cancel_token}, - store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, -}; -use crate::{ - bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, - cmd::bucket_replication::ReplicationStatusType, - disk, - // heal::data_usage::DATA_USAGE_ROOT, -}; -use crate::{ - cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, - config::{ - com::{read_config, save_config}, - heal::Config, - }, - disk::{DiskInfoOptions, DiskStore}, - global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasureSD}, - heal::{ - data_usage::BACKGROUND_HEAL_INFO_PATH, - data_usage_cache::{DataUsageHashMap, hash_path}, - error::ERR_IGNORE_FILE_CONTRIB, - heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}, - heal_ops::{BG_HEALING_UUID, HealSource}, - }, - new_object_layer_fn, - store::ECStore, - store_utils::is_reserved_or_invalid_bucket, -}; -use crate::{disk::DiskAPI, store_api::ObjectInfo}; -use crate::{ - disk::error::DiskError, - error::{Error, Result}, -}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use rand::Rng; -use rmp_serde::{Deserializer, Serializer}; -use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_utils::path::encode_dir_object; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{ - BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, - ReplicationRuleStatus, -}; -use serde::{Deserialize, Serialize}; -use tokio::{ - sync::{ - RwLock, broadcast, - mpsc::{self, Sender}, - }, - time::sleep, -}; -use tracing::{debug, error, info}; - -const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders. -const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles. -const DATA_SCANNER_COMPACT_LEAST_OBJECT: u64 = 500; // Compact when there are less than this many objects in a branch. -const DATA_SCANNER_COMPACT_AT_CHILDREN: u64 = 10000; // Compact when there are this many children in a branch. -const DATA_SCANNER_COMPACT_AT_FOLDERS: u64 = DATA_SCANNER_COMPACT_AT_CHILDREN / 4; // Compact when this many subfolders in a single folder. -pub const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: u64 = 250_000; // Compact when this many subfolders in a single folder (even top level). -const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(60); // Time to wait on startup and between cycles. - -pub const HEAL_DELETE_DANGLING: bool = true; -const HEAL_OBJECT_SELECT_PROB: u64 = 1024; // Overall probability of a file being scanned; one in n. - -static SCANNER_CYCLE: AtomicU64 = AtomicU64::new(DATA_SCANNER_START_DELAY.as_secs()); -static _SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle -static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100); -static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB -static SCANNER_EXCESS_FOLDERS: AtomicU64 = AtomicU64::new(50_000); - -lazy_static! { - static ref SCANNER_SLEEPER: RwLock = RwLock::new(new_dynamic_sleeper(2.0, Duration::from_secs(1), true)); - pub static ref globalHealConfig: Arc> = Arc::new(RwLock::new(Config::default())); -} - -struct DynamicSleeper { - factor: f64, - max_sleep: Duration, - min_sleep: Duration, - _is_scanner: bool, -} - -type TimerFn = Pin + Send>>; -impl DynamicSleeper { - fn timer() -> TimerFn { - let t = SystemTime::now(); - Box::pin(async move { - let done_at = SystemTime::now().duration_since(t).unwrap_or_default(); - SCANNER_SLEEPER.read().await.sleep(done_at).await; - }) - } - - async fn sleep(&self, base: Duration) { - let (min_wait, max_wait) = (self.min_sleep, self.max_sleep); - let factor = self.factor; - - let want_sleep = { - let tmp = base.mul_f64(factor); - if tmp < min_wait { - return; - } - - if max_wait > Duration::from_secs(0) && tmp > max_wait { - max_wait - } else { - tmp - } - }; - sleep(want_sleep).await; - } - - fn _update(&mut self, factor: f64, max_wait: Duration) -> Result<()> { - if (self.factor - factor).abs() < 1e-10 && self.max_sleep == max_wait { - return Ok(()); - } - - self.factor = factor; - self.max_sleep = max_wait; - - Ok(()) - } -} - -fn new_dynamic_sleeper(factor: f64, max_wait: Duration, is_scanner: bool) -> DynamicSleeper { - DynamicSleeper { - factor, - max_sleep: max_wait, - min_sleep: Duration::from_micros(100), - _is_scanner: is_scanner, - } -} - -/// Initialize and start the data scanner in the background -/// -/// This function starts a background task that continuously runs the data scanner -/// with randomized intervals between cycles to avoid resource contention. -/// -/// # Features -/// - Graceful shutdown support via cancellation token -/// - Randomized sleep intervals to prevent synchronized scanning across nodes -/// - Minimum sleep duration to avoid excessive CPU usage -/// - Proper error handling and logging -/// -/// # Architecture -/// 1. Initialize with random seed for sleep intervals -/// 2. Run scanner cycles in a loop -/// 3. Use randomized sleep between cycles to avoid thundering herd -/// 4. Ensure minimum sleep duration to prevent CPU thrashing -pub async fn init_data_scanner() { - info!("Initializing data scanner background task"); - - let Some(cancel_token) = get_background_services_cancel_token() else { - error!("Background services cancel token not initialized"); - return; - }; - - let cancel_clone = cancel_token.clone(); - tokio::spawn(async move { - info!("Data scanner background task started"); - - loop { - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Data scanner received shutdown signal, exiting gracefully"); - break; - } - _ = run_data_scanner_cycle() => { - // Calculate randomized sleep duration - let random_factor = { - let mut rng = rand::rng(); - rng.random_range(1.0..10.0) - }; - let base_cycle_duration = SCANNER_CYCLE.load(Ordering::SeqCst) as f64; - let sleep_duration_secs = random_factor * base_cycle_duration; - - let sleep_duration = Duration::from_secs_f64(sleep_duration_secs); - - debug!( - duration_secs = sleep_duration.as_secs(), - "Data scanner sleeping before next cycle" - ); - - // Interruptible sleep - tokio::select! { - _ = cancel_clone.cancelled() => { - info!("Data scanner received shutdown signal during sleep, exiting"); - break; - } - _ = sleep(sleep_duration) => { - // Continue to next cycle - } - } - } - } - } - - info!("Data scanner background task stopped gracefully"); - }); -} - -/// Run a single data scanner cycle -/// -/// This function performs one complete scan cycle, including: -/// - Loading and updating cycle information -/// - Determining scan mode based on healing configuration -/// - Running the namespace scanner -/// - Saving cycle completion state -/// -/// # Error Handling -/// - Gracefully handles missing object layer -/// - Continues operation even if individual steps fail -/// - Logs errors appropriately without terminating the scanner -async fn run_data_scanner_cycle() { - debug!("Starting data scanner cycle"); - - // Get the object layer, return early if not available - let Some(store) = new_object_layer_fn() else { - error!("Object layer not initialized, skipping scanner cycle"); - return; - }; - - // Check for cancellation before starting expensive operations - if let Some(token) = get_background_services_cancel_token() { - if token.is_cancelled() { - debug!("Scanner cancelled before starting cycle"); - return; - } - } - - // Load current cycle information from persistent storage - let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .unwrap_or_else(|err| { - info!(error = %err, "Failed to read cycle info, starting fresh"); - Vec::new() - }); - - let mut cycle_info = if buf.is_empty() { - CurrentScannerCycle::default() - } else { - let mut buf_cursor = Deserializer::new(Cursor::new(buf)); - Deserialize::deserialize(&mut buf_cursor).unwrap_or_else(|err| { - error!(error = %err, "Failed to deserialize cycle info, using default"); - CurrentScannerCycle::default() - }) - }; - - // Start metrics collection for this cycle - // let stop_fn = ScannerMetrics::log(ScannerMetric::ScanCycle); - - // Update cycle information - cycle_info.current = cycle_info.next; - cycle_info.started = Utc::now(); - - // Update global scanner metrics - // globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await; - - // Read background healing information and determine scan mode - let bg_heal_info = read_background_heal_info(store.clone()).await; - let scan_mode = - get_cycle_scan_mode(cycle_info.current, bg_heal_info.bitrot_start_cycle, bg_heal_info.bitrot_start_time).await; - - // Update healing info if scan mode changed - if bg_heal_info.current_scan_mode != scan_mode { - let mut new_heal_info = bg_heal_info; - new_heal_info.current_scan_mode = scan_mode; - if scan_mode == HEAL_DEEP_SCAN { - new_heal_info.bitrot_start_time = SystemTime::now(); - new_heal_info.bitrot_start_cycle = cycle_info.current; - } - save_background_heal_info(store.clone(), &new_heal_info).await; - } - - // Set up data usage storage channel - let (tx, rx) = mpsc::channel::(100); - tokio::spawn(async move { - let _ = store_data_usage_in_backend(rx).await; - }); - - // Prepare result tracking - let mut scan_result = HashMap::new(); - scan_result.insert("cycle".to_string(), cycle_info.current.to_string()); - - info!( - cycle = cycle_info.current, - scan_mode = ?scan_mode, - "Starting namespace scanner" - ); - - // Run the namespace scanner with cancellation support - match execute_namespace_scan(&store, tx, cycle_info.current, scan_mode).await { - Ok(_) => { - info!(cycle = cycle_info.current, "Namespace scanner completed successfully"); - - // Update cycle completion information - cycle_info.next += 1; - cycle_info.current = 0; - cycle_info.cycle_completed.push(Utc::now()); - - // Maintain cycle completion history (keep only recent cycles) - if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize { - let _ = cycle_info.cycle_completed.remove(0); - } - - // Update global metrics with completion info - // globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await; - - // Persist updated cycle information - // ignore error, continue. - let mut serialized_data = Vec::new(); - if let Err(err) = cycle_info.serialize(&mut Serializer::new(&mut serialized_data)) { - error!(error = %err, "Failed to serialize cycle info"); - } else if let Err(err) = save_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH, serialized_data).await { - error!(error = %err, "Failed to save cycle info to storage"); - } - } - Err(err) => { - error!( - cycle = cycle_info.current, - error = %err, - "Namespace scanner failed" - ); - scan_result.insert("error".to_string(), err.to_string()); - } - } - - // Complete metrics collection for this cycle - // stop_fn(&scan_result); -} - -/// Execute namespace scan with cancellation support -async fn execute_namespace_scan( - store: &Arc, - tx: Sender, - cycle: u64, - scan_mode: HealScanMode, -) -> Result<()> { - let cancel_token = - get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?; - - tokio::select! { - result = store.ns_scanner(tx, cycle as usize, scan_mode) => { - result.map_err(|e| Error::other(format!("Namespace scan failed: {e}"))) - } - _ = cancel_token.cancelled() => { - info!("Namespace scan cancelled"); - Err(Error::other("Scan cancelled")) - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct BackgroundHealInfo { - bitrot_start_time: SystemTime, - bitrot_start_cycle: u64, - current_scan_mode: HealScanMode, -} - -impl Default for BackgroundHealInfo { - fn default() -> Self { - Self { - bitrot_start_time: SystemTime::now(), - bitrot_start_cycle: Default::default(), - current_scan_mode: Default::default(), - } - } -} - -async fn read_background_heal_info(store: Arc) -> BackgroundHealInfo { - if *GLOBAL_IsErasureSD.read().await { - return BackgroundHealInfo::default(); - } - - let buf = read_config(store, &BACKGROUND_HEAL_INFO_PATH) - .await - .map_or(Vec::new(), |buf| buf); - if buf.is_empty() { - return BackgroundHealInfo::default(); - } - serde_json::from_slice::(&buf).map_or(BackgroundHealInfo::default(), |b| b) -} - -async fn save_background_heal_info(store: Arc, info: &BackgroundHealInfo) { - if *GLOBAL_IsErasureSD.read().await { - return; - } - let b = match serde_json::to_vec(info) { - Ok(info) => info, - Err(_) => return, - }; - let _ = save_config(store, &BACKGROUND_HEAL_INFO_PATH, b).await; -} - -async fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: SystemTime) -> HealScanMode { - let bitrot_cycle = globalHealConfig.read().await.bitrot_scan_cycle(); - let v = bitrot_cycle.as_secs_f64(); - if v == -1.0 { - return HEAL_NORMAL_SCAN; - } else if v == 0.0 { - return HEAL_DEEP_SCAN; - } - - if current_cycle - bitrot_start_cycle < HEAL_OBJECT_SELECT_PROB { - return HEAL_DEEP_SCAN; - } - - if SystemTime::now().duration_since(bitrot_start_time).unwrap_or_default() > bitrot_cycle { - return HEAL_DEEP_SCAN; - } - - HEAL_NORMAL_SCAN -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CurrentScannerCycle { - pub current: u64, - pub next: u64, - pub started: DateTime, - pub cycle_completed: Vec>, -} - -impl Default for CurrentScannerCycle { - fn default() -> Self { - Self { - current: Default::default(), - next: Default::default(), - started: Utc::now(), - cycle_completed: Default::default(), - } - } -} - -impl CurrentScannerCycle { - pub fn marshal_msg(&self, next_buf: &[u8]) -> Result> { - let len: u32 = 4; - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // write "current" - rmp::encode::write_str(&mut wr, "current")?; - rmp::encode::write_uint(&mut wr, self.current)?; - - // write "next" - rmp::encode::write_str(&mut wr, "next")?; - rmp::encode::write_uint(&mut wr, self.next)?; - - // write "started" - rmp::encode::write_str(&mut wr, "started")?; - rmp::encode::write_sint(&mut wr, system_time_to_timestamp(&self.started))?; - - // write "cycle_completed" - rmp::encode::write_str(&mut wr, "cycle_completed")?; - let mut buf = Vec::new(); - self.cycle_completed - .serialize(&mut Serializer::new(&mut buf)) - .expect("Serialization failed"); - rmp::encode::write_bin(&mut wr, &buf)?; - let mut result = next_buf.to_vec(); - result.extend(wr.iter()); - Ok(result) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - match field.as_str() { - "current" => { - let u: u64 = rmp::decode::read_int(&mut cur)?; - self.current = u; - } - - // "next" => { - // let u: u64 = rmp::decode::read_int(&mut cur)?; - // self.next = u; - // } - "started" => { - let u: i64 = rmp::decode::read_int(&mut cur)?; - let started = timestamp_to_system_time(u); - self.started = started; - } - "cycleCompleted" => { - let mut buf = Vec::new(); - let _ = cur.read_to_end(&mut buf)?; - let u: Vec> = - Deserialize::deserialize(&mut Deserializer::new(&buf[..])).expect("Deserialization failed"); - self.cycle_completed = u; - } - name => return Err(Error::other(format!("not support field name {name}"))), - } - } - - Ok(cur.position()) - } -} - -// Convert `SystemTime` to timestamp -fn system_time_to_timestamp(time: &DateTime) -> i64 { - time.timestamp_micros() -} - -// Convert timestamp to `SystemTime` -fn timestamp_to_system_time(timestamp: i64) -> DateTime { - DateTime::from_timestamp_micros(timestamp).unwrap_or_default() -} - -#[derive(Clone, Debug, Default)] -pub struct Heal { - enabled: bool, - bitrot: bool, -} - -#[derive(Clone)] -pub struct ScannerItem { - pub path: String, - pub bucket: String, - pub prefix: String, - pub object_name: String, - pub replication: Option, - pub lifecycle: Option, - // typ: fs::Permissions, - pub heal: Heal, - pub debug: bool, -} - -impl ScannerItem { - pub fn transform_meta_dir(&mut self) { - let split = self.prefix.split(SLASH_SEPARATOR).map(PathBuf::from).collect::>(); - if split.len() > 1 { - self.prefix = path_join(&split[0..split.len() - 1]).to_string_lossy().to_string(); - } else { - self.prefix = "".to_string(); - } - self.object_name = split.last().map_or("".to_string(), |v| v.to_string_lossy().to_string()); - } - - pub fn object_path(&self) -> PathBuf { - path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())]) - } - - async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) { - let mut size = oi.get_actual_size().expect("err!"); - if self.debug { - info!("apply_lifecycle debug"); - } - if self.lifecycle.is_none() { - return (lifecycle::IlmAction::NoneAction, size); - } - - let version_id = oi.version_id; - - let mut vc = None; - let mut lr = None; - let mut rcfg = None; - if !is_meta_bucketname(&self.bucket) { - vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap()); - lr = BucketObjectLockSys::get(&self.bucket).await; - rcfg = (metadata_sys::get_replication_config(&self.bucket).await).ok(); - } - - let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; - if self.debug { - if version_id.is_some() { - info!( - "lifecycle: {} (version-id={}), Initial scan: {}", - self.object_path().to_string_lossy().to_string(), - version_id.expect("err"), - lc_evt.action - ); - } else { - info!( - "lifecycle: {} Initial scan: {}", - self.object_path().to_string_lossy().to_string(), - lc_evt.action - ); - } - } - - match lc_evt.action { - lifecycle::IlmAction::DeleteVersionAction - | lifecycle::IlmAction::DeleteAllVersionsAction - | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - size = 0; - } - lifecycle::IlmAction::DeleteAction => { - if !vc.unwrap().prefix_enabled(&oi.name) { - size = 0 - } - } - _ => (), - } - - apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; - (lc_evt.action, size) - } - - pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result> { - let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?; - if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize { - // todo - } - - let mut cumulative_size = 0; - for obj_info in obj_infos.iter() { - cumulative_size += obj_info.size; - } - - if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst) as i64 { - //todo - } - - Ok(obj_infos) - } - - pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result> { - // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent); - - let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { - rcfg.mode.is_some() - } else { - false - }; - let _vcfg = BucketVersioningSys::get(&self.bucket).await?; - - let versioned = match BucketVersioningSys::get(&self.bucket).await { - Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()), - Err(_) => false, - }; - let mut object_infos = Vec::with_capacity(fivs.len()); - - if self.lifecycle.is_none() { - for info in fivs.iter() { - object_infos.push(ObjectInfo::from_file_info( - info, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - return Ok(object_infos); - } - - let event = self - .lifecycle - .as_ref() - .expect("lifecycle err.") - .noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { - name: self.object_path().to_string_lossy().to_string(), - ..Default::default() - }) - .await; - let lim = event.newer_noncurrent_versions; - if lim == 0 || fivs.len() <= lim + 1 { - for fi in fivs.iter() { - object_infos.push(ObjectInfo::from_file_info( - fi, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - return Ok(object_infos); - } - - let overflow_versions = &fivs[lim + 1..]; - for fi in fivs[..lim + 1].iter() { - object_infos.push(ObjectInfo::from_file_info( - fi, - &self.bucket, - &self.object_path().to_string_lossy(), - versioned, - )); - } - - let mut to_del = Vec::::with_capacity(overflow_versions.len()); - for fi in overflow_versions.iter() { - let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); - if lock_enabled && enforce_retention_for_deletion(&obj) { - //if enforce_retention_for_deletion(&obj) { - if self.debug { - if obj.version_id.is_some() { - info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); - } else { - info!("lifecycle: {} is locked, not deleting\n", obj.name); - } - } - object_infos.push(obj); - continue; - } - - if OffsetDateTime::now_utc().unix_timestamp() - < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32) - .unix_timestamp() - { - object_infos.push(obj); - continue; - } - - to_del.push(ObjectToDelete { - object_name: obj.name, - version_id: obj.version_id, - }); - } - - if !to_del.is_empty() { - let mut expiry_state = GLOBAL_ExpiryState.write().await; - expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await; - } - // done().await; - - Ok(object_infos) - } - - pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, i64) { - // let done = ScannerMetrics::time(ScannerMetric::Ilm); - - let (action, _size) = self.apply_lifecycle(oi).await; - - info!( - "apply_actions {} {} {:?} {:?}", - oi.bucket.clone(), - oi.name.clone(), - oi.version_id.clone(), - oi.user_defined.clone() - ); - - // Create a mutable clone if you need to modify fields - let mut oi = oi.clone(); - - let versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; - if versioned { - oi.replication_status = ReplicationStatusType::from( - oi.user_defined - .get("x-amz-bucket-replication-status") - .unwrap_or(&"PENDING".to_string()), - ); - debug!("apply status is: {:?}", oi.replication_status); - self.heal_replication(&oi, _size_s).await; - } - - // done(); - - if action.delete_all() { - return (true, 0); - } - - (false, oi.size) - } - - pub async fn heal_replication(&mut self, oi: &ObjectInfo, size_s: &mut SizeSummary) { - if oi.version_id.is_none() { - error!( - "heal_replication: no version_id or replication config {} {} {}", - oi.bucket, - oi.name, - oi.version_id.is_none() - ); - return; - } - - //let config = s3s::dto::ReplicationConfiguration{ role: todo!(), rules: todo!() }; - // Use the provided variable instead of borrowing self mutably. - let replication = match metadata_sys::get_replication_config(&oi.bucket).await { - Ok((replication, _)) => replication, - Err(_) => { - error!( - "heal_replication: failed to get replication config for bucket: {} and object name: {}", - oi.bucket, oi.name - ); - return; - } - }; - if replication.rules.is_empty() { - error!("heal_replication: no replication rules for bucket {} {}", oi.bucket, oi.name); - return; - } - if replication.role.is_empty() { - // error!("heal_replication: no replication role for bucket {} {}", oi.bucket, oi.name); - // return; - } - - //if oi.delete_marker || !oi.version_purge_status.is_empty() { - if oi.delete_marker { - error!( - "heal_replication: delete marker or version purge status {} {} {:?} {} {:?}", - oi.bucket, oi.name, oi.version_id, oi.delete_marker, oi.version_purge_status - ); - return; - } - - if oi.replication_status == ReplicationStatusType::Completed { - return; - } - - info!("replication status is: {:?} and user define {:?}", oi.replication_status, oi.user_defined); - - let roi = queue_replication_heal(&oi.bucket, oi, &replication, 3).await; - - if roi.is_none() { - info!("not need heal {} {} {:?}", oi.bucket, oi.name, oi.version_id); - return; - } - - for (arn, tgt_status) in &roi.unwrap().target_statuses { - let tgt_size_s = size_s.repl_target_stats.entry(arn.clone()).or_default(); - - match tgt_status { - ReplicationStatusType::Pending => { - tgt_size_s.pending_count += 1; - tgt_size_s.pending_size += oi.size as usize; - size_s.pending_count += 1; - size_s.pending_size += oi.size as usize; - } - ReplicationStatusType::Failed => { - tgt_size_s.failed_count += 1; - tgt_size_s.failed_size += oi.size as usize; - size_s.failed_count += 1; - size_s.failed_size += oi.size as usize; - } - ReplicationStatusType::Completed | ReplicationStatusType::CompletedLegacy => { - tgt_size_s.replicated_count += 1; - tgt_size_s.replicated_size += oi.size as usize; - size_s.replicated_count += 1; - size_s.replicated_size += oi.size as usize; - } - _ => {} - } - } - - if matches!(oi.replication_status, ReplicationStatusType::Replica) { - size_s.replica_count += 1; - size_s.replica_size += oi.size as usize; - } - } -} - -#[derive(Debug, Default)] -pub struct SizeSummary { - pub total_size: usize, - pub versions: usize, - pub delete_markers: usize, - pub replicated_size: usize, - pub replicated_count: usize, - pub pending_size: usize, - pub failed_size: usize, - pub replica_size: usize, - pub replica_count: usize, - pub pending_count: usize, - pub failed_count: usize, - pub repl_target_stats: HashMap, - // Todo: tires -} - -#[derive(Debug, Default)] -pub struct ReplTargetSizeSummary { - pub replicated_size: usize, - pub replicated_count: usize, - pub pending_size: usize, - pub failed_size: usize, - pub pending_count: usize, - pub failed_count: usize, -} - -#[derive(Debug, Clone)] -struct CachedFolder { - name: String, - parent: DataUsageHash, - object_heal_prob_div: u32, -} - -pub type GetSizeFn = - Box Pin> + Send>> + Send + Sync + 'static>; -pub type UpdateCurrentPathFn = Arc Pin + Send>> + Send + Sync + 'static>; -pub type ShouldSleepFn = Option bool + Send + Sync + 'static>>; - -struct FolderScanner { - root: String, - get_size: GetSizeFn, - old_cache: DataUsageCache, - new_cache: DataUsageCache, - update_cache: DataUsageCache, - data_usage_scanner_debug: bool, - heal_object_select: u32, - scan_mode: HealScanMode, - disks: Vec>, - disks_quorum: usize, - updates: Sender, - last_update: SystemTime, - update_current_path: UpdateCurrentPathFn, - skip_heal: AtomicBool, - drive: LocalDrive, - we_sleep: ShouldSleepFn, -} - -impl FolderScanner { - async fn should_heal(&self) -> bool { - if self.skip_heal.load(Ordering::SeqCst) { - return false; - } - if self.heal_object_select == 0 { - return false; - } - if let Ok(info) = self.drive.disk_info(&DiskInfoOptions::default()).await { - if info.healing { - self.skip_heal.store(true, Ordering::SeqCst); - return false; - } - } - true - } - - #[tracing::instrument(level = "info", skip_all)] - async fn scan_folder(&mut self, folder: &CachedFolder, into: &mut DataUsageEntry) -> Result<()> { - let this_hash = hash_path(&folder.name); - let was_compacted = into.compacted; - - 'outer: { - let mut abandoned_children: DataUsageHashMap = if !into.compacted { - self.old_cache.find_children_copy(this_hash.clone()) - } else { - HashSet::new() - }; - - let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name); - // Todo: lifeCycle - let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() { - if lc_has_active_rules(lc, &prefix) { - self.old_cache.info.lifecycle.clone() - } else { - None - } - } else { - None - }; - - let replication_cfg = if self.old_cache.info.replication.is_some() - && rep_has_active_rules(self.old_cache.info.replication.as_ref().unwrap(), &prefix, true) - { - self.old_cache.info.replication.clone() - } else { - None - }; - - if let Some(should_sleep) = &self.we_sleep { - if should_sleep() { - SCANNER_SLEEPER.read().await.sleep(DATA_SCANNER_SLEEP_PER_FOLDER).await; - } - } - - let mut existing_folders = Vec::new(); - let mut new_folders = Vec::new(); - let mut found_objects: bool = false; - - let path = Path::new(&self.root).join(&folder.name); - if path.is_dir() { - for entry in fs::read_dir(path)? { - let entry = entry?; - let sub_path = entry.path(); - let ent_name = Path::new(&folder.name).join(&sub_path); - let (bucket, prefix) = path_to_bucket_object_with_base_path(&self.root, ent_name.to_str().unwrap()); - if bucket.is_empty() { - continue; - } - if is_reserved_or_invalid_bucket(&bucket, false) { - continue; - } - - if sub_path.is_dir() { - let h = hash_path(ent_name.to_str().unwrap()); - if h == this_hash { - continue; - } - let this = CachedFolder { - name: ent_name.to_string_lossy().to_string(), - parent: this_hash.clone(), - object_heal_prob_div: folder.object_heal_prob_div, - }; - abandoned_children.remove(&h.key()); - if self.old_cache.cache.contains_key(&h.key()) { - existing_folders.push(this); - self.update_cache - .copy_with_children(&self.old_cache, &h, &Some(this_hash.clone())); - } else { - new_folders.push(this); - } - continue; - } - - let _wait = if let Some(should_sleep) = &self.we_sleep { - if should_sleep() { - DynamicSleeper::timer() - } else { - Box::pin(async {}) - } - } else { - Box::pin(async {}) - }; - - let mut item = ScannerItem { - path: Path::new(&self.root).join(&ent_name).to_string_lossy().to_string(), - bucket, - prefix: Path::new(&prefix) - .parent() - .unwrap_or(Path::new("")) - .to_string_lossy() - .to_string(), - object_name: ent_name - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(), - debug: self.data_usage_scanner_debug, - replication: replication_cfg.clone(), - lifecycle: active_life_cycle.clone(), - heal: Heal::default(), - }; - - item.heal.enabled = this_hash.mod_alt( - self.old_cache.info.next_cycle / folder.object_heal_prob_div, - self.heal_object_select / folder.object_heal_prob_div, - ) && self.should_heal().await; - item.heal.bitrot = self.scan_mode == HEAL_DEEP_SCAN; - - let (sz, err) = match (self.get_size)(&item).await { - Ok(sz) => (sz, None), - Err(err) => { - if err.to_string() != ERR_IGNORE_FILE_CONTRIB { - continue; - } - (SizeSummary::default(), Some(err)) - } - }; - // successfully read means we have a valid object. - found_objects = true; - // Remove filename i.e is the meta file to construct object name - item.transform_meta_dir(); - // Object already accounted for, remove from heal map, - // simply because getSize() function already heals the - // object. - abandoned_children.remove( - &path_join(&[PathBuf::from(item.bucket.clone()), item.object_path()]) - .to_string_lossy() - .to_string(), - ); - - if err.is_none() || err.unwrap().to_string() != ERR_IGNORE_FILE_CONTRIB { - into.add_sizes(&sz); - into.objects += 1; - } - } - } - // if found_objects && *GLOBAL_IsErasure.read().await { - if found_objects { - break 'outer; - } - - let should_compact = self.new_cache.info.name != folder.name - && (existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS as usize - || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS as usize); - - let total_folders = existing_folders.len() + new_folders.len(); - if total_folders > SCANNER_EXCESS_FOLDERS.load(Ordering::SeqCst) as usize { - let _prefix_name = format!("{}/", folder.name.trim_end_matches('/')); - // todo: notification - } - - if !into.compacted && should_compact { - into.compacted = true; - new_folders.extend(existing_folders.clone()); - existing_folders.clear(); - } - - // Transfer existing - if !into.compacted { - for folder in existing_folders.iter() { - let h = hash_path(&folder.name); - self.update_cache - .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); - } - } - - // Scan new... - for folder in new_folders.iter() { - let h = hash_path(&folder.name); - if !into.compacted { - let mut found_any = false; - let mut parent = this_hash.clone(); - while parent != hash_path(&self.update_cache.info.name) { - let e = self.update_cache.find(&parent.key()); - if e.is_none() || e.as_ref().unwrap().compacted { - found_any = true; - break; - } - match self.update_cache.search_parent(&parent) { - Some(next) => { - parent = next; - } - None => { - found_any = true; - break; - } - } - } - if !found_any { - self.update_cache - .replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default()); - } - } - (self.update_current_path)(&folder.name).await; - scan(folder, into, self).await; - // Add new folders if this is new and we don't have existing. - if !into.compacted { - if let Some(parent) = self.update_cache.find(&this_hash.key()) { - if !parent.compacted { - self.update_cache.delete_recursive(&h); - self.update_cache - .copy_with_children(&self.new_cache, &h, &Some(this_hash.clone())); - } - } - } - } - - // Scan existing... - for folder in existing_folders.iter() { - let h = hash_path(&folder.name); - if !into.compacted - && self.old_cache.is_compacted(&h) - && !h.mod_(self.old_cache.info.next_cycle, DATA_USAGE_UPDATE_DIR_CYCLES) - { - self.new_cache - .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); - into.add_child(&h); - continue; - } - (self.update_current_path)(&folder.name).await; - scan(folder, into, self).await; - } - - // Scan for healing - if abandoned_children.is_empty() || !self.should_heal().await { - break 'outer; - } - - if self.disks.is_empty() || self.disks_quorum == 0 { - break 'outer; - } - - let (bg_seq, found) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if !found { - break 'outer; - } - let bg_seq = bg_seq.unwrap(); - - let mut resolver = MetadataResolutionParams { - dir_quorum: self.disks_quorum, - obj_quorum: self.disks_quorum, - bucket: "".to_string(), - strict: false, - ..Default::default() - }; - - for k in abandoned_children.iter() { - if !self.should_heal().await { - break; - } - - let (bucket, prefix) = path_to_bucket_object(k); - (self.update_current_path)(k).await; - - if bucket != resolver.bucket { - bg_seq - .clone() - .queue_heal_task( - HealSource { - bucket: bucket.clone(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_owned(), - ) - .await?; - } - - resolver.bucket = bucket.clone(); - let found_objs = Arc::new(RwLock::new(false)); - let found_objs_clone = found_objs.clone(); - let (tx, _rx) = broadcast::channel(1); - // let tx_partial = tx.clone(); - let tx_finished = tx.clone(); - let update_current_path_agreed = self.update_current_path.clone(); - let update_current_path_partial = self.update_current_path.clone(); - let resolver_clone = resolver.clone(); - let bg_seq_clone = bg_seq.clone(); - let lopts = ListPathRawOptions { - disks: self.disks.clone(), - bucket: bucket.clone(), - path: prefix.clone(), - recursive: true, - report_not_found: true, - min_disks: self.disks_quorum, - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - Box::pin({ - let update_current_path_agreed = update_current_path_agreed.clone(); - async move { - update_current_path_agreed(&entry.name).await; - } - }) - })), - partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - Box::pin({ - let update_current_path_partial = update_current_path_partial.clone(); - // let tx_partial = tx_partial.clone(); - let resolver_partial = resolver_clone.clone(); - let bucket_partial = bucket.clone(); - let found_objs_clone = found_objs_clone.clone(); - let bg_seq_partial = bg_seq_clone.clone(); - async move { - // Todo - // if !fs.should_heal().await { - // let _ = tx_partial.send(true); - // return; - // } - let entry = match entries.resolve(resolver_partial) { - Some(entry) => entry, - _ => match entries.first_found() { - (Some(entry), _) => entry, - _ => return, - }, - }; - - update_current_path_partial(&entry.name).await; - let mut custom = HashMap::new(); - if entry.is_dir() { - return; - } - - // We got an entry which we should be able to heal. - let fiv = match entry.file_info_versions(&bucket_partial) { - Ok(fiv) => fiv, - Err(_) => { - if let Err(err) = bg_seq_partial - .queue_heal_task( - HealSource { - bucket: bucket_partial.clone(), - object: entry.name.clone(), - version_id: "".to_string(), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await - { - match err { - Error::FileNotFound | Error::FileVersionNotFound => {} - _ => { - info!("{}", err.to_string()); - } - } - } else { - let mut w = found_objs_clone.write().await; - *w = true; - } - return; - } - }; - - custom.insert("versions", fiv.versions.len().to_string()); - let (mut success_versions, mut fail_versions) = (0, 0); - for ver in fiv.versions.iter() { - match bg_seq_partial - .queue_heal_task( - HealSource { - bucket: bucket_partial.clone(), - object: fiv.name.clone(), - version_id: ver.version_id.map_or("".to_string(), |ver_id| ver_id.to_string()), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await - { - Ok(_) => { - success_versions += 1; - - let mut w = found_objs_clone.write().await; - *w = true; - } - Err(_) => { - fail_versions += 1; - } - } - } - custom.insert("success_versions", success_versions.to_string()); - custom.insert("failed_versions", fail_versions.to_string()); - } - }) - })), - finished: Some(Box::new(move |_: &[Option]| { - Box::pin({ - let tx_finished = tx_finished.clone(); - async move { - let _ = tx_finished.send(true); - } - }) - })), - ..Default::default() - }; - let _ = list_path_raw(lopts).await; - - if *found_objs.read().await { - let this: CachedFolder = CachedFolder { - name: k.clone(), - parent: this_hash.clone(), - object_heal_prob_div: 1, - }; - scan(&this, into, self).await; - } - } - } - if !was_compacted { - self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), into); - } - - if !into.compacted && self.new_cache.info.name != folder.name { - let mut flat = self.new_cache.size_recursive(&this_hash.key()).unwrap_or_default(); - flat.compacted = true; - let compact = if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT as usize { - true - } else { - // Compact if we only have objects as children... - let mut compact = true; - for k in into.children.iter() { - if let Some(v) = self.new_cache.cache.get(k) { - if !v.children.is_empty() || v.objects > 1 { - compact = false; - break; - } - } - } - compact - }; - if compact { - self.new_cache.delete_recursive(&this_hash); - self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); - let mut total: HashMap = HashMap::new(); - total.insert("objects".to_string(), flat.objects.to_string()); - total.insert("size".to_string(), flat.size.to_string()); - if flat.versions > 0 { - total.insert("versions".to_string(), flat.versions.to_string()); - } - } - } - // Compact if too many children... - if !into.compacted { - self.new_cache.reduce_children_of( - &this_hash, - DATA_SCANNER_COMPACT_AT_CHILDREN as usize, - self.new_cache.info.name != folder.name, - ); - } - if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted { - // Replace if existed before. - if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) { - self.update_cache.delete_recursive(&this_hash); - self.update_cache - .replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); - } - } - Ok(()) - } - - #[tracing::instrument(level = "info", skip_all)] - async fn send_update(&mut self) { - if SystemTime::now().duration_since(self.last_update).unwrap() < Duration::from_secs(60) { - return; - } - if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name) { - let _ = self.updates.send(flat).await; - self.last_update = SystemTime::now(); - } - } -} - -#[tracing::instrument(level = "info", skip(into, folder_scanner))] -async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) { - let mut dst = if !into.compacted { - DataUsageEntry::default() - } else { - into.clone() - }; - - if Box::pin(folder_scanner.scan_folder(folder, &mut dst)).await.is_err() { - return; - } - if !into.compacted { - let h = DataUsageHash(folder.name.clone()); - into.add_child(&h); - folder_scanner.update_cache.delete_recursive(&h); - folder_scanner - .update_cache - .copy_with_children(&folder_scanner.new_cache, &h, &Some(folder.parent.clone())); - folder_scanner.send_update().await; - } -} - -fn lc_get_prefix(rule: &LifecycleRule) -> String { - if let Some(p) = &rule.prefix { - return p.to_string(); - } else if let Some(filter) = &rule.filter { - if let Some(p) = &filter.prefix { - return p.to_string(); - } else if let Some(and) = &filter.and { - if let Some(p) = &and.prefix { - return p.to_string(); - } - } - } - - "".into() -} - -pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool { - if config.rules.is_empty() { - return false; - } - - for rule in config.rules.iter() { - if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) { - continue; - } - let rule_prefix = lc_get_prefix(rule); - if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix) - { - continue; - } - - if let Some(e) = &rule.noncurrent_version_expiration { - if let Some(true) = e.noncurrent_days.map(|d| d > 0) { - return true; - } - if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) { - return true; - } - } - - if rule.noncurrent_version_transitions.is_some() { - return true; - } - if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) { - return true; - } - - if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) { - return true; - } - - if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) { - return true; - } - - if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) { - return true; - } - - if rule.transitions.is_some() { - return true; - } - } - false -} - -pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool { - if config.rules.is_empty() { - return false; - } - - for rule in config.rules.iter() { - if rule - .status - .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)) - { - continue; - } - if !prefix.is_empty() { - if let Some(filter) = &rule.filter { - if let Some(r_prefix) = &filter.prefix { - if !r_prefix.is_empty() { - // incoming prefix must be in rule prefix - if !recursive && !prefix.starts_with(r_prefix) { - continue; - } - // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix - // does not match - if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) { - continue; - } - } - } - } - } - return true; - } - false -} - -pub type LocalDrive = Arc; -pub async fn scan_data_folder( - _disks: &[Option], - _drive: LocalDrive, - _cache: &DataUsageCache, - _get_size_fn: GetSizeFn, - _heal_scan_mode: HealScanMode, - _should_sleep: ShouldSleepFn, -) -> disk::error::Result { - // if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT { - // return Err(DiskError::other("internal error: root scan attempted")); - // } - - // let base_path = drive.to_string(); - // // let (update_path, close_disk) = current_path_updater(&base_path, &cache.info.name); - // let skip_heal = if *GLOBAL_IsErasure.read().await || cache.info.skip_healing { - // AtomicBool::new(true) - // } else { - // AtomicBool::new(false) - // }; - // let mut s = FolderScanner { - // root: base_path, - // get_size: get_size_fn, - // old_cache: cache.clone(), - // new_cache: DataUsageCache { - // info: cache.info.clone(), - // ..Default::default() - // }, - // update_cache: DataUsageCache { - // info: cache.info.clone(), - // ..Default::default() - // }, - // data_usage_scanner_debug: false, - // heal_object_select: 0, - // scan_mode: heal_scan_mode, - // updates: cache.info.updates.clone().unwrap(), - // last_update: SystemTime::now(), - // update_current_path: update_path, - // disks: disks.to_vec(), - // disks_quorum: disks.len() / 2, - // skip_heal, - // drive: drive.clone(), - // we_sleep: should_sleep, - // }; - - // if *GLOBAL_IsErasure.read().await || !cache.info.skip_healing { - // s.heal_object_select = HEAL_OBJECT_SELECT_PROB as u32; - // } - - // let mut root = DataUsageEntry::default(); - // let folder = CachedFolder { - // name: cache.info.name.clone(), - // object_heal_prob_div: 1, - // parent: DataUsageHash("".to_string()), - // }; - - // if s.scan_folder(&folder, &mut root).await.is_err() { - // close_disk().await; - // } - // s.new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN as usize); - // s.new_cache.info.last_update = Some(SystemTime::now()); - // s.new_cache.info.next_cycle = cache.info.next_cycle; - // close_disk().await; - // Ok(s.new_cache) - todo!() -} - -pub async fn eval_action_from_lifecycle( - lc: &BucketLifecycleConfiguration, - lr: Option, - rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, - oi: &ObjectInfo, -) -> lifecycle::Event { - let event = lc.eval(&oi.to_lifecycle_opts()).await; - //if serverDebugLog { - info!("lifecycle: Secondary scan: {}", event.action); - //} - - let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; - - match event.action { - lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - if lock_enabled { - return lifecycle::Event::default(); - } - } - lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => { - if oi.version_id.is_none() { - return lifecycle::Event::default(); - } - if lock_enabled && enforce_retention_for_deletion(oi) { - //if serverDebugLog { - if oi.version_id.is_some() { - info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); - } else { - info!("lifecycle: {} is locked, not deleting", oi.name); - } - //} - return lifecycle::Event::default(); - } - if let Some(rcfg) = rcfg { - if rep_has_active_rules(&rcfg.0, &oi.name, true) { - return lifecycle::Event::default(); - } - } - } - _ => (), - } - - event -} - -async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - if oi.delete_marker || oi.is_dir { - return false; - } - GLOBAL_TransitionState.queue_transition_task(oi, event, src).await; - true -} - -pub async fn apply_expiry_on_transitioned_object( - api: Arc, - oi: &ObjectInfo, - lc_event: &lifecycle::Event, - src: &LcEventSrc, -) -> bool { - // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); - if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { - return false; - } - // let _ = time_ilm(1); - - true -} - -pub async fn apply_expiry_on_non_transitioned_objects( - api: Arc, - oi: &ObjectInfo, - lc_event: &lifecycle::Event, - _src: &LcEventSrc, -) -> bool { - let mut opts = ObjectOptions { - expiration: ExpirationOptions { expire: true }, - ..Default::default() - }; - - if lc_event.action.delete_versioned() { - opts.version_id = Some(oi.version_id.expect("err").to_string()); - } - - opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; - opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await; - - if lc_event.action.delete_all() { - opts.delete_prefix = true; - opts.delete_prefix_object = true; - } - - // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); - - let mut dobj = api - .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts) - .await - .unwrap(); - if dobj.name.is_empty() { - dobj = oi.clone(); - } - - //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags(); - //tags["version-id"] = dobj.version_id; - - let mut event_name = EventName::ObjectRemovedDelete; - if oi.delete_marker { - event_name = EventName::ObjectRemovedDeleteMarkerCreated; - } - match lc_event.action { - lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions, - lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete, - _ => (), - } - send_event(EventArgs { - event_name: event_name.as_ref().to_string(), - bucket_name: dobj.bucket.clone(), - object: dobj, - user_agent: "Internal: [ILM-Expiry]".to_string(), - host: GLOBAL_LocalNodeName.to_string(), - ..Default::default() - }); - - if lc_event.action != lifecycle::IlmAction::NoneAction { - // let mut num_versions = 1_u64; - // if lc_event.action.delete_all() { - // num_versions = oi.num_versions as u64; - // } - // let _ = time_ilm(num_versions); - } - - true -} - -async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - let mut expiry_state = GLOBAL_ExpiryState.write().await; - expiry_state.enqueue_by_days(oi, event, src).await; - true -} - -pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { - let mut success = false; - match event.action { - lifecycle::IlmAction::DeleteVersionAction - | lifecycle::IlmAction::DeleteAction - | lifecycle::IlmAction::DeleteRestoredAction - | lifecycle::IlmAction::DeleteRestoredVersionAction - | lifecycle::IlmAction::DeleteAllVersionsAction - | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - success = apply_expiry_rule(event, src, oi).await; - } - lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { - success = apply_transition_rule(event, src, oi).await; - } - _ => (), - } - success -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use chrono::Utc; - use rmp_serde::{Deserializer, Serializer}; - use serde::{Deserialize, Serialize}; - - use super::CurrentScannerCycle; - - #[test] - fn test_current_cycle() { - let cycle_info = CurrentScannerCycle { - current: 0, - next: 1, - started: Utc::now(), - cycle_completed: vec![Utc::now(), Utc::now()], - }; - - println!("{cycle_info:?}"); - - let mut wr = Vec::new(); - cycle_info.serialize(&mut Serializer::new(&mut wr)).unwrap(); - - let mut buf_t = Deserializer::new(Cursor::new(wr)); - let c: CurrentScannerCycle = Deserialize::deserialize(&mut buf_t).unwrap(); - - println!("{c:?}"); - } -} diff --git a/crates/ecstore/src/heal/data_scanner_metric.rs b/crates/ecstore/src/heal/data_scanner_metric.rs deleted file mode 100644 index 8e0b8ccc7..000000000 --- a/crates/ecstore/src/heal/data_scanner_metric.rs +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::data_scanner::CurrentScannerCycle; -use crate::bucket::lifecycle::lifecycle; -use chrono::Utc; -use lazy_static::lazy_static; -use rustfs_common::last_minute::{AccElem, LastMinuteLatency}; -use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics; -use std::{ - collections::HashMap, - pin::Pin, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, - time::{Duration, SystemTime}, -}; -use tokio::sync::{Mutex, RwLock}; - -lazy_static! { - pub static ref globalScannerMetrics: Arc = Arc::new(ScannerMetrics::new()); -} - -#[derive(Clone, Debug, PartialEq, PartialOrd)] -pub enum ScannerMetric { - // START Realtime metrics, that only records - // last minute latencies and total operation count. - ReadMetadata = 0, - CheckMissing, - SaveUsage, - ApplyAll, - ApplyVersion, - TierObjSweep, - HealCheck, - Ilm, - CheckReplication, - Yield, - CleanAbandoned, - ApplyNonCurrent, - HealAbandonedVersion, - - // START Trace metrics: - StartTrace, - ScanObject, // Scan object. All operations included. - HealAbandonedObject, - - // END realtime metrics: - LastRealtime, - - // Trace only metrics: - ScanFolder, // Scan a folder on disk, recursively. - ScanCycle, // Full cycle, cluster global. - ScanBucketDrive, // Single bucket on one drive. - CompactFolder, // Folder compacted. - - // Must be last: - Last, -} - -impl ScannerMetric { - /// Convert to string representation for metrics - pub fn as_str(self) -> &'static str { - match self { - Self::ReadMetadata => "read_metadata", - Self::CheckMissing => "check_missing", - Self::SaveUsage => "save_usage", - Self::ApplyAll => "apply_all", - Self::ApplyVersion => "apply_version", - Self::TierObjSweep => "tier_obj_sweep", - Self::HealCheck => "heal_check", - Self::Ilm => "ilm", - Self::CheckReplication => "check_replication", - Self::Yield => "yield", - Self::CleanAbandoned => "clean_abandoned", - Self::ApplyNonCurrent => "apply_non_current", - Self::HealAbandonedVersion => "heal_abandoned_version", - Self::StartTrace => "start_trace", - Self::ScanObject => "scan_object", - Self::HealAbandonedObject => "heal_abandoned_object", - Self::LastRealtime => "last_realtime", - Self::ScanFolder => "scan_folder", - Self::ScanCycle => "scan_cycle", - Self::ScanBucketDrive => "scan_bucket_drive", - Self::CompactFolder => "compact_folder", - Self::Last => "last", - } - } - - /// Convert from index back to enum (safe version) - pub fn from_index(index: usize) -> Option { - if index >= Self::Last as usize { - return None; - } - // Safe conversion using match instead of unsafe transmute - match index { - 0 => Some(Self::ReadMetadata), - 1 => Some(Self::CheckMissing), - 2 => Some(Self::SaveUsage), - 3 => Some(Self::ApplyAll), - 4 => Some(Self::ApplyVersion), - 5 => Some(Self::TierObjSweep), - 6 => Some(Self::HealCheck), - 7 => Some(Self::Ilm), - 8 => Some(Self::CheckReplication), - 9 => Some(Self::Yield), - 10 => Some(Self::CleanAbandoned), - 11 => Some(Self::ApplyNonCurrent), - 12 => Some(Self::HealAbandonedVersion), - 13 => Some(Self::StartTrace), - 14 => Some(Self::ScanObject), - 15 => Some(Self::HealAbandonedObject), - 16 => Some(Self::LastRealtime), - 17 => Some(Self::ScanFolder), - 18 => Some(Self::ScanCycle), - 19 => Some(Self::ScanBucketDrive), - 20 => Some(Self::CompactFolder), - 21 => Some(Self::Last), - _ => None, - } - } -} - -/// Thread-safe wrapper for LastMinuteLatency with atomic operations -#[derive(Default)] -pub struct LockedLastMinuteLatency { - latency: Arc>, -} - -impl Clone for LockedLastMinuteLatency { - fn clone(&self) -> Self { - Self { - latency: Arc::clone(&self.latency), - } - } -} - -impl LockedLastMinuteLatency { - pub fn new() -> Self { - Self { - latency: Arc::new(Mutex::new(LastMinuteLatency::default())), - } - } - - /// Add a duration measurement - pub async fn add(&self, duration: Duration) { - self.add_size(duration, 0).await; - } - - /// Add a duration measurement with size - pub async fn add_size(&self, duration: Duration, size: u64) { - let mut latency = self.latency.lock().await; - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - let elem = AccElem { - n: 1, - total: duration.as_secs(), - size, - }; - latency.add_all(now, &elem); - } - - /// Get total accumulated metrics for the last minute - pub async fn total(&self) -> AccElem { - let mut latency = self.latency.lock().await; - latency.get_total() - } -} - -/// Current path tracker for monitoring active scan paths -struct CurrentPathTracker { - current_path: Arc>, -} - -impl CurrentPathTracker { - fn new(initial_path: String) -> Self { - Self { - current_path: Arc::new(RwLock::new(initial_path)), - } - } - - async fn update_path(&self, path: String) { - *self.current_path.write().await = path; - } - - async fn get_path(&self) -> String { - self.current_path.read().await.clone() - } -} - -/// Main scanner metrics structure -pub struct ScannerMetrics { - // All fields must be accessed atomically and aligned. - operations: Vec, - latency: Vec, - actions: Vec, - actions_latency: Vec, - // Current paths contains disk -> tracker mappings - current_paths: Arc>>>, - - // Cycle information - cycle_info: Arc>>, -} - -impl ScannerMetrics { - pub fn new() -> Self { - let operations = (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(); - - let latency = (0..ScannerMetric::LastRealtime as usize) - .map(|_| LockedLastMinuteLatency::new()) - .collect(); - - Self { - operations, - latency, - actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(), - actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize], - current_paths: Arc::new(RwLock::new(HashMap::new())), - cycle_info: Arc::new(RwLock::new(None)), - } - } - - /// Log scanner action with custom metadata - compatible with existing usage - pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap) { - let metric = metric as usize; - let start_time = SystemTime::now(); - move |_custom: &HashMap| { - let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); - - // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); - - // Update latency for realtime metrics (spawn async task for this) - if (metric) < ScannerMetric::LastRealtime as usize { - let metric_index = metric; - tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; - }); - } - - // Log trace metrics - if metric as u8 > ScannerMetric::StartTrace as u8 { - //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); - } - } - } - - /// Time scanner action with size - returns function that takes size - pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) { - let metric = metric as usize; - let start_time = SystemTime::now(); - move |size: u64| { - let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); - - // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); - - // Update latency for realtime metrics with size (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { - let metric_index = metric; - tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add_size(duration, size).await; - }); - } - } - } - - /// Time a scanner action - returns a closure to call when done - pub fn time(metric: ScannerMetric) -> impl Fn() { - let metric = metric as usize; - let start_time = SystemTime::now(); - move || { - let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); - - // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); - - // Update latency for realtime metrics (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { - let metric_index = metric; - tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; - }); - } - } - } - - /// Time N scanner actions - returns function that takes count, then returns completion function - pub fn time_n(metric: ScannerMetric) -> Box Box + Send + Sync> { - let metric = metric as usize; - let start_time = SystemTime::now(); - Box::new(move |count: usize| { - Box::new(move || { - let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); - - // Update operation count - globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed); - - // Update latency for realtime metrics (spawn async task) - if (metric) < ScannerMetric::LastRealtime as usize { - let metric_index = metric; - tokio::spawn(async move { - globalScannerMetrics.latency[metric_index].add(duration).await; - }); - } - }) - }) - } - - pub fn time_ilm(a: lifecycle::IlmAction) -> Box Box + Send + Sync> { - let a_clone = a as usize; - if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { - return Box::new(move |_: u64| Box::new(move || {})); - } - let start = SystemTime::now(); - Box::new(move |versions: u64| { - Box::new(move || { - let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); - tokio::spawn(async move { - globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed); - globalScannerMetrics.actions_latency[a_clone].add(duration).await; - }); - }) - }) - } - - /// Increment time with specific duration - pub async fn inc_time(metric: ScannerMetric, duration: Duration) { - let metric = metric as usize; - // Update operation count - globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); - - // Update latency for realtime metrics - if (metric) < ScannerMetric::LastRealtime as usize { - globalScannerMetrics.latency[metric].add(duration).await; - } - } - - /// Get lifetime operation count for a metric - pub fn lifetime(&self, metric: ScannerMetric) -> u64 { - let metric = metric as usize; - if (metric) >= ScannerMetric::Last as usize { - return 0; - } - self.operations[metric].load(Ordering::Relaxed) - } - - /// Get last minute statistics for a metric - pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem { - let metric = metric as usize; - if (metric) >= ScannerMetric::LastRealtime as usize { - return AccElem::default(); - } - self.latency[metric].total().await - } - - /// Set current cycle information - pub async fn set_cycle(&self, cycle: Option) { - *self.cycle_info.write().await = cycle; - } - - /// Get current cycle information - pub async fn get_cycle(&self) -> Option { - self.cycle_info.read().await.clone() - } - - /// Get current active paths - pub async fn get_current_paths(&self) -> Vec { - let mut result = Vec::new(); - let paths = self.current_paths.read().await; - - for (disk, tracker) in paths.iter() { - let path = tracker.get_path().await; - result.push(format!("{disk}/{path}")); - } - - result - } - - /// Get number of active drives - pub async fn active_drives(&self) -> usize { - self.current_paths.read().await.len() - } - - /// Generate metrics report - pub async fn report(&self) -> M_ScannerMetrics { - let mut metrics = M_ScannerMetrics::default(); - - // Set cycle information - if let Some(cycle) = self.get_cycle().await { - metrics.current_cycle = cycle.current; - metrics.cycles_completed_at = cycle.cycle_completed; - metrics.current_started = cycle.started; - } - - metrics.collected_at = Utc::now(); - metrics.active_paths = self.get_current_paths().await; - - // Lifetime operations - for i in 0..ScannerMetric::Last as usize { - let count = self.operations[i].load(Ordering::Relaxed); - if count > 0 { - if let Some(metric) = ScannerMetric::from_index(i) { - metrics.life_time_ops.insert(metric.as_str().to_string(), count); - } - } - } - - // Last minute statistics for realtime metrics - for i in 0..ScannerMetric::LastRealtime as usize { - let last_min = self.latency[i].total().await; - if last_min.n > 0 { - if let Some(_metric) = ScannerMetric::from_index(i) { - // Convert to madmin TimedAction format if needed - // This would require implementing the conversion - } - } - } - - metrics - } -} - -// Type aliases for compatibility with existing code -pub type UpdateCurrentPathFn = Arc Pin + Send>> + Send + Sync>; -pub type CloseDiskFn = Arc Pin + Send>> + Send + Sync>; - -/// Create a current path updater for tracking scan progress -pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) { - let tracker = Arc::new(CurrentPathTracker::new(initial.to_string())); - let disk_name = disk.to_string(); - - // Store the tracker in global metrics - let tracker_clone = Arc::clone(&tracker); - let disk_clone = disk_name.clone(); - tokio::spawn(async move { - globalScannerMetrics - .current_paths - .write() - .await - .insert(disk_clone, tracker_clone); - }); - - let update_fn = { - let tracker = Arc::clone(&tracker); - Arc::new(move |path: &str| -> Pin + Send>> { - let tracker = Arc::clone(&tracker); - let path = path.to_string(); - Box::pin(async move { - tracker.update_path(path).await; - }) - }) - }; - - let done_fn = { - let disk_name = disk_name.clone(); - Arc::new(move || -> Pin + Send>> { - let disk_name = disk_name.clone(); - Box::pin(async move { - globalScannerMetrics.current_paths.write().await.remove(&disk_name); - }) - }) - }; - - (update_fn, done_fn) -} - -impl Default for ScannerMetrics { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/ecstore/src/heal/data_usage.rs b/crates/ecstore/src/heal/data_usage.rs deleted file mode 100644 index a6d569971..000000000 --- a/crates/ecstore/src/heal/data_usage.rs +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::error::{Error, Result}; -use crate::{ - bucket::metadata_sys::get_replication_config, - config::com::{read_config, save_config}, - disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, - error::to_object_err, - new_object_layer_fn, - store::ECStore, -}; -use lazy_static::lazy_static; -use rustfs_utils::path::SLASH_SEPARATOR; -use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, sync::Arc, time::SystemTime}; -use tokio::sync::mpsc::Receiver; -use tracing::{error, warn}; - -pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; -const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; -const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; -pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; -lazy_static! { - pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX); - pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME); - pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = - format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME); - pub static ref BACKGROUND_HEAL_INFO_PATH: String = - format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json"); -} - -// BucketTargetUsageInfo - bucket target usage info provides -// - replicated size for all objects sent to this target -// - replica size for all objects received from this target -// - replication pending size for all objects pending replication to this target -// - replication failed size for all objects failed replication to this target -// - replica pending count -// - replica failed count -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BucketTargetUsageInfo { - pub replication_pending_size: u64, - pub replication_failed_size: u64, - pub replicated_size: u64, - pub replica_size: u64, - pub replication_pending_count: u64, - pub replication_failed_count: u64, - pub replicated_count: u64, -} - -// BucketUsageInfo - bucket usage info provides -// - total size of the bucket -// - total objects in a bucket -// - object size histogram per bucket -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BucketUsageInfo { - pub size: u64, - // Following five fields suffixed with V1 are here for backward compatibility - // Total Size for objects that have not yet been replicated - pub replication_pending_size_v1: u64, - // Total size for objects that have witness one or more failures and will be retried - pub replication_failed_size_v1: u64, - // Total size for objects that have been replicated to destination - pub replicated_size_v1: u64, - // Total number of objects pending replication - pub replication_pending_count_v1: u64, - // Total number of objects that failed replication - pub replication_failed_count_v1: u64, - - pub objects_count: u64, - pub object_size_histogram: HashMap, - pub object_versions_histogram: HashMap, - pub versions_count: u64, - pub delete_markers_count: u64, - pub replica_size: u64, - pub replica_count: u64, - pub replication_info: HashMap, -} - -// DataUsageInfo represents data usage stats of the underlying Object API -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct DataUsageInfo { - pub total_capacity: u64, - pub total_used_capacity: u64, - pub total_free_capacity: u64, - - // LastUpdate is the timestamp of when the data usage info was last updated. - // This does not indicate a full scan. - pub last_update: Option, - - // Objects total count across all buckets - pub objects_total_count: u64, - // Versions total count across all buckets - pub versions_total_count: u64, - // Delete markers total count across all buckets - pub delete_markers_total_count: u64, - // Objects total size across all buckets - pub objects_total_size: u64, - pub replication_info: HashMap, - - // Total number of buckets in this cluster - pub buckets_count: u64, - // Buckets usage info provides following information across all buckets - // - total size of the bucket - // - total objects in a bucket - // - object size histogram per bucket - pub buckets_usage: HashMap, - // Deprecated kept here for backward compatibility reasons. - pub bucket_sizes: HashMap, - // Todo: TierStats - // TierStats contains per-tier stats of all configured remote tiers -} - -pub async fn store_data_usage_in_backend(mut rx: Receiver) { - let Some(store) = new_object_layer_fn() else { - error!("errServerNotInitialized"); - return; - }; - - let mut attempts = 1; - loop { - match rx.recv().await { - Some(data_usage_info) => { - if let Ok(data) = serde_json::to_vec(&data_usage_info) { - if attempts > 10 { - let _ = - save_config(store.clone(), &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), data.clone()).await; - attempts += 1; - } - let _ = save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await; - attempts += 1; - } else { - continue; - } - } - None => { - return; - } - } - } -} - -// TODO: cancel ctx -pub async fn load_data_usage_from_backend(store: Arc) -> Result { - let buf = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await { - Ok(data) => data, - Err(e) => { - error!("Failed to read data usage info from backend: {}", e); - if e == Error::ConfigNotFound { - return Ok(DataUsageInfo::default()); - } - - return Err(to_object_err(e, vec![RUSTFS_META_BUCKET, &DATA_USAGE_OBJ_NAME_PATH])); - } - }; - - let mut data_usage_info: DataUsageInfo = serde_json::from_slice(&buf)?; - - warn!("Loaded data usage info from backend {:?}", &data_usage_info); - - if data_usage_info.buckets_usage.is_empty() { - data_usage_info.buckets_usage = data_usage_info - .bucket_sizes - .iter() - .map(|(bucket, &size)| { - ( - bucket.clone(), - BucketUsageInfo { - size, - ..Default::default() - }, - ) - }) - .collect(); - } - - if data_usage_info.bucket_sizes.is_empty() { - data_usage_info.bucket_sizes = data_usage_info - .buckets_usage - .iter() - .map(|(bucket, bui)| (bucket.clone(), bui.size)) - .collect(); - } - - for (bucket, bui) in &data_usage_info.buckets_usage { - if bui.replicated_size_v1 > 0 - || bui.replication_failed_count_v1 > 0 - || bui.replication_failed_size_v1 > 0 - || bui.replication_pending_count_v1 > 0 - { - if let Ok((cfg, _)) = get_replication_config(bucket).await { - if !cfg.role.is_empty() { - data_usage_info.replication_info.insert( - cfg.role.clone(), - BucketTargetUsageInfo { - replication_failed_size: bui.replication_failed_size_v1, - replication_failed_count: bui.replication_failed_count_v1, - replicated_size: bui.replicated_size_v1, - replication_pending_count: bui.replication_pending_count_v1, - replication_pending_size: bui.replication_pending_size_v1, - ..Default::default() - }, - ); - } - } - } - } - - Ok(data_usage_info) -} diff --git a/crates/ecstore/src/heal/data_usage_cache.rs b/crates/ecstore/src/heal/data_usage_cache.rs deleted file mode 100644 index 70b509cf6..000000000 --- a/crates/ecstore/src/heal/data_usage_cache.rs +++ /dev/null @@ -1,928 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::config::com::save_config; -use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; -use crate::error::{Error, Result}; -use crate::new_object_layer_fn; -use crate::set_disk::SetDisks; -use crate::store_api::{BucketInfo, ObjectIO, ObjectOptions}; -use bytesize::ByteSize; -use http::HeaderMap; -use path_clean::PathClean; -use rand::Rng; -use rmp_serde::Serializer; -use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration}; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::path::Path; -use std::time::{Duration, SystemTime}; -use tokio::sync::mpsc::Sender; -use tokio::time::sleep; - -use super::data_scanner::{DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS, SizeSummary}; -use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo}; - -// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals -pub const DATA_USAGE_BUCKET_LEN: usize = 11; -pub const DATA_USAGE_VERSION_LEN: usize = 7; - -pub type DataUsageHashMap = HashSet; - -struct ObjectHistogramInterval { - name: &'static str, - start: u64, - end: u64, -} - -const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [ - ObjectHistogramInterval { - name: "LESS_THAN_1024_B", - start: 0, - end: ByteSize::kib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1024_B_AND_64_KB", - start: ByteSize::kib(1).as_u64(), - end: ByteSize::kib(64).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_64_KB_AND_256_KB", - start: ByteSize::kib(64).as_u64(), - end: ByteSize::kib(256).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_256_KB_AND_512_KB", - start: ByteSize::kib(256).as_u64(), - end: ByteSize::kib(512).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_512_KB_AND_1_MB", - start: ByteSize::kib(512).as_u64(), - end: ByteSize::mib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1024B_AND_1_MB", - start: ByteSize::kib(1).as_u64(), - end: ByteSize::mib(1).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_1_MB_AND_10_MB", - start: ByteSize::mib(1).as_u64(), - end: ByteSize::mib(10).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_10_MB_AND_64_MB", - start: ByteSize::mib(10).as_u64(), - end: ByteSize::mib(64).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_64_MB_AND_128_MB", - start: ByteSize::mib(64).as_u64(), - end: ByteSize::mib(128).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_128_MB_AND_512_MB", - start: ByteSize::mib(128).as_u64(), - end: ByteSize::mib(512).as_u64() - 1, - }, - ObjectHistogramInterval { - name: "GREATER_THAN_512_MB", - start: ByteSize::mib(512).as_u64(), - end: u64::MAX, - }, -]; - -const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [ - ObjectHistogramInterval { - name: "UNVERSIONED", - start: 0, - end: 0, - }, - ObjectHistogramInterval { - name: "SINGLE_VERSION", - start: 1, - end: 1, - }, - ObjectHistogramInterval { - name: "BETWEEN_2_AND_10", - start: 2, - end: 9, - }, - ObjectHistogramInterval { - name: "BETWEEN_10_AND_100", - start: 10, - end: 99, - }, - ObjectHistogramInterval { - name: "BETWEEN_100_AND_1000", - start: 100, - end: 999, - }, - ObjectHistogramInterval { - name: "BETWEEN_1000_AND_10000", - start: 1000, - end: 9999, - }, - ObjectHistogramInterval { - name: "GREATER_THAN_10000", - start: 10000, - end: u64::MAX, - }, -]; - -#[derive(Clone, Copy, Default)] -pub struct TierStats { - pub total_size: u64, - pub num_versions: i32, - pub num_objects: i32, -} - -impl TierStats { - pub fn add(&self, u: &TierStats) -> TierStats { - TierStats { - total_size: self.total_size + u.total_size, - num_versions: self.num_versions + u.num_versions, - num_objects: self.num_objects + u.num_objects, - } - } -} - -struct AllTierStats { - tiers: HashMap, -} - -impl AllTierStats { - pub fn new() -> Self { - Self { tiers: HashMap::new() } - } - - fn add_sizes(&mut self, tiers: HashMap) { - for (tier, st) in tiers { - self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); - } - } - - fn merge(&mut self, other: AllTierStats) { - for (tier, st) in other.tiers { - self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); - } - } - - fn populate_stats(&self, stats: &mut HashMap) { - for (tier, st) in &self.tiers { - stats.insert( - tier.clone(), - TierStats { - total_size: st.total_size, - num_versions: st.num_versions, - num_objects: st.num_objects, - }, - ); - } - } -} - -// sizeHistogram is a size histogram. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SizeHistogram(Vec); - -impl Default for SizeHistogram { - fn default() -> Self { - Self(vec![0; DATA_USAGE_BUCKET_LEN]) - } -} - -impl SizeHistogram { - fn add(&mut self, size: u64) { - for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() { - if size >= interval.start && size <= interval.end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let mut res = HashMap::new(); - let mut spl_count = 0; - for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) { - if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 { - res.insert(oh.name.to_string(), spl_count); - } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() { - spl_count += count; - res.insert(oh.name.to_string(), *count); - } else { - res.insert(oh.name.to_string(), *count); - } - } - res - } -} - -// versionsHistogram is a histogram of number of versions in an object. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct VersionsHistogram(Vec); - -impl Default for VersionsHistogram { - fn default() -> Self { - Self(vec![0; DATA_USAGE_VERSION_LEN]) - } -} - -impl VersionsHistogram { - fn add(&mut self, size: u64) { - for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() { - if size >= interval.start && size <= interval.end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let mut res = HashMap::new(); - for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) { - res.insert(ov.name.to_string(), *count); - } - res - } -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationStats { - pub pending_size: u64, - pub replicated_size: u64, - pub failed_size: u64, - pub failed_count: u64, - pub pending_count: u64, - pub missed_threshold_size: u64, - pub after_threshold_size: u64, - pub missed_threshold_count: u64, - pub after_threshold_count: u64, - pub replicated_count: u64, -} - -impl ReplicationStats { - pub fn empty(&self) -> bool { - self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0 - } -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationAllStats { - pub targets: HashMap, - pub replica_size: u64, - pub replica_count: u64, -} - -impl ReplicationAllStats { - pub fn empty(&self) -> bool { - if self.replica_size != 0 && self.replica_count != 0 { - return false; - } - for (_, v) in self.targets.iter() { - if !v.empty() { - return false; - } - } - - true - } -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageEntry { - pub children: DataUsageHashMap, - // These fields do not include any children. - pub size: usize, - pub objects: usize, - pub versions: usize, - pub delete_markers: usize, - pub obj_sizes: SizeHistogram, - pub obj_versions: VersionsHistogram, - pub replication_stats: Option, - // Todo: tier - // pub all_tier_stats: , - pub compacted: bool, -} - -impl DataUsageEntry { - pub fn add_child(&mut self, hash: &DataUsageHash) { - if self.children.contains(&hash.key()) { - return; - } - - self.children.insert(hash.key()); - } - - pub fn add_sizes(&mut self, summary: &SizeSummary) { - self.size += summary.total_size; - self.versions += summary.versions; - self.delete_markers += summary.delete_markers; - self.obj_sizes.add(summary.total_size as u64); - self.obj_versions.add(summary.versions as u64); - - let replication_stats = if self.replication_stats.is_none() { - self.replication_stats = Some(ReplicationAllStats::default()); - self.replication_stats.as_mut().unwrap() - } else { - self.replication_stats.as_mut().unwrap() - }; - replication_stats.replica_size += summary.replica_size as u64; - replication_stats.replica_count += summary.replica_count as u64; - - for (arn, st) in &summary.repl_target_stats { - let tgt_stat = replication_stats - .targets - .entry(arn.to_string()) - .or_insert(ReplicationStats::default()); - tgt_stat.pending_size += st.pending_size as u64; - tgt_stat.failed_size += st.failed_size as u64; - tgt_stat.replicated_size += st.replicated_size as u64; - tgt_stat.replicated_count += st.replicated_count as u64; - tgt_stat.failed_count += st.failed_count as u64; - tgt_stat.pending_count += st.pending_count as u64; - } - // Todo:: tiers - } - - pub fn merge(&mut self, other: &DataUsageEntry) { - self.objects += other.objects; - self.versions += other.versions; - self.delete_markers += other.delete_markers; - self.size += other.size; - if let Some(o_rep) = &other.replication_stats { - if self.replication_stats.is_none() { - self.replication_stats = Some(ReplicationAllStats::default()); - } - let s_rep = self.replication_stats.as_mut().unwrap(); - s_rep.targets.clear(); - s_rep.replica_size += o_rep.replica_size; - s_rep.replica_count += o_rep.replica_count; - for (arn, stat) in o_rep.targets.iter() { - let st = s_rep.targets.entry(arn.clone()).or_default(); - *st = ReplicationStats { - pending_size: stat.pending_size + st.pending_size, - failed_size: stat.failed_size + st.failed_size, - replicated_size: stat.replicated_size + st.replicated_size, - pending_count: stat.pending_count + st.pending_count, - failed_count: stat.failed_count + st.failed_count, - replicated_count: stat.replicated_count + st.replicated_count, - ..Default::default() - }; - } - } - - for (i, v) in other.obj_sizes.0.iter().enumerate() { - self.obj_sizes.0[i] += v; - } - - for (i, v) in other.obj_versions.0.iter().enumerate() { - self.obj_versions.0[i] += v; - } - - // todo: tiers - } -} - -#[derive(Clone)] -pub struct DataUsageEntryInfo { - pub name: String, - pub parent: String, - pub entry: DataUsageEntry, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageCacheInfo { - pub name: String, - pub next_cycle: u32, - pub last_update: Option, - pub skip_healing: bool, - #[serde(skip)] - pub lifecycle: Option, - #[serde(skip)] - pub updates: Option>, - #[serde(skip)] - pub replication: Option, -} - -// impl Default for DataUsageCacheInfo { -// fn default() -> Self { -// Self { -// name: Default::default(), -// next_cycle: Default::default(), -// last_update: SystemTime::now(), -// skip_healing: Default::default(), -// updates: Default::default(), -// replication: Default::default(), -// } -// } -// } - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageCache { - pub info: DataUsageCacheInfo, - pub cache: HashMap, -} - -impl DataUsageCache { - pub async fn load(store: &SetDisks, name: &str) -> Result { - let mut d = DataUsageCache::default(); - let mut retries = 0; - while retries < 5 { - let path = Path::new(BUCKET_META_PREFIX).join(name); - // warn!("Loading data usage cache from backend: {}", path.display()); - match store - .get_object_reader( - RUSTFS_META_BUCKET, - path.to_str().unwrap(), - None, - HeaderMap::new(), - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(mut reader) => { - if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { - d = info - } - break; - } - Err(err) => { - // warn!("Failed to load data usage cache from backend: {}", &err); - match err { - Error::FileNotFound | Error::VolumeNotFound => { - match store - .get_object_reader( - RUSTFS_META_BUCKET, - name, - None, - HeaderMap::new(), - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(mut reader) => { - if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { - d = info - } - break; - } - Err(_) => match err { - Error::FileNotFound | Error::VolumeNotFound => { - break; - } - _ => {} - }, - } - } - _ => { - break; - } - } - } - } - retries += 1; - let dur = { - let mut rng = rand::rng(); - rng.random_range(0..1_000) - }; - sleep(Duration::from_millis(dur)).await; - } - Ok(d) - } - - pub async fn save(&self, name: &str) -> Result<()> { - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let buf = self.marshal_msg()?; - let buf_clone = buf.clone(); - - let store_clone = store.clone(); - - let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string(); - - let name_clone = name.clone(); - tokio::spawn(async move { - let _ = save_config(store_clone, &format!("{}{}", &name_clone, ".bkp"), buf_clone).await; - }); - save_config(store, &name, buf).await?; - Ok(()) - } - - pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) { - let hash = hash_path(path); - self.cache.insert(hash.key(), e); - if !parent.is_empty() { - let phash = hash_path(parent); - let p = { - let p = self.cache.entry(phash.key()).or_default(); - p.add_child(&hash); - p.clone() - }; - self.cache.insert(phash.key(), p); - } - } - - pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option, e: &DataUsageEntry) { - self.cache.insert(hash.key(), e.clone()); - if let Some(parent) = parent { - self.cache.entry(parent.key()).or_default().add_child(hash); - } - } - - pub fn find(&self, path: &str) -> Option { - self.cache.get(&hash_path(path).key()).cloned() - } - - pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { - self.cache.entry(h.string()).or_default().children.clone() - } - - pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { - let mut root = root.clone(); - for id in root.children.clone().iter() { - if let Some(e) = self.cache.get(id) { - let mut e = e.clone(); - if !e.children.is_empty() { - e = self.flatten(&e); - } - root.merge(&e); - } - } - root.children.clear(); - root - } - - pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option) { - if let Some(e) = src.cache.get(&hash.string()) { - self.cache.insert(hash.key(), e.clone()); - for ch in e.children.iter() { - if *ch == hash.key() { - return; - } - self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone())); - } - if let Some(parent) = parent { - let p = self.cache.entry(parent.key()).or_default(); - p.add_child(hash); - } - } - } - - pub fn delete_recursive(&mut self, hash: &DataUsageHash) { - let mut need_remove = Vec::new(); - if let Some(v) = self.cache.get(&hash.string()) { - for child in v.children.iter() { - need_remove.push(child.clone()); - } - } - self.cache.remove(&hash.string()); - need_remove.iter().for_each(|child| { - self.delete_recursive(&DataUsageHash(child.to_string())); - }); - } - - pub fn size_recursive(&self, path: &str) -> Option { - match self.find(path) { - Some(root) => { - if root.children.is_empty() { - return Some(root); - } - let mut flat = self.flatten(&root); - if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() { - flat.replication_stats = None; - } - Some(flat) - } - None => None, - } - } - - pub fn search_parent(&self, hash: &DataUsageHash) -> Option { - let want = hash.key(); - if let Some(last_index) = want.rfind('/') { - if let Some(v) = self.find(&want[0..last_index]) { - if v.children.contains(&want) { - let found = hash_path(&want[0..last_index]); - return Some(found); - } - } - } - - for (k, v) in self.cache.iter() { - if v.children.contains(&want) { - let found = DataUsageHash(k.clone()); - return Some(found); - } - } - None - } - - pub fn is_compacted(&self, hash: &DataUsageHash) -> bool { - match self.cache.get(&hash.key()) { - Some(due) => due.compacted, - None => false, - } - } - - pub fn force_compact(&mut self, limit: usize) { - if self.cache.len() < limit { - return; - } - let top = hash_path(&self.info.name).key(); - let top_e = match self.find(&top) { - Some(e) => e, - None => return, - }; - if top_e.children.len() > >::try_into(DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS).unwrap() { - self.reduce_children_of(&hash_path(&self.info.name), limit, true); - } - if self.cache.len() <= limit { - return; - } - - let mut found = HashSet::new(); - found.insert(top); - mark(self, &top_e, &mut found); - self.cache.retain(|k, _| { - if !found.contains(k) { - return false; - } - true - }); - } - - pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) { - let e = match self.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - - if e.compacted { - return; - } - - if e.children.len() > limit && compact_self { - let mut flat = self.size_recursive(&path.key()).unwrap_or_default(); - flat.compacted = true; - self.delete_recursive(path); - self.replace_hashed(path, &None, &flat); - return; - } - let total = self.total_children_rec(&path.key()); - if total < limit { - return; - } - - let mut leaves = Vec::new(); - let mut remove = total - limit; - add(self, path, &mut leaves); - leaves.sort_by(|a, b| a.objects.cmp(&b.objects)); - - while remove > 0 && !leaves.is_empty() { - let e = leaves.first().unwrap(); - let candidate = e.path.clone(); - if candidate == *path && !compact_self { - break; - } - let removing = self.total_children_rec(&candidate.key()); - let mut flat = match self.size_recursive(&candidate.key()) { - Some(flat) => flat, - None => { - leaves.remove(0); - continue; - } - }; - - flat.compacted = true; - self.delete_recursive(&candidate); - self.replace_hashed(&candidate, &None, &flat); - - remove -= removing; - leaves.remove(0); - } - } - - pub fn total_children_rec(&self, path: &str) -> usize { - let root = self.find(path); - - if root.is_none() { - return 0; - } - let root = root.unwrap(); - if root.children.is_empty() { - return 0; - } - - let mut n = root.children.len(); - for ch in root.children.iter() { - n += self.total_children_rec(ch); - } - n - } - - pub fn merge(&mut self, o: &DataUsageCache) { - let mut existing_root = self.root(); - let other_root = o.root(); - if existing_root.is_none() && other_root.is_none() { - return; - } - if other_root.is_none() { - return; - } - if existing_root.is_none() { - *self = o.clone(); - return; - } - if o.info.last_update.gt(&self.info.last_update) { - self.info.last_update = o.info.last_update; - } - - existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap()); - self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap()); - let e_hash = self.root_hash(); - for key in other_root.as_ref().unwrap().children.iter() { - let entry = &o.cache[key]; - let flat = o.flatten(entry); - let mut existing = self.cache[key].clone(); - existing.merge(&flat); - self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing); - } - } - - pub fn root_hash(&self) -> DataUsageHash { - hash_path(&self.info.name) - } - - pub fn root(&self) -> Option { - self.find(&self.info.name) - } - - pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo { - let e = match self.find(path) { - Some(e) => e, - None => return DataUsageInfo::default(), - }; - let flat = self.flatten(&e); - DataUsageInfo { - last_update: self.info.last_update, - objects_total_count: flat.objects as u64, - versions_total_count: flat.versions as u64, - delete_markers_total_count: flat.delete_markers as u64, - objects_total_size: flat.size as u64, - buckets_count: e.children.len() as u64, - buckets_usage: self.buckets_usage_info(buckets), - ..Default::default() - } - } - - pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap { - let mut dst = HashMap::new(); - for bucket in buckets.iter() { - let e = match self.find(&bucket.name) { - Some(e) => e, - None => continue, - }; - let flat = self.flatten(&e); - let mut bui = BucketUsageInfo { - size: flat.size as u64, - versions_count: flat.versions as u64, - objects_count: flat.objects as u64, - delete_markers_count: flat.delete_markers as u64, - object_size_histogram: flat.obj_sizes.to_map(), - object_versions_histogram: flat.obj_versions.to_map(), - ..Default::default() - }; - if let Some(rs) = &flat.replication_stats { - bui.replica_size = rs.replica_size; - bui.replica_count = rs.replica_count; - - for (arn, stat) in rs.targets.iter() { - bui.replication_info.insert( - arn.clone(), - BucketTargetUsageInfo { - replication_pending_size: stat.pending_size, - replicated_size: stat.replicated_size, - replication_failed_size: stat.failed_size, - replication_pending_count: stat.pending_count, - replication_failed_count: stat.failed_count, - replicated_count: stat.replicated_count, - ..Default::default() - }, - ); - } - } - dst.insert(bucket.name.clone(), bui); - } - dst - } - - pub fn marshal_msg(&self) -> Result> { - let mut buf = Vec::new(); - - self.serialize(&mut Serializer::new(&mut buf))?; - - Ok(buf) - } - - pub fn unmarshal(buf: &[u8]) -> Result { - let t: Self = rmp_serde::from_slice(buf)?; - Ok(t) - } -} - -#[derive(Default, Clone)] -struct Inner { - objects: usize, - path: DataUsageHash, -} - -fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec) { - let e = match data_usage_cache.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - if !e.children.is_empty() { - return; - } - - let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default(); - leaves.push(Inner { - objects: sz.objects, - path: path.clone(), - }); - for ch in e.children.iter() { - add(data_usage_cache, &DataUsageHash(ch.clone()), leaves); - } -} - -fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet) { - for k in entry.children.iter() { - found.insert(k.to_string()); - if let Some(ch) = duc.cache.get(k) { - mark(duc, ch, found); - } - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct DataUsageHash(pub String); - -impl DataUsageHash { - pub fn string(&self) -> String { - self.0.clone() - } - - pub fn key(&self) -> String { - self.0.clone() - } - - pub fn mod_(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - hash as u32 % cycles == cycle % cycles - } - - pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - (hash >> 32) as u32 % cycles == cycle % cycles - } - - fn calculate_hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.0.hash(&mut hasher); - hasher.finish() - } -} - -pub fn hash_path(data: &str) -> DataUsageHash { - DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string()) -} diff --git a/crates/ecstore/src/heal/error.rs b/crates/ecstore/src/heal/error.rs deleted file mode 100644 index dd2db424e..000000000 --- a/crates/ecstore/src/heal/error.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage"; -pub const ERR_SKIP_FILE: &str = "skip this file"; -pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled"; -pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long"; -pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again"; diff --git a/crates/ecstore/src/heal/heal_commands.rs b/crates/ecstore/src/heal/heal_commands.rs deleted file mode 100644 index 3df0f3ba1..000000000 --- a/crates/ecstore/src/heal/heal_commands.rs +++ /dev/null @@ -1,544 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - collections::{HashMap, HashSet}, - path::Path, - time::SystemTime, -}; - -use crate::{ - config::storageclass::{RRS, STANDARD}, - disk::{BUCKET_META_PREFIX, DeleteOptions, DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError, fs::read_file}, - global::GLOBAL_BackgroundHealState, - heal::heal_ops::HEALING_TRACKER_FILENAME, - new_object_layer_fn, - store_api::{BucketInfo, StorageAPI}, -}; -use crate::{disk, error::Result}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; -use tokio::sync::RwLock; - -use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID}; - -pub type HealScanMode = usize; - -pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0; -pub const HEAL_NORMAL_SCAN: HealScanMode = 1; -pub const HEAL_DEEP_SCAN: HealScanMode = 2; - -pub const HEAL_ITEM_METADATA: &str = "metadata"; -pub const HEAL_ITEM_BUCKET: &str = "bucket"; -pub const HEAL_ITEM_BUCKET_METADATA: &str = "bucket-metadata"; -pub const HEAL_ITEM_OBJECT: &str = "object"; - -pub const DRIVE_STATE_OK: &str = "ok"; -pub const DRIVE_STATE_OFFLINE: &str = "offline"; -pub const DRIVE_STATE_CORRUPT: &str = "corrupt"; -pub const DRIVE_STATE_MISSING: &str = "missing"; -pub const DRIVE_STATE_PERMISSION: &str = "permission-denied"; -pub const DRIVE_STATE_FAULTY: &str = "faulty"; -pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount"; -pub const DRIVE_STATE_UNKNOWN: &str = "unknown"; -pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk - -lazy_static! { - pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap(); -} - -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] -pub struct HealOpts { - pub recursive: bool, - #[serde(rename = "dryRun")] - pub dry_run: bool, - pub remove: bool, - pub recreate: bool, - #[serde(rename = "scanMode")] - pub scan_mode: HealScanMode, - #[serde(rename = "updateParity")] - pub update_parity: bool, - #[serde(rename = "nolock")] - pub no_lock: bool, - pub pool: Option, - pub set: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct HealStartSuccess { - #[serde(rename = "clientToken")] - pub client_token: String, - #[serde(rename = "clientAddress")] - pub client_address: String, - #[serde(rename = "startTime")] - pub start_time: DateTime, -} - -impl Default for HealStartSuccess { - fn default() -> Self { - Self { - client_token: Default::default(), - client_address: Default::default(), - start_time: Utc::now(), - } - } -} - -pub type HealStopSuccess = HealStartSuccess; - -#[derive(Debug, Default, Deserialize, Serialize)] -pub struct HealingTracker { - #[serde(skip_serializing, skip_deserializing)] - pub disk: Option, - pub id: String, - pub pool_index: Option, - pub set_index: Option, - pub disk_index: Option, - pub path: String, - pub endpoint: String, - pub started: Option, - pub last_update: Option, - pub objects_total_count: u64, - pub objects_total_size: u64, - pub items_healed: u64, - pub items_failed: u64, - pub item_skipped: u64, - pub bytes_done: u64, - pub bytes_failed: u64, - pub bytes_skipped: u64, - pub bucket: String, - pub object: String, - pub resume_items_healed: u64, - pub resume_items_failed: u64, - pub resume_items_skipped: u64, - pub resume_bytes_done: u64, - pub resume_bytes_failed: u64, - pub resume_bytes_skipped: u64, - pub queue_buckets: Vec, - pub healed_buckets: Vec, - pub heal_id: String, - pub retry_attempts: u64, - pub finished: bool, - #[serde(skip_serializing, skip_deserializing)] - pub mu: RwLock, -} - -impl HealingTracker { - pub fn marshal_msg(&self) -> disk::error::Result> { - Ok(serde_json::to_vec(self)?) - } - - pub fn unmarshal_msg(data: &[u8]) -> disk::error::Result { - Ok(serde_json::from_slice::(data)?) - } - - pub async fn reset_healing(&mut self) { - let _ = self.mu.write().await; - self.items_healed = 0; - self.items_failed = 0; - self.bytes_done = 0; - self.bytes_failed = 0; - self.resume_items_healed = 0; - self.resume_items_failed = 0; - self.resume_bytes_done = 0; - self.resume_bytes_failed = 0; - self.item_skipped = 0; - self.bytes_skipped = 0; - - self.healed_buckets = Vec::new(); - self.bucket = String::new(); - self.object = String::new(); - } - - pub async fn get_last_update(&self) -> Option { - let _ = self.mu.read().await; - - self.last_update - } - - pub async fn get_bucket(&self) -> String { - let _ = self.mu.read().await; - - self.bucket.clone() - } - - pub async fn set_bucket(&mut self, bucket: &str) { - let _ = self.mu.write().await; - - self.bucket = bucket.to_string(); - } - - pub async fn get_object(&self) -> String { - let _ = self.mu.read().await; - - self.object.clone() - } - - pub async fn set_object(&mut self, object: &str) { - let _ = self.mu.write().await; - - self.object = object.to_string(); - } - - pub async fn update_progress(&mut self, success: bool, skipped: bool, by: u64) { - let _ = self.mu.write().await; - - if success { - self.items_healed += 1; - self.bytes_done += by; - } else if skipped { - self.item_skipped += 1; - self.bytes_skipped += by; - } else { - self.items_failed += 1; - self.bytes_failed += by; - } - } - - pub async fn update(&mut self) -> disk::error::Result<()> { - if let Some(disk) = &self.disk { - if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() { - return Err(DiskError::other(format!("healingTracker: drive {} is not marked as healing", self.id))); - } - let _ = self.mu.write().await; - if self.id.is_empty() || self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { - self.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); - let disk_location = disk.get_disk_location(); - self.pool_index = disk_location.pool_idx; - self.set_index = disk_location.set_idx; - self.disk_index = disk_location.disk_idx; - } - } - - self.save().await - } - - pub async fn save(&mut self) -> disk::error::Result<()> { - let _ = self.mu.write().await; - if self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { - let Some(store) = new_object_layer_fn() else { - return Err(DiskError::other("errServerNotInitialized")); - }; - - // TODO: check error type - (self.pool_index, self.set_index, self.disk_index) = - store.get_pool_and_set(&self.id).await.map_err(|_| DiskError::DiskNotFound)?; - } - - self.last_update = Some(SystemTime::now()); - - let htracker_bytes = self.marshal_msg()?; - - GLOBAL_BackgroundHealState.update_heal_status(self).await; - - if let Some(disk) = &self.disk { - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - disk.write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes.into()) - .await?; - } - Ok(()) - } - - pub async fn delete(&self) -> Result<()> { - if let Some(disk) = &self.disk { - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - disk.delete( - RUSTFS_META_BUCKET, - file_path.to_str().unwrap(), - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await?; - } - - Ok(()) - } - - pub async fn is_healed(&self, bucket: &str) -> bool { - let _ = self.mu.read().await; - for v in self.healed_buckets.iter() { - if v == bucket { - return true; - } - } - - false - } - - pub async fn resume(&mut self) { - let _ = self.mu.write().await; - - self.items_healed = self.resume_items_healed; - self.items_failed = self.resume_items_failed; - self.item_skipped = self.resume_items_skipped; - self.bytes_done = self.resume_bytes_done; - self.bytes_failed = self.resume_bytes_failed; - self.bytes_skipped = self.resume_bytes_skipped; - } - - pub async fn bucket_done(&mut self, bucket: &str) { - let _ = self.mu.write().await; - - self.resume_items_healed = self.items_healed; - self.resume_items_failed = self.items_failed; - self.resume_items_skipped = self.item_skipped; - self.resume_bytes_done = self.bytes_done; - self.resume_bytes_failed = self.bytes_failed; - self.resume_bytes_skipped = self.bytes_skipped; - self.healed_buckets.push(bucket.to_string()); - - self.queue_buckets.retain(|x| x != bucket); - } - - pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { - let _ = self.mu.write().await; - - buckets.iter().for_each(|bucket| { - if !self.healed_buckets.contains(&bucket.name) { - self.queue_buckets.push(bucket.name.clone()); - } - }); - } - - pub async fn to_healing_disk(&self) -> rustfs_madmin::HealingDisk { - let _ = self.mu.read().await; - - rustfs_madmin::HealingDisk { - id: self.id.clone(), - heal_id: self.heal_id.clone(), - pool_index: self.pool_index, - set_index: self.set_index, - disk_index: self.disk_index, - endpoint: self.endpoint.clone(), - path: self.path.clone(), - started: self.started, - last_update: self.last_update, - retry_attempts: self.retry_attempts, - objects_total_count: self.objects_total_count, - objects_total_size: self.objects_total_size, - items_healed: self.items_healed, - items_failed: self.items_failed, - item_skipped: self.item_skipped, - bytes_done: self.bytes_done, - bytes_failed: self.bytes_failed, - bytes_skipped: self.bytes_skipped, - objects_healed: self.items_healed, - objects_failed: self.items_failed, - bucket: self.bucket.clone(), - object: self.object.clone(), - queue_buckets: self.queue_buckets.clone(), - healed_buckets: self.healed_buckets.clone(), - finished: self.finished, - } - } -} - -impl Clone for HealingTracker { - fn clone(&self) -> Self { - Self { - disk: self.disk.clone(), - id: self.id.clone(), - pool_index: self.pool_index, - set_index: self.set_index, - disk_index: self.disk_index, - path: self.path.clone(), - endpoint: self.endpoint.clone(), - started: self.started, - last_update: self.last_update, - objects_total_count: self.objects_total_count, - objects_total_size: self.objects_total_size, - items_healed: self.items_healed, - items_failed: self.items_failed, - item_skipped: self.item_skipped, - bytes_done: self.bytes_done, - bytes_failed: self.bytes_failed, - bytes_skipped: self.bytes_skipped, - bucket: self.bucket.clone(), - object: self.object.clone(), - resume_items_healed: self.resume_items_healed, - resume_items_failed: self.resume_items_failed, - resume_items_skipped: self.resume_items_skipped, - resume_bytes_done: self.resume_bytes_done, - resume_bytes_failed: self.resume_bytes_failed, - resume_bytes_skipped: self.resume_bytes_skipped, - queue_buckets: self.queue_buckets.clone(), - healed_buckets: self.healed_buckets.clone(), - heal_id: self.heal_id.clone(), - retry_attempts: self.retry_attempts, - finished: self.finished, - mu: RwLock::new(false), - } - } -} - -pub async fn load_healing_tracker(disk: &Option) -> disk::error::Result { - if let Some(disk) = disk { - let disk_id = disk.get_disk_id().await?; - if let Some(disk_id) = disk_id { - let disk_id = disk_id.to_string(); - let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - let data = disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await?; - let mut healing_tracker = HealingTracker::unmarshal_msg(&data)?; - if healing_tracker.id != disk_id && !healing_tracker.id.is_empty() { - return Err(DiskError::other(format!( - "loadHealingTracker: drive id mismatch expected {}, got {}", - healing_tracker.id, disk_id - ))); - } - healing_tracker.id = disk_id; - healing_tracker.disk = Some(disk.clone()); - Ok(healing_tracker) - } else { - Err(DiskError::other("loadHealingTracker: disk not have id")) - } - } else { - Err(DiskError::other("loadHealingTracker: nil drive given")) - } -} - -pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> disk::error::Result { - let disk_location = disk.get_disk_location(); - Ok(HealingTracker { - id: disk - .get_disk_id() - .await - .map_or("".to_string(), |id| id.map_or("".to_string(), |id| id.to_string())), - heal_id: heal_id.to_string(), - path: disk.to_string(), - endpoint: disk.endpoint().to_string(), - started: Some(OffsetDateTime::now_utc()), - pool_index: disk_location.pool_idx, - set_index: disk_location.set_idx, - disk_index: disk_location.disk_idx, - disk: Some(disk), - ..Default::default() - }) -} - -pub async fn healing(derive_path: &str) -> disk::error::Result> { - let healing_file = Path::new(derive_path) - .join(RUSTFS_META_BUCKET) - .join(BUCKET_META_PREFIX) - .join(HEALING_TRACKER_FILENAME); - - let b = read_file(healing_file).await?; - if b.is_empty() { - return Ok(None); - } - - let healing_tracker = HealingTracker::unmarshal_msg(&b)?; - - Ok(Some(healing_tracker)) -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct MRFStatus { - bytes_healed: u64, - items_healed: u64, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct SetStatus { - pub id: String, - pub pool_index: i32, - pub set_index: i32, - pub heal_status: String, - pub heal_priority: String, - pub total_objects: usize, - pub disks: Vec, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct BgHealState { - offline_endpoints: Vec, - scanned_items_count: u64, - heal_disks: Vec, - sets: Vec, - mrf: HashMap, - scparity: HashMap, -} - -pub async fn get_local_background_heal_status() -> (BgHealState, bool) { - let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await; - if !ok { - return (BgHealState::default(), false); - } - let bg_seq = bg_seq.unwrap(); - let mut status = BgHealState { - scanned_items_count: bg_seq.get_scanned_items_count().await as u64, - ..Default::default() - }; - let mut heal_disks_map = HashSet::new(); - for ep in get_local_disks_to_heal().await.iter() { - heal_disks_map.insert(ep.to_string()); - } - - let Some(store) = new_object_layer_fn() else { - let healing = GLOBAL_BackgroundHealState.get_local_healing_disks().await; - for disk in healing.values() { - status.heal_disks.push(disk.endpoint.clone()); - } - return (status, true); - }; - - let si = store.local_storage_info().await; - let mut indexed = HashMap::new(); - for disk in si.disks.iter() { - let set_idx = format!("{}-{}", disk.pool_index, disk.set_index); - // indexed.insert(set_idx, disk); - indexed.entry(set_idx).or_insert(Vec::new()).push(disk); - } - - for (id, disks) in indexed { - let mut ss = SetStatus { - id, - set_index: disks[0].set_index, - pool_index: disks[0].pool_index, - ..Default::default() - }; - for disk in disks { - ss.disks.push(disk.clone()); - if disk.healing { - ss.heal_status = "healing".to_string(); - ss.heal_priority = "high".to_string(); - status.heal_disks.push(disk.endpoint.clone()); - } - } - ss.disks.sort_by(|a, b| { - if a.pool_index != b.pool_index { - return a.pool_index.cmp(&b.pool_index); - } - if a.set_index != b.set_index { - return a.set_index.cmp(&b.set_index); - } - a.disk_index.cmp(&b.disk_index) - }); - status.sets.push(ss); - } - status.sets.sort_by(|a, b| a.id.cmp(&b.id)); - let backend_info = store.backend_info().await; - status - .scparity - .insert(STANDARD.to_string(), backend_info.standard_sc_parity.unwrap_or_default()); - status - .scparity - .insert(RRS.to_string(), backend_info.rr_sc_parity.unwrap_or_default()); - - (status, true) -} diff --git a/crates/ecstore/src/heal/heal_ops.rs b/crates/ecstore/src/heal/heal_ops.rs deleted file mode 100644 index 2ee4aee10..000000000 --- a/crates/ecstore/src/heal/heal_ops.rs +++ /dev/null @@ -1,842 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - background_heal_ops::HealTask, - data_scanner::HEAL_DELETE_DANGLING, - error::ERR_SKIP_FILE, - heal_commands::{HEAL_ITEM_BUCKET_METADATA, HealOpts, HealScanMode, HealStopSuccess, HealingTracker}, -}; -use crate::error::{Error, Result}; -use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}; -use crate::store_api::StorageAPI; -use crate::{ - config::com::CONFIG_PREFIX, - disk::RUSTFS_META_BUCKET, - global::GLOBAL_BackgroundHealRoutine, - heal::{error::ERR_HEAL_STOP_SIGNALLED, heal_commands::DRIVE_STATE_OK}, -}; -use crate::{ - disk::endpoint::Endpoint, - endpoints::Endpoints, - global::GLOBAL_IsDistErasure, - heal::heal_commands::{HEAL_UNKNOWN_SCAN, HealStartSuccess}, - new_object_layer_fn, -}; -use chrono::Utc; -use futures::join; -use lazy_static::lazy_static; -use rustfs_filemeta::MetaCacheEntry; -use rustfs_madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem}; -use rustfs_utils::path::has_prefix; -use rustfs_utils::path::path_join; -use serde::{Deserialize, Serialize}; -use std::{ - collections::HashMap, - future::Future, - path::PathBuf, - pin::Pin, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; -use tokio::{ - select, spawn, - sync::{ - RwLock, broadcast, - mpsc::{self, Receiver as M_Receiver, Sender as M_Sender}, - watch::{self, Receiver as W_Receiver, Sender as W_Sender}, - }, - time::{interval, sleep}, -}; -use tracing::{error, info}; -use uuid::Uuid; - -type HealStatusSummary = String; -type ItemsMap = HashMap; -pub type HealEntryFn = - Arc Pin> + Send>> + Send + Sync + 'static>; - -pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000"; -pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin"; -const KEEP_HEAL_SEQ_STATE_DURATION: Duration = Duration::from_secs(10 * 60); -const HEAL_NOT_STARTED_STATUS: &str = "not started"; -const HEAL_RUNNING_STATUS: &str = "running"; -const HEAL_STOPPED_STATUS: &str = "stopped"; -const HEAL_FINISHED_STATUS: &str = "finished"; - -pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs"; -pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs"; -pub const LOGIN_PATH_PREFIX: &str = "/login"; - -const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000; -const HEAL_UNCONSUMED_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); -pub const NOP_HEAL: &str = ""; - -lazy_static! {} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct HealSequenceStatus { - pub summary: HealStatusSummary, - pub failure_detail: String, - pub start_time: u64, - pub heal_setting: HealOpts, - pub items: Vec, -} - -#[derive(Debug, Default)] -pub struct HealSource { - pub bucket: String, - pub object: String, - pub version_id: String, - pub no_wait: bool, - pub opts: Option, -} - -#[derive(Debug)] -pub struct HealSequence { - pub bucket: String, - pub object: String, - pub report_progress: bool, - pub start_time: SystemTime, - pub end_time: Arc>, - pub client_token: String, - pub client_address: String, - pub force_started: bool, - pub setting: HealOpts, - pub current_status: Arc>, - pub last_sent_result_index: RwLock, - pub scanned_items_map: RwLock, - pub healed_items_map: RwLock, - pub heal_failed_items_map: RwLock, - pub last_heal_activity: RwLock, - - traverse_and_heal_done_tx: Arc>>>, - traverse_and_heal_done_rx: Arc>>>, - - tx: W_Sender, - rx: W_Receiver, -} - -pub fn new_bg_heal_sequence() -> HealSequence { - let hs = HealOpts { - remove: HEAL_DELETE_DANGLING, - ..Default::default() - }; - - HealSequence { - start_time: SystemTime::now(), - client_token: BG_HEALING_UUID.to_string(), - bucket: RUSTFS_RESERVED_BUCKET.to_string(), - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - report_progress: false, - scanned_items_map: HashMap::new().into(), - healed_items_map: HashMap::new().into(), - heal_failed_items_map: HashMap::new().into(), - ..Default::default() - } -} - -pub fn new_heal_sequence(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> HealSequence { - let client_token = Uuid::new_v4().to_string(); - let (tx, rx) = mpsc::channel(10); - HealSequence { - bucket: bucket.to_string(), - object: obj_prefix.to_string(), - report_progress: true, - start_time: SystemTime::now(), - client_token, - client_address: client_addr.to_string(), - force_started: force_start, - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - traverse_and_heal_done_tx: Arc::new(RwLock::new(tx)), - traverse_and_heal_done_rx: Arc::new(RwLock::new(rx)), - scanned_items_map: HashMap::new().into(), - healed_items_map: HashMap::new().into(), - heal_failed_items_map: HashMap::new().into(), - ..Default::default() - } -} - -impl Default for HealSequence { - fn default() -> Self { - let (h_tx, h_rx) = mpsc::channel(1); - let (tx, rx) = watch::channel(false); - Self { - bucket: Default::default(), - object: Default::default(), - report_progress: Default::default(), - start_time: SystemTime::now(), - end_time: Arc::new(RwLock::new(SystemTime::now())), - client_token: Default::default(), - client_address: Default::default(), - force_started: Default::default(), - setting: Default::default(), - current_status: Default::default(), - last_sent_result_index: Default::default(), - scanned_items_map: Default::default(), - healed_items_map: Default::default(), - heal_failed_items_map: Default::default(), - last_heal_activity: RwLock::new(SystemTime::now()), - traverse_and_heal_done_tx: Arc::new(RwLock::new(h_tx)), - traverse_and_heal_done_rx: Arc::new(RwLock::new(h_rx)), - tx, - rx, - } - } -} - -impl HealSequence { - pub fn new(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self { - let client_token = Uuid::new_v4().to_string(); - - Self { - bucket: bucket.to_string(), - object: obj_prefix.to_string(), - report_progress: true, - client_token, - client_address: client_addr.to_string(), - force_started: force_start, - setting: hs, - current_status: Arc::new(RwLock::new(HealSequenceStatus { - summary: HEAL_NOT_STARTED_STATUS.to_string(), - heal_setting: hs, - ..Default::default() - })), - ..Default::default() - } - } -} - -impl HealSequence { - pub async fn get_scanned_items_count(&self) -> usize { - self.scanned_items_map.read().await.values().sum() - } - - async fn _get_scanned_items_map(&self) -> ItemsMap { - self.scanned_items_map.read().await.clone() - } - - async fn _get_healed_items_map(&self) -> ItemsMap { - self.healed_items_map.read().await.clone() - } - - async fn _get_heal_failed_items_map(&self) -> ItemsMap { - self.heal_failed_items_map.read().await.clone() - } - - pub async fn count_failed(&self, heal_type: HealItemType) { - *self.heal_failed_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - pub async fn count_scanned(&self, heal_type: HealItemType) { - *self.scanned_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - pub async fn count_healed(&self, heal_type: HealItemType) { - *self.healed_items_map.write().await.entry(heal_type).or_insert(0) += 1; - *self.last_heal_activity.write().await = SystemTime::now(); - } - - async fn is_quitting(&self) -> bool { - if let Ok(true) = self.rx.has_changed() { - info!("quited"); - return true; - } - false - } - - async fn has_ended(&self) -> bool { - if self.client_token == *BG_HEALING_UUID { - return false; - } - - *(self.end_time.read().await) != self.start_time - } - - async fn stop(&self) { - let _ = self.tx.send(true); - } - - async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> { - let mut r = r.clone(); - let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT); - #[allow(unused_assignments)] - let mut items_len = 0; - loop { - { - let current_status_r = self.current_status.read().await; - items_len = current_status_r.items.len(); - } - - if items_len == MAX_UNCONSUMED_HEAL_RESULT_ITEMS { - select! { - _ = sleep(Duration::from_secs(1)) => { - - } - _ = self.is_done() => { - return Err(Error::other("stopped")); - } - _ = interval_timer.tick() => { - return Err(Error::other("timeout")); - } - } - } else { - break; - } - } - - let mut current_status_w = self.current_status.write().await; - if items_len > 0 { - r.result_index = 1 + current_status_w.items[items_len - 1].result_index; - } else { - r.result_index = 1 + *self.last_sent_result_index.read().await; - } - - current_status_w.items.push(r); - - Ok(()) - } - - pub async fn queue_heal_task(&self, source: HealSource, heal_type: HealItemType) -> Result<()> { - let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting); - info!("queue_heal_task, {:?}", task); - if let Some(opts) = source.opts { - task.opts = opts; - } else { - task.opts.scan_mode = HEAL_UNKNOWN_SCAN; - } - - self.count_scanned(heal_type.clone()).await; - - if source.no_wait { - let task_str = format!("{task:?}"); - if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() { - info!("Task in the queue: {:?}", task_str); - } - return Ok(()); - } - - let (resp_tx, mut resp_rx) = mpsc::channel(1); - task.resp_tx = Some(resp_tx); - - let task_str = format!("{task:?}"); - if GLOBAL_BackgroundHealRoutine.tasks_tx.try_send(task).is_ok() { - info!("Task in the queue: {:?}", task_str); - } else { - error!("push task to queue failed"); - } - let count_ok_drives = |drivers: &[HealDriveInfo]| { - let mut count = 0; - for drive in drivers.iter() { - if drive.state == DRIVE_STATE_OK { - count += 1; - } - } - count - }; - - match resp_rx.recv().await { - Some(mut res) => { - if res.err.is_none() { - self.count_healed(heal_type.clone()).await; - } else { - self.count_failed(heal_type.clone()).await; - } - if !self.report_progress { - return if let Some(err) = res.err { - if err.to_string() == ERR_SKIP_FILE { - return Ok(()); - } - Err(err) - } else { - Ok(()) - }; - } - res.result.heal_item_type = heal_type.clone(); - if let Some(err) = res.err.as_ref() { - res.result.detail = err.to_string(); - } - if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks - { - let got = count_ok_drives(&res.result.after.drives); - if got < res.result.parity_blocks { - res.result.detail = format!( - "quorum loss - expected {} minimum, got drive states in OK {}", - res.result.parity_blocks, got - ); - } - } - - info!("queue_heal_task, HealResult: {:?}", res); - self.push_heal_result_item(&res.result).await - } - None => Ok(()), - } - } - - async fn heal_disk_meta(h: Arc) -> Result<()> { - HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await - } - - async fn heal_items(h: Arc, buckets_only: bool) -> Result<()> { - if h.client_token == *BG_HEALING_UUID { - return Ok(()); - } - - let bucket = h.bucket.clone(); - let task1 = Self::heal_disk_meta(h.clone()); - let task2 = Self::heal_bucket(h.clone(), &bucket, buckets_only); - let results = join!(task1, task2); - results.0?; - results.1?; - - Ok(()) - } - - async fn traverse_and_heal(h: Arc) { - let buckets_only = false; - let result = Self::heal_items(h.clone(), buckets_only).await.err(); - let _ = h.traverse_and_heal_done_tx.read().await.send(result).await; - } - - async fn heal_rustfs_sys_meta(h: Arc, meta_prefix: &str) -> Result<()> { - info!("heal_rustfs_sys_meta, h: {:?}", h); - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - let setting = h.setting; - store - .heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true) - .await - } - - async fn is_done(&self) -> bool { - if let Ok(true) = self.rx.has_changed() { - return true; - } - false - } - - pub async fn heal_bucket(hs: Arc, bucket: &str, bucket_only: bool) -> Result<()> { - info!("heal_bucket, hs: {:?}", hs); - let (object, setting) = { - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET.to_string(), - ) - .await?; - - if bucket_only { - return Ok(()); - } - - if !hs.setting.recursive { - if !hs.object.is_empty() { - HealSequence::heal_object(hs.clone(), bucket, &hs.object, "", hs.setting.scan_mode).await?; - } - return Ok(()); - } - (hs.object.clone(), hs.setting) - }; - let Some(store) = new_object_layer_fn() else { - return Err(Error::other("errServerNotInitialized")); - }; - store.heal_objects(bucket, &object, &setting, hs.clone(), false).await - } - - pub async fn heal_object( - hs: Arc, - bucket: &str, - object: &str, - version_id: &str, - _scan_mode: HealScanMode, - ) -> Result<()> { - info!("heal_object"); - if hs.is_quitting().await { - info!("heal_object hs is quitting"); - return Err(Error::other(ERR_HEAL_STOP_SIGNALLED)); - } - - info!("will queue task"); - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - opts: Some(hs.setting), - ..Default::default() - }, - HEAL_ITEM_OBJECT.to_string(), - ) - .await?; - - Ok(()) - } - - pub async fn heal_meta_object( - hs: Arc, - bucket: &str, - object: &str, - version_id: &str, - _scan_mode: HealScanMode, - ) -> Result<()> { - if hs.is_quitting().await { - return Err(Error::other(ERR_HEAL_STOP_SIGNALLED)); - } - - hs.queue_heal_task( - HealSource { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - ..Default::default() - }, - HEAL_ITEM_BUCKET_METADATA.to_string(), - ) - .await?; - - Ok(()) - } -} - -pub async fn heal_sequence_start(h: Arc) { - { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_RUNNING_STATUS.to_string(); - current_status_w.start_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - } - - let h_clone = h.clone(); - spawn(async move { - HealSequence::traverse_and_heal(h_clone).await; - }); - - let h_clone_1 = h.clone(); - let mut x = h.traverse_and_heal_done_rx.write().await; - select! { - _ = h.is_done() => { - *(h.end_time.write().await) = SystemTime::now(); - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_FINISHED_STATUS.to_string(); - - spawn(async move { - let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await; - rx_w.recv().await; - }); - } - result = x.recv() => { - if let Some(err) = result { - match err { - Some(err) => { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_STOPPED_STATUS.to_string(); - current_status_w.failure_detail = err.to_string(); - }, - None => { - let mut current_status_w = h.current_status.write().await; - current_status_w.summary = HEAL_FINISHED_STATUS.to_string(); - } - } - } - - } - } -} - -#[derive(Debug, Default)] -pub struct AllHealState { - mu: RwLock, - - heal_seq_map: RwLock>>, - heal_local_disks: RwLock>, - heal_status: RwLock>, -} - -impl AllHealState { - pub fn new(cleanup: bool) -> Arc { - let state = Arc::new(AllHealState::default()); - let (_, mut rx) = broadcast::channel(1); - if cleanup { - let state_clone = state.clone(); - spawn(async move { - loop { - select! { - result = rx.recv() =>{ - if let Ok(true) = result { - return; - } - } - _ = sleep(Duration::from_secs(5 * 60)) => { - state_clone.periodic_heal_seqs_clean().await; - } - } - } - }); - } - - state - } - - pub async fn pop_heal_local_disks(&self, heal_local_disks: &[Endpoint]) { - let _ = self.mu.write().await; - - self.heal_local_disks.write().await.retain(|k, _| { - if heal_local_disks.contains(k) { - return false; - } - true - }); - - let heal_local_disks = heal_local_disks.iter().map(|s| s.to_string()).collect::>(); - self.heal_status.write().await.retain(|_, v| { - if heal_local_disks.contains(&v.endpoint) { - return false; - } - - true - }); - } - - pub async fn pop_heal_status_json(&self, heal_path: &str, client_token: &str) -> Result> { - match self.get_heal_sequence(heal_path).await { - Some(h) => { - if client_token != h.client_token { - info!("err heal invalid client token"); - return Err(Error::other("err heal invalid client token")); - } - let num_items = h.current_status.read().await.items.len(); - let mut last_result_index = *h.last_sent_result_index.read().await; - if num_items > 0 { - if let Some(item) = h.current_status.read().await.items.last() { - last_result_index = item.result_index; - } - } - *h.last_sent_result_index.write().await = last_result_index; - let data = h.current_status.read().await.clone(); - match serde_json::to_vec(&data) { - Ok(b) => { - h.current_status.write().await.items.clear(); - Ok(b) - } - Err(e) => { - h.current_status.write().await.items.clear(); - info!("json encode err, e: {}", e); - Err(Error::other(e.to_string())) - } - } - } - None => serde_json::to_vec(&HealSequenceStatus { - summary: HEAL_FINISHED_STATUS.to_string(), - ..Default::default() - }) - .map_err(|e| { - info!("json encode err, e: {}", e); - Error::other(e.to_string()) - }), - } - } - - pub async fn update_heal_status(&self, tracker: &HealingTracker) { - let _ = self.mu.write().await; - let _ = tracker.mu.read().await; - - self.heal_status.write().await.insert(tracker.id.clone(), tracker.clone()); - } - - pub async fn get_local_healing_disks(&self) -> HashMap { - let _ = self.mu.read().await; - - let mut dst = HashMap::new(); - for v in self.heal_status.read().await.values() { - dst.insert(v.endpoint.clone(), v.to_healing_disk().await); - } - - dst - } - - pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints { - let _ = self.mu.read().await; - - let mut endpoints = Vec::new(); - self.heal_local_disks.read().await.iter().for_each(|(k, v)| { - if !v { - endpoints.push(k.clone()); - } - }); - - Endpoints::from(endpoints) - } - - pub async fn set_disk_healing_status(&self, ep: Endpoint, healing: bool) { - let _ = self.mu.write().await; - - self.heal_local_disks.write().await.insert(ep, healing); - } - - pub async fn push_heal_local_disks(&self, heal_local_disks: &[Endpoint]) { - let _ = self.mu.write().await; - - for heal_local_disk in heal_local_disks.iter() { - self.heal_local_disks.write().await.insert(heal_local_disk.clone(), false); - } - } - - pub async fn periodic_heal_seqs_clean(&self) { - let _ = self.mu.write().await; - let now = SystemTime::now(); - - let mut keys_to_remove = Vec::new(); - for (k, v) in self.heal_seq_map.read().await.iter() { - if v.has_ended().await && now.duration_since(*(v.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION { - keys_to_remove.push(k.clone()) - } - } - for key in keys_to_remove.iter() { - self.heal_seq_map.write().await.remove(key); - } - } - - pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option>, bool) { - let _ = self.mu.read().await; - - for v in self.heal_seq_map.read().await.values() { - if v.client_token == token { - return (Some(v.clone()), true); - } - } - - (None, false) - } - - pub async fn get_heal_sequence(&self, path: &str) -> Option> { - let _ = self.mu.read().await; - - self.heal_seq_map.read().await.get(path).cloned() - } - - pub async fn stop_heal_sequence(&self, path: &str) -> Result> { - let mut hsp = HealStopSuccess::default(); - if let Some(he) = self.get_heal_sequence(path).await { - let client_token = he.client_token.clone(); - if *GLOBAL_IsDistErasure.read().await { - // TODO: proxy - } - - hsp.client_token = client_token; - hsp.client_address = he.client_address.clone(); - hsp.start_time = Utc::now(); - - he.stop().await; - - loop { - if he.has_ended().await { - break; - } - - sleep(Duration::from_secs(1)).await; - } - - let _ = self.mu.write().await; - self.heal_seq_map.write().await.remove(path); - } else { - hsp.client_token = "unknown".to_string(); - } - - let b = serde_json::to_string(&hsp)?; - Ok(b.as_bytes().to_vec()) - } - - // LaunchNewHealSequence - launches a background routine that performs - // healing according to the healSequence argument. For each heal - // sequence, state is stored in the `globalAllHealState`, which is a - // map of the heal path to `healSequence` which holds state about the - // heal sequence. - // - // Heal results are persisted in server memory for - // `keepHealSeqStateDuration`. This function also launches a - // background routine to clean up heal results after the - // aforementioned duration. - pub async fn launch_new_heal_sequence(&self, heal_sequence: Arc) -> Result> { - let path = path_join(&[ - PathBuf::from(heal_sequence.bucket.clone()), - PathBuf::from(heal_sequence.object.clone()), - ]); - let path_s = path.to_str().unwrap(); - if heal_sequence.force_started { - self.stop_heal_sequence(path_s).await?; - } else if let Some(hs) = self.get_heal_sequence(path_s).await { - if !hs.has_ended().await { - return Err(Error::other(format!( - "Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", - heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token - ))); - } - } - - let _ = self.mu.write().await; - - for (k, v) in self.heal_seq_map.read().await.iter() { - if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await { - return Err(Error::other(format!( - "The provided heal sequence path overlaps with an existing heal path: {k}" - ))); - } - } - - self.heal_seq_map - .write() - .await - .insert(path_s.to_string(), heal_sequence.clone()); - - let client_token = heal_sequence.client_token.clone(); - if *GLOBAL_IsDistErasure.read().await { - // TODO: proxy - } - - if heal_sequence.client_token == BG_HEALING_UUID { - // For background heal do nothing, do not spawn an unnecessary goroutine. - } else { - let heal_sequence_clone = heal_sequence.clone(); - spawn(async { - heal_sequence_start(heal_sequence_clone).await; - }); - } - - let b = serde_json::to_vec(&HealStartSuccess { - client_token, - client_address: heal_sequence.client_address.clone(), - // start_time: Utc::now(), - start_time: heal_sequence.start_time.into(), - })?; - Ok(b) - } -} diff --git a/crates/ecstore/src/heal/mod.rs b/crates/ecstore/src/heal/mod.rs deleted file mode 100644 index 6b34dc620..000000000 --- a/crates/ecstore/src/heal/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// pub mod background_heal_ops; -// pub mod data_scanner; -// pub mod data_scanner_metric; -// pub mod data_usage; -// pub mod data_usage_cache; -// pub mod error; -// pub mod heal_commands; -// pub mod heal_ops; -// pub mod mrf; diff --git a/crates/ecstore/src/heal/mrf.rs b/crates/ecstore/src/heal/mrf.rs deleted file mode 100644 index 3f42fa202..000000000 --- a/crates/ecstore/src/heal/mrf.rs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; -use crate::heal::background_heal_ops::{heal_bucket, heal_object}; -use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN}; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; -use regex::Regex; -use rustfs_utils::path::SLASH_SEPARATOR; -use std::ops::Sub; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; -use tokio::sync::RwLock; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::time::sleep; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; -use uuid::Uuid; - -pub const MRF_OPS_QUEUE_SIZE: u64 = 100000; -pub const HEAL_DIR: &str = ".heal"; -pub const HEAL_MRFMETA_FORMAT: u64 = 1; -pub const HEAL_MRFMETA_VERSION_V1: u64 = 1; - -lazy_static! { - pub static ref HEAL_MRF_DIR: String = - format!("{}{}{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, HEAL_DIR, SLASH_SEPARATOR, "mrf"); - static ref PATTERNS: Vec = vec![ - Regex::new(r"^buckets/.*/.metacache/.*").unwrap(), - Regex::new(r"^tmp/.*").unwrap(), - Regex::new(r"^multipart/.*").unwrap(), - Regex::new(r"^tmp-old/.*").unwrap(), - ]; -} - -#[derive(Default)] -pub struct PartialOperation { - pub bucket: String, - pub object: String, - pub version_id: Option, - pub versions: Vec, - pub set_index: usize, - pub pool_index: usize, - pub queued: DateTime, - pub bitrot_scan: bool, -} - -pub struct MRFState { - tx: Sender, - rx: RwLock>, - closed: AtomicBool, - closing: AtomicBool, -} - -impl Default for MRFState { - fn default() -> Self { - Self::new() - } -} - -impl MRFState { - pub fn new() -> MRFState { - let (tx, rx) = tokio::sync::mpsc::channel(MRF_OPS_QUEUE_SIZE as usize); - MRFState { - tx, - rx: RwLock::new(rx), - closed: Default::default(), - closing: Default::default(), - } - } - - pub async fn add_partial(&self, op: PartialOperation) { - if self.closed.load(Ordering::SeqCst) || self.closing.load(Ordering::SeqCst) { - return; - } - let _ = self.tx.send(op).await; - } - - /// Enhanced heal routine with cancellation support - /// - /// This method implements the same healing logic as the original heal_routine, - /// but adds proper cancellation support via CancellationToken. - /// The core logic remains identical to maintain compatibility. - pub async fn heal_routine_with_cancel(&self, cancel_token: CancellationToken) { - info!("MRF heal routine started with cancellation support"); - - loop { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("MRF heal routine received shutdown signal, exiting gracefully"); - break; - } - op_result = async { - let mut rx_guard = self.rx.write().await; - rx_guard.recv().await - } => { - if let Some(op) = op_result { - // Special path filtering (original logic) - if op.bucket == RUSTFS_META_BUCKET { - for pattern in &*PATTERNS { - if pattern.is_match(&op.object) { - continue; // Skip this operation, continue with next - } - } - } - - // Network reconnection delay (original logic) - let now = Utc::now(); - if now.sub(op.queued).num_seconds() < 1 { - tokio::select! { - _ = cancel_token.cancelled() => { - info!("MRF heal routine cancelled during reconnection delay"); - break; - } - _ = sleep(Duration::from_secs(1)) => {} - } - } - - // Core healing logic (original logic preserved) - let scan_mode = if op.bitrot_scan { HEAL_DEEP_SCAN } else { HEAL_NORMAL_SCAN }; - - if op.object.is_empty() { - // Heal bucket (original logic) - if let Err(err) = heal_bucket(&op.bucket).await { - error!("heal bucket failed, bucket: {}, err: {:?}", op.bucket, err); - } - } else if op.versions.is_empty() { - // Heal single object (original logic) - if let Err(err) = heal_object( - &op.bucket, - &op.object, - &op.version_id.clone().unwrap_or_default(), - scan_mode - ).await { - error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err); - } - } else { - // Heal multiple versions (original logic) - let vers = op.versions.len() / 16; - if vers > 0 { - for i in 0..vers { - // Check for cancellation before each version - if cancel_token.is_cancelled() { - info!("MRF heal routine cancelled during version processing"); - return; - } - - let start = i * 16; - let end = start + 16; - if let Err(err) = heal_object( - &op.bucket, - &op.object, - &Uuid::from_slice(&op.versions[start..end]).expect("").to_string(), - scan_mode, - ).await { - error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err); - } - } - } - } - } else { - info!("MRF heal routine channel closed, exiting"); - break; - } - } - } - } - - info!("MRF heal routine stopped gracefully"); - } -} diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 177618892..770e04027 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -30,7 +30,6 @@ pub mod endpoints; pub mod erasure_coding; pub mod error; pub mod global; -pub mod heal; pub mod lock_utils; pub mod metrics_realtime; pub mod notification_sys;