diff --git a/Cargo.lock b/Cargo.lock index f9bef4b33..92a93536a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9899,14 +9899,18 @@ name = "rustfs-scanner" version = "1.0.0-beta.11" dependencies = [ "async-trait", + "bytes", "chrono", "futures", + "hex-simd", + "hmac 0.13.0", "http 1.4.2", "metrics", "rand 0.10.2", "rmp-serde", "rustfs-common", "rustfs-config", + "rustfs-credentials", "rustfs-data-usage", "rustfs-ecstore", "rustfs-filemeta", @@ -9916,7 +9920,9 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sha2 0.11.0", "temp-env", + "tempfile", "thiserror 2.0.19", "time", "tokio", diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 0f218c8c3..29213edeb 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -768,6 +768,7 @@ pub struct Metrics { last_scan_cycle_replication_checks: AtomicU64, last_scan_cycle_usage_saves: AtomicU64, failed_scan_cycles: AtomicU64, + superseded_scan_cycles: AtomicU64, partial_scan_cycles_unknown: AtomicU64, partial_scan_cycles_runtime: AtomicU64, partial_scan_cycles_objects: AtomicU64, @@ -833,10 +834,12 @@ const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0; const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1; const SCAN_CYCLE_RESULT_ERROR: u8 = 2; const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3; +const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4; const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown"; const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success"; const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error"; const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial"; +const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded"; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum ScanCyclePartialReason { @@ -1208,6 +1211,8 @@ pub struct ScannerMetricsReport { pub last_cycle_usage_saves: u64, pub failed_cycles: u64, #[serde(default)] + pub superseded_cycles: u64, + #[serde(default)] pub partial_cycles_unknown: u64, #[serde(default)] pub partial_cycles_runtime: u64, @@ -1310,6 +1315,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str { SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL, SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL, SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL, + SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL, _ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL, } } @@ -1633,6 +1639,11 @@ pub fn emit_scan_cycle_partial_with_source( metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1); } +pub fn emit_scan_cycle_superseded(duration: Duration) { + global_metrics().record_scan_cycle_superseded(duration); + metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1); +} + pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) { let result = if success { "success" } else { "error" }; metrics::counter!( @@ -1726,6 +1737,7 @@ impl Metrics { last_scan_cycle_replication_checks: AtomicU64::new(0), last_scan_cycle_usage_saves: AtomicU64::new(0), failed_scan_cycles: AtomicU64::new(0), + superseded_scan_cycles: AtomicU64::new(0), partial_scan_cycles_unknown: AtomicU64::new(0), partial_scan_cycles_runtime: AtomicU64::new(0), partial_scan_cycles_objects: AtomicU64::new(0), @@ -2349,6 +2361,18 @@ impl Metrics { .store(duration_millis_saturated(duration), Ordering::Relaxed); } + pub fn record_scan_cycle_superseded(&self, duration: Duration) { + self.record_scanner_cycle_end_time(); + self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed); + self.last_scan_cycle_result + .store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed); + self.last_scan_cycle_partial_reason + .store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed); + self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed); + self.last_scan_cycle_duration_millis + .store(duration_millis_saturated(duration), Ordering::Relaxed); + } + pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) { self.record_scan_cycle_partial_with_source(duration, reason, None); } @@ -2802,6 +2826,7 @@ impl Metrics { m.last_cycle_replication_repair = self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work); m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed); + m.superseded_cycles = self.superseded_scan_cycles.load(Ordering::Relaxed); m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed); m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed); m.partial_cycles_objects = self.partial_scan_cycles_objects.load(Ordering::Relaxed); @@ -2950,19 +2975,15 @@ pub type CloseDiskFn = Arc Pin + Send>> /// Register a new disk in the global path tracker and return two callbacks: /// one to update the current path and one to deregister the disk when done. -pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) { +pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) { let tracker = Arc::new(CurrentPathTracker::new(initial.to_string())); let disk_name = disk.to_string(); - let tracker_clone = Arc::clone(&tracker); - let disk_insert = disk_name.clone(); - tokio::spawn(async move { - global_metrics() - .current_paths - .write() - .await - .insert(disk_insert, tracker_clone); - }); + global_metrics() + .current_paths + .write() + .await + .insert(disk_name.clone(), Arc::clone(&tracker)); let update_fn: UpdateCurrentPathFn = { let tracker = Arc::clone(&tracker); @@ -2990,23 +3011,28 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, // CloseDiskGuard // --------------------------------------------------------------------------- -pub struct CloseDiskGuard(CloseDiskFn); +pub struct CloseDiskGuard(Option); impl CloseDiskGuard { pub fn new(close_disk: CloseDiskFn) -> Self { - Self(close_disk) + Self(Some(close_disk)) } - pub async fn close(&self) { - self.0().await; + pub async fn close(&mut self) { + let Some(close_disk) = self.0.clone() else { + return; + }; + close_disk().await; + self.0 = None; } } impl Drop for CloseDiskGuard { fn drop(&mut self) { - if let Ok(handle) = tokio::runtime::Handle::try_current() { - let close_fn = self.0.clone(); - handle.spawn(async move { close_fn().await }); + if let Some(close_disk) = self.0.take() + && let Ok(handle) = tokio::runtime::Handle::try_current() + { + handle.spawn(close_disk()); } // If there is no runtime we are in a test or shutdown path; skip cleanup. } @@ -3016,6 +3042,61 @@ impl Drop for CloseDiskGuard { mod tests { use super::*; + #[tokio::test] + async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() { + let (closed_tx, closed_rx) = tokio::sync::oneshot::channel(); + let closed_tx = Arc::new(std::sync::Mutex::new(Some(closed_tx))); + let close_disk: CloseDiskFn = { + let closed_tx = Arc::clone(&closed_tx); + Arc::new(move || { + let closed_tx = closed_tx.lock().expect("close callback lock").take(); + Box::pin(async move { + if let Some(closed_tx) = closed_tx { + let _ = closed_tx.send(()); + } + }) + }) + }; + + let guard = CloseDiskGuard::new(close_disk); + drop(guard); + + tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx) + .await + .expect("drop cleanup should run") + .expect("drop cleanup should signal"); + } + + #[tokio::test] + async fn close_disk_guard_runs_explicit_cleanup_once() { + let close_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let close_disk: CloseDiskFn = { + let close_count = Arc::clone(&close_count); + Arc::new(move || { + close_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Box::pin(std::future::ready(())) + }) + }; + + let mut guard = CloseDiskGuard::new(close_disk); + guard.close().await; + drop(guard); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + assert_eq!(close_count.load(std::sync::atomic::Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn current_path_updater_registers_before_return() { + let disk = format!("test-disk-{}", uuid::Uuid::new_v4()); + let (_update_path, close_disk) = current_path_updater(&disk, "bucket-a").await; + + assert!(global_metrics().current_paths.read().await.contains_key(&disk)); + + close_disk().await; + assert!(!global_metrics().current_paths.read().await.contains_key(&disk)); + } + #[tokio::test] async fn report_counts_active_scan_paths() { let metrics = Metrics::new(); @@ -3850,6 +3931,21 @@ mod tests { assert_eq!(report.failed_cycles, 1); } + #[tokio::test] + async fn report_tracks_superseded_cycle_without_failed_increment() { + let metrics = Metrics::new(); + metrics.record_scan_cycle_superseded(Duration::from_millis(750)); + + let report = metrics.report().await; + + assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUPERSEDED_LABEL); + assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_SUPERSEDED)); + assert_eq!(report.last_cycle_duration_seconds, 0.75); + assert_eq!(report.failed_cycles, 0); + assert_eq!(report.superseded_cycles, 1); + assert_eq!(report.partial_cycles, 0); + } + #[tokio::test] async fn report_tracks_successful_scan_cycle_without_failed_increment() { let metrics = Metrics::new(); diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index 350ef51a4..bcb730c2d 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -220,12 +220,10 @@ pub const ENV_SCANNER_YIELD_EVERY_N_OBJECTS: &str = "RUSTFS_SCANNER_YIELD_EVERY_ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true; /// Default set scan concurrency budget. -/// `0` means no additional limit beyond deployment topology. -pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 0; +pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 4; /// Default disk scan concurrency budget. -/// `0` means no additional limit beyond available disks in the set. -pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 0; +pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4; /// Default object interval for cooperative scanner yields. pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128; diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index 0373c8fd2..cca5d673c 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -32,6 +32,20 @@ use std::{ /// save forever and freeze admin usage stats; callers must bypass the skip instead. pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60); +/// Authoritative cluster-wide usage snapshot written by coordinated scanners. +/// +/// This object name is intentionally distinct from the legacy unfenced +/// snapshot. Older binaries can continue writing the legacy object during a +/// rolling upgrade without overwriting a snapshot produced by the current +/// scanner protocol. +pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json"; + +/// Usage snapshot written by scanner implementations predating distributed +/// leadership fencing. It is read only when neither authoritative snapshot +/// copy exists. +// RUSTFS_COMPAT_TODO(scanner-usage-v2): keep .usage.json readable and removable during rolling upgrades from pre-v2 scanners. Remove after supported direct-upgrade sources all write .usage.v2.json. +pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json"; + /// Returns true when `existing_last_update` is ahead of `now` by more than /// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be /// trusted for staleness comparisons and a fresh snapshot save must be allowed. @@ -95,7 +109,7 @@ impl AllTierStats { } /// Bucket target usage info provides replication statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct BucketTargetUsageInfo { pub replication_pending_size: u64, pub replication_failed_size: u64, @@ -107,7 +121,7 @@ pub struct BucketTargetUsageInfo { } /// Bucket usage info provides bucket-level statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct BucketUsageInfo { pub size: u64, // Following five fields suffixed with V1 are here for backward compatibility @@ -133,7 +147,7 @@ pub struct BucketUsageInfo { } /// DataUsageInfo represents data usage stats of the underlying storage -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct DataUsageInfo { /// Total capacity pub total_capacity: u64, @@ -145,6 +159,22 @@ pub struct DataUsageInfo { /// LastUpdate is the timestamp of when the data usage info was last updated pub last_update: Option, + /// Monotonic scanner cycle that produced this complete snapshot. + /// + /// Older snapshots omit this field and continue to use `last_update` for + /// compatibility. New scanner snapshots use the cycle to fence stale + /// leaders independently of wall-clock skew. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scanner_cycle: Option, + + /// Persisted scanner leadership epoch that produced this snapshot. + /// + /// The epoch is claimed through the cycle-state CAS before scanning. It + /// orders snapshots from different leaders even when their wall clocks or + /// cycle counters coincide. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scanner_epoch: Option, + /// Objects total count across all buckets pub objects_total_count: u64, /// Versions total count across all buckets @@ -168,7 +198,7 @@ pub struct DataUsageInfo { } /// Metadata describing the status of a disk-level data usage snapshot. -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct DiskUsageStatus { pub disk_id: String, pub pool_index: Option, @@ -268,12 +298,30 @@ impl DataUsageHash { pub type DataUsageHashMap = HashSet; /// Size histogram for object size distribution -#[derive(Clone, Debug, Serialize, Deserialize)] +const SIZE_HISTOGRAM_LEN: usize = 11; + +#[derive(Clone, Debug, Serialize)] pub struct SizeHistogram(Vec); impl Default for SizeHistogram { fn default() -> Self { - Self(vec![0; 11]) // DATA_USAGE_BUCKET_LEN = 11 + Self(vec![0; SIZE_HISTOGRAM_LEN]) + } +} + +impl<'de> Deserialize<'de> for SizeHistogram { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let values = Vec::::deserialize(deserializer)?; + if values.len() != SIZE_HISTOGRAM_LEN { + return Err(serde::de::Error::invalid_length( + values.len(), + &"exactly 11 object-size histogram buckets", + )); + } + Ok(Self(values)) } } @@ -343,7 +391,7 @@ impl SizeHistogram { .zip(names.iter()) .filter(|((_, (start, end)), name)| name != &&"BETWEEN_1024B_AND_1_MB" && *start >= 1024 && *end < ONE_MIB) .map(|((count, _), _)| *count) - .sum(); + .fold(0, u64::saturating_add); let mut res = HashMap::new(); for (count, name) in self.0.iter().zip(names.iter()) { @@ -364,12 +412,30 @@ impl SizeHistogram { } /// Versions histogram for version count distribution -#[derive(Clone, Debug, Serialize, Deserialize)] +const VERSIONS_HISTOGRAM_LEN: usize = 7; + +#[derive(Clone, Debug, Serialize)] pub struct VersionsHistogram(Vec); impl Default for VersionsHistogram { fn default() -> Self { - Self(vec![0; 7]) // DATA_USAGE_VERSION_LEN = 7 + Self(vec![0; VERSIONS_HISTOGRAM_LEN]) + } +} + +impl<'de> Deserialize<'de> for VersionsHistogram { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let values = Vec::::deserialize(deserializer)?; + if values.len() != VERSIONS_HISTOGRAM_LEN { + return Err(serde::de::Error::invalid_length( + values.len(), + &"exactly 7 object-version histogram buckets", + )); + } + Ok(Self(values)) } } @@ -535,13 +601,74 @@ impl DataUsageEntry { } } - for (i, v) in other.obj_sizes.0.iter().enumerate() { - self.obj_sizes.0[i] += v; - } + self.obj_sizes.merge_from(&other.obj_sizes); + self.obj_versions.merge_from(&other.obj_versions); + } - for (i, v) in other.obj_versions.0.iter().enumerate() { - self.obj_versions.0[i] += v; + pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool { + let scalar_counts_fit = self.objects.checked_add(other.objects).is_some() + && self.versions.checked_add(other.versions).is_some() + && self.delete_markers.checked_add(other.delete_markers).is_some() + && self.size.checked_add(other.size).is_some() + && self.failed_objects.checked_add(other.failed_objects).is_some(); + let histograms_fit = self.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN + && other.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN + && self.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN + && other.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN + && self + .obj_sizes + .0 + .iter() + .zip(other.obj_sizes.0.iter()) + .all(|(left, right)| left.checked_add(*right).is_some()) + && self + .obj_versions + .0 + .iter() + .zip(other.obj_versions.0.iter()) + .all(|(left, right)| left.checked_add(*right).is_some()); + let replication_fits = match (&self.replication_stats, &other.replication_stats) { + (_, None) | (None, Some(_)) => true, + (Some(left), Some(right)) => { + left.replica_size.checked_add(right.replica_size).is_some() + && left.replica_count.checked_add(right.replica_count).is_some() + && right.targets.iter().all(|(target, right_stats)| { + left.targets.get(target).is_none_or(|left_stats| { + left_stats.pending_size.checked_add(right_stats.pending_size).is_some() + && left_stats.replicated_size.checked_add(right_stats.replicated_size).is_some() + && left_stats.failed_size.checked_add(right_stats.failed_size).is_some() + && left_stats.failed_count.checked_add(right_stats.failed_count).is_some() + && left_stats.pending_count.checked_add(right_stats.pending_count).is_some() + && left_stats + .missed_threshold_size + .checked_add(right_stats.missed_threshold_size) + .is_some() + && left_stats + .after_threshold_size + .checked_add(right_stats.after_threshold_size) + .is_some() + && left_stats + .missed_threshold_count + .checked_add(right_stats.missed_threshold_count) + .is_some() + && left_stats + .after_threshold_count + .checked_add(right_stats.after_threshold_count) + .is_some() + && left_stats + .replicated_count + .checked_add(right_stats.replicated_count) + .is_some() + }) + }) + } + }; + + if !scalar_counts_fit || !histograms_fit || !replication_fits { + return false; } + self.merge(other); + true } } @@ -1395,6 +1522,17 @@ mod tests { assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1); } + #[test] + fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() { + let mut hist = SizeHistogram::default(); + hist.0[1] = u64::MAX; + hist.0[2] = 1; + + let map = hist.to_map(); + + assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX); + } + #[test] fn test_data_usage_cache_merge_adds_missing_child() { let mut base = DataUsageCache::default(); @@ -1708,4 +1846,86 @@ mod tests { assert!(cache.find("bucket/large/a").is_some()); assert!(cache.find("bucket/large/b").is_some()); } + + #[test] + fn checked_merge_rejects_scalar_and_replication_overflow_without_mutation() { + let mut entry = DataUsageEntry { + objects: usize::MAX, + replication_stats: Some(ReplicationAllStats { + replica_size: 7, + ..Default::default() + }), + ..Default::default() + }; + let other = DataUsageEntry { + objects: 1, + replication_stats: Some(ReplicationAllStats { + replica_size: u64::MAX, + ..Default::default() + }), + ..Default::default() + }; + + assert!(!entry.checked_merge(&other)); + assert_eq!(entry.objects, usize::MAX); + assert_eq!(entry.replication_stats.as_ref().map(|stats| stats.replica_size), Some(7)); + } + + #[test] + fn checked_merge_accepts_valid_usage() { + let mut entry = DataUsageEntry { + objects: 2, + size: 20, + ..Default::default() + }; + let other = DataUsageEntry { + objects: 3, + size: 30, + ..Default::default() + }; + + assert!(entry.checked_merge(&other)); + assert_eq!(entry.objects, 5); + assert_eq!(entry.size, 50); + } + + #[test] + fn histogram_deserialization_rejects_noncanonical_lengths() { + let invalid_sizes = + rmp_serde::to_vec(&vec![0_u64; SIZE_HISTOGRAM_LEN + 1]).expect("encode invalid object-size histogram fixture"); + let invalid_versions = + rmp_serde::to_vec(&vec![0_u64; VERSIONS_HISTOGRAM_LEN - 1]).expect("encode invalid object-version histogram fixture"); + + assert!(rmp_serde::from_slice::(&invalid_sizes).is_err()); + assert!(rmp_serde::from_slice::(&invalid_versions).is_err()); + } + + #[test] + fn replication_target_deserialization_preserves_large_historical_maps() { + let mut stats = ReplicationAllStats::default(); + for index in 0..=1024 { + stats.targets.insert(format!("target-{index}"), ReplicationStats::default()); + } + let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode"); + let decoded = rmp_serde::from_slice::(&encoded) + .expect("historical replication target maps must remain readable"); + + assert_eq!(decoded.targets.len(), stats.targets.len()); + } + + #[test] + fn checked_merge_rejects_noncanonical_histograms_without_mutation() { + let mut entry = DataUsageEntry { + objects: 2, + ..Default::default() + }; + let other = DataUsageEntry { + objects: 3, + obj_sizes: SizeHistogram(vec![0; SIZE_HISTOGRAM_LEN + 1]), + ..Default::default() + }; + + assert!(!entry.checked_merge(&other)); + assert_eq!(entry.objects, 2); + } } diff --git a/crates/e2e_test/src/content_encoding_test.rs b/crates/e2e_test/src/content_encoding_test.rs index c0a741d95..f99117424 100644 --- a/crates/e2e_test/src/content_encoding_test.rs +++ b/crates/e2e_test/src/content_encoding_test.rs @@ -80,6 +80,25 @@ mod tests { assert_eq!(head_resp.content_encoding(), Some("zstd"), "HEAD should return Content-Encoding: zstd"); assert_eq!(head_resp.content_type(), Some("text/plain"), "HEAD should return correct Content-Type"); + client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("DELETE object failed"); + client + .delete_bucket() + .bucket(bucket) + .send() + .await + .expect("DELETE bucket failed"); + client + .list_buckets() + .send() + .await + .expect("RustFS must remain available after deleting a bucket"); + env.stop_server(); } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index db4a082c7..41395f53e 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -244,11 +244,12 @@ pub mod config { pub mod com { pub use crate::config::com::{ COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS, - ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config, - read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock, - read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config, - save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, - with_server_config_read_lock, with_server_config_write_lock, + ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs, + read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate, + read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config, + save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock, + save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock, + with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock, }; } @@ -287,9 +288,9 @@ pub mod disk { pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, - FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - new_disk, validate_batch_read_version_item_count, + FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET, + ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, + WalkDirOptions, new_disk, validate_batch_read_version_item_count, }; pub use bytes::Bytes; pub use endpoint::Endpoint; @@ -390,12 +391,12 @@ pub mod rio { pub mod rpc { pub use crate::cluster::rpc::{ - LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, - gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, - node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, - sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, - verify_tonic_rpc_signature, + LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity, + TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, + node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, + set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, + verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, }; } diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index 1c387160e..3921c3322 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -27,6 +27,7 @@ //! Advisory: use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers}; +use crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION; use base64::Engine as _; use base64::engine::general_purpose; use hmac::{Hmac, KeyInit, Mac}; @@ -61,6 +62,7 @@ const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned"; const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601); const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536; +const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3"; pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService"; static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock = LazyLock::new(|| { get_env_bool( @@ -209,6 +211,42 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si mac.verify_slice(&signature).is_ok() } +fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) { + mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN); + mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes()); + mac.update(challenge.as_bytes()); + mac.update(server_epoch.as_bytes()); +} + +fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result> { + if challenge.is_nil() || server_epoch.is_nil() { + return Err(std::io::Error::other("Invalid namespace scanner capability scope")); + } + let mut mac = + ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; + update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch); + Ok(mac.finalize().into_bytes().to_vec()) +} + +fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> { + if challenge.is_nil() || server_epoch.is_nil() { + return Err(std::io::Error::other("Invalid namespace scanner capability scope")); + } + let mut mac = + ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; + update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch); + mac.verify_slice(proof) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof")) +} + +pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result> { + generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch) +} + +pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> { + verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof) +} + #[derive(Clone, Copy)] struct SignatureV2Scope<'a> { audience: &'a str, @@ -629,6 +667,20 @@ mod tests { runtime_sources::ensure_test_rpc_secret(); } + #[test] + fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() { + let secret = "test-scanner-capability-secret"; + let challenge = Uuid::new_v4(); + let server_epoch = Uuid::new_v4(); + let proof = + generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated"); + + assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok()); + assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err()); + assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err()); + assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err()); + } + /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed, /// fixed in rustfs/rustfs#4402): secret resolution must never silently fall /// back to a default/empty shared secret. Missing and default secrets both diff --git a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs index b7138b111..2d228f43e 100644 --- a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::cluster::rpc::build_auth_headers; +use crate::cluster::rpc::{build_auth_headers, verify_ns_scanner_capability}; use crate::disk::error::{Error, Result}; use crate::disk::{FileReader, FileWriter}; use crate::storage_api_contracts::internode::{ - WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, + NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, + NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, + NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, + WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, }; use async_trait::async_trait; use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE}; @@ -28,13 +31,18 @@ use rustfs_rio::{HttpReader, HttpWriter}; use sha2::{Digest, Sha256}; use std::sync::{Arc, OnceLock}; use std::time::Duration; +use tokio::io::AsyncReadExt; +use uuid::Uuid; static INTERNODE_DATA_TRANSPORT: OnceLock, String>> = OnceLock::new(); const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream"; const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir"; +const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner"; +const NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE: usize = 1024; const CONTENT_TYPE_JSON: &str = "application/json"; +const CONTENT_TYPE_MSGPACK: &str = "application/msgpack"; fn unsupported_transport_message(transport: &str) -> String { format!( @@ -101,6 +109,25 @@ pub struct WalkDirStreamRequest { pub stall_timeout: Option, } +#[derive(Debug, Clone)] +pub struct NsScannerStreamRequest { + pub endpoint: String, + pub disk: String, + pub request_id: Uuid, + pub server_epoch: Uuid, + pub session_id: Uuid, + pub session_sequence: u64, + pub next_cycle: u64, + pub leader_epoch: u64, + pub body: Vec, + pub stall_timeout: Option, +} + +#[derive(Debug, Clone)] +pub struct NsScannerCapabilityRequest { + pub endpoint: String, +} + /// Data-plane stream opener used by `RemoteDisk`. /// /// This boundary is limited to remote disk streams that can move large payloads. @@ -114,6 +141,12 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { async fn open_read(&self, request: ReadStreamRequest) -> Result; async fn open_write(&self, request: WriteStreamRequest) -> Result; async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result; + async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result { + Err(Error::MethodNotAllowed) + } + async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result { + Err(Error::MethodNotAllowed) + } fn name(&self) -> &'static str; fn capabilities(&self) -> InternodeDataTransportCapabilities; } @@ -148,6 +181,39 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport { )) } + async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result { + let url = build_ns_scanner_url(&request); + let mut headers = msgpack_headers(); + build_auth_headers(&url, &Method::POST, &mut headers)?; + Ok(Box::new( + HttpReader::new_with_stall_timeout(url, Method::POST, headers, Some(request.body), request.stall_timeout).await?, + )) + } + + async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result { + let challenge = Uuid::new_v4(); + let url = build_ns_scanner_capability_url(&request, challenge); + let mut headers = msgpack_headers(); + build_auth_headers(&url, &Method::GET, &mut headers)?; + let reader = HttpReader::new(url, Method::GET, headers, None).await?; + let mut body = Vec::new(); + reader + .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX)) + .read_to_end(&mut body) + .await?; + if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE { + return Err(Error::other("invalid remote namespace scanner capability response size")); + } + let response: NsScannerCapabilityResponse = + rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?; + if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() { + return Err(Error::other("incompatible remote namespace scanner capability response")); + } + verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof) + .map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?; + Ok(response.server_epoch) + } + fn name(&self) -> &'static str { DEFAULT_INTERNODE_DATA_TRANSPORT } @@ -197,12 +263,54 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String { ) } +fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String { + let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower); + format!( + "{}{}?disk={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}", + request.endpoint, + NS_SCANNER_PATH, + urlencoding::encode(&request.disk), + NS_SCANNER_REQUEST_ID_QUERY, + request.request_id, + NS_SCANNER_SERVER_EPOCH_QUERY, + request.server_epoch, + NS_SCANNER_SESSION_ID_QUERY, + request.session_id, + NS_SCANNER_SESSION_SEQUENCE_QUERY, + request.session_sequence, + NS_SCANNER_CYCLE_QUERY, + request.next_cycle, + NS_SCANNER_LEADER_EPOCH_QUERY, + request.leader_epoch, + NS_SCANNER_BODY_SHA256_QUERY, + body_sha256 + ) +} + +fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String { + format!( + "{}{}?{}={}&{}={}", + request.endpoint, + NS_SCANNER_PATH, + NS_SCANNER_PROTOCOL_VERSION_QUERY, + NS_SCANNER_PROTOCOL_VERSION, + NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, + challenge + ) +} + fn json_headers() -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_JSON)); headers } +fn msgpack_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_MSGPACK)); + headers +} + fn build_internode_data_transport_result( configured_transport: Option<&str>, ) -> std::result::Result, String> { @@ -241,6 +349,65 @@ pub fn build_internode_data_transport_from_env() -> Result Result { + Ok(Box::new(tokio::io::empty())) + } + + async fn open_write(&self, _request: WriteStreamRequest) -> Result { + Ok(Box::new(tokio::io::sink())) + } + + async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result { + Ok(Box::new(tokio::io::empty())) + } + + fn name(&self) -> &'static str { + "legacy-test" + } + + fn capabilities(&self) -> InternodeDataTransportCapabilities { + InternodeDataTransportCapabilities::tcp_http() + } + } + + #[tokio::test] + async fn legacy_transport_defaults_namespace_scanner_to_unsupported() { + let transport = LegacyTestTransport; + + let probe_err = transport + .probe_ns_scanner(NsScannerCapabilityRequest { + endpoint: "http://node1:9000".to_string(), + }) + .await + .expect_err("legacy transport should report namespace scanner as unsupported"); + assert!(matches!(probe_err, Error::MethodNotAllowed)); + + let open_result = transport + .open_ns_scanner(NsScannerStreamRequest { + endpoint: "http://node1:9000".to_string(), + disk: "http://node1:9000/data/rustfs0".to_string(), + request_id: Uuid::new_v4(), + server_epoch: Uuid::new_v4(), + session_id: Uuid::new_v4(), + session_sequence: 0, + next_cycle: 7, + leader_epoch: 9, + body: Vec::new(), + stall_timeout: None, + }) + .await; + let open_err = match open_result { + Ok(_) => panic!("legacy transport should not open namespace scanner streams"), + Err(err) => err, + }; + assert!(matches!(open_err, Error::MethodNotAllowed)); + } + #[test] fn tcp_http_capabilities_are_behavior_preserving() { let transport = TcpHttpInternodeDataTransport; @@ -322,6 +489,57 @@ mod tests { ); } + #[test] + fn ns_scanner_url_binds_body_and_encodes_disk_ref() { + let request_id = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("request ID"); + let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch"); + let session_id = Uuid::parse_str("99999999-8888-4777-8666-555555555555").expect("session ID"); + let url = build_ns_scanner_url(&NsScannerStreamRequest { + endpoint: "http://node1:9000".to_string(), + disk: "http://node1:9000/data/rustfs0".to_string(), + request_id, + server_epoch, + session_id, + session_sequence: 3, + next_cycle: 7, + leader_epoch: 9, + body: b"scanner-request".to_vec(), + stall_timeout: None, + }); + + assert_eq!( + url, + concat!( + "http://node1:9000/rustfs/rpc/ns_scanner?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0", + "&ns_scanner_request_id=11111111-2222-4333-8444-555555555555", + "&ns_scanner_server_epoch=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + "&ns_scanner_session_id=99999999-8888-4777-8666-555555555555", + "&ns_scanner_session_sequence=3", + "&ns_scanner_cycle=7", + "&ns_scanner_leader_epoch=9", + "&ns_scanner_body_sha256=c958f15ca28422275c1245399f4c44eaba628ca453fcd77d6b3d4484573e4387" + ) + ); + } + + #[test] + fn ns_scanner_capability_url_binds_version_and_challenge() { + let challenge = Uuid::parse_str("12345678-1234-4234-8234-123456789abc").expect("challenge"); + let url = build_ns_scanner_capability_url( + &NsScannerCapabilityRequest { + endpoint: "http://node1:9000".to_string(), + }, + challenge, + ); + + assert_eq!( + url, + format!( + "http://node1:9000/rustfs/rpc/ns_scanner?ns_scanner_protocol={NS_SCANNER_PROTOCOL_VERSION}&ns_scanner_challenge={challenge}" + ) + ); + } + #[test] fn transport_config_defaults_to_tcp_http() { let transport = build_internode_data_transport(None).unwrap(); diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index cfa97cd60..51025e927 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -30,8 +30,8 @@ pub use client::{ }; pub use http_auth::{ TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, - set_tonic_canonical_body_digest, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, - verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, + set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability, + verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, }; #[cfg(test)] pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport; @@ -41,6 +41,6 @@ pub use peer_rest_client::{ SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, }; pub(crate) use peer_s3_client::heal_bucket_local_on_disks; -pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys}; +pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys, ScannerBucketListing, ScannerSetBucketListing}; pub use remote_disk::RemoteDisk; pub use remote_locker::RemoteClient; diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 7bdda0d6d..8617ac416 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -18,6 +18,9 @@ use crate::cluster::rpc::client::{ }; use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof}; use crate::error::{Error, Result}; +use crate::storage_api_contracts::internode::{ + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, +}; use crate::{ bucket::replication::BucketStats, disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout}, @@ -27,6 +30,7 @@ use crate::{ }; use bytes::Bytes; use rmp_serde::{Deserializer, Serializer}; +use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_madmin::{ ServerProperties, health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices}, @@ -97,15 +101,34 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result< Ok(stats) } +fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> { + if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC + && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS) + && protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION + { + return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence"))); + } + Ok(()) +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ScannerPeerActivity { pub instance_id: String, pub namespace_generation: u64, pub maintenance_generation: u64, + pub protocol_version: u32, + pub topology_digest: Option<[u8; 32]>, + pub data_movement_active: Option, + pub dirty_usage_generation: Option, + pub dirty_usage_pending: Option, } -fn decode_scanner_activity(response: ScannerActivityResponse) -> Result { - let instance_id = response.instance_id; +fn decode_scanner_activity_with_verifier( + response: ScannerActivityResponse, + challenge: &[u8; 16], + verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>, +) -> Result { + let instance_id = &response.instance_id; if instance_id.len() != 32 || !instance_id .as_bytes() @@ -114,10 +137,80 @@ fn decode_scanner_activity(response: ScannerActivityResponse) -> Result + { + (None, None, None, None) + } + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => { + return Err(Error::other("legacy scanner activity peer returned unexpected extended fields")); + } + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + if response.dirty_usage_generation != 0 || response.dirty_usage_pending { + return Err(Error::other("scanner activity protocol v4 peer returned unauthenticated v5 fields")); + } + let canonical = rustfs_protos::canonical_scanner_activity_v4_response_body(challenge, &response) + .map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?; + verify_proof(&canonical, &response.response_proof)?; + ( + Some( + response + .topology_digest + .as_ref() + .try_into() + .map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?, + ), + Some(response.data_movement_active), + None, + None, + ) + } + SCANNER_ACTIVITY_PROTOCOL_VERSION => { + if response.dirty_usage_pending && response.dirty_usage_generation == 0 { + return Err(Error::other("scanner activity peer returned pending dirty usage without a generation")); + } + let canonical = rustfs_protos::canonical_scanner_activity_response_body(challenge, &response) + .map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?; + verify_proof(&canonical, &response.response_proof)?; + ( + Some( + response + .topology_digest + .as_ref() + .try_into() + .map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?, + ), + Some(response.data_movement_active), + Some(response.dirty_usage_generation), + Some(response.dirty_usage_pending), + ) + } + version => { + return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))); + } + }; Ok(ScannerPeerActivity { - instance_id, + instance_id: response.instance_id, namespace_generation: response.namespace_generation, maintenance_generation: response.maintenance_generation, + protocol_version: response.protocol_version, + topology_digest, + data_movement_active, + dirty_usage_generation, + dirty_usage_pending, + }) +} + +fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 16]) -> Result { + decode_scanner_activity_with_verifier(response, challenge, |canonical, proof| { + verify_tonic_rpc_response_proof(canonical, proof) + .map_err(|_| Error::other("peer returned an invalid scanner activity response proof")) }) } @@ -1331,6 +1424,7 @@ impl PeerRestClient { } return Err(Error::other("")); } + validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?; Ok(()) } .await, @@ -1338,25 +1432,44 @@ impl PeerRestClient { .await } - pub async fn scanner_activity(&self) -> Result { + async fn scanner_activity_request( + &self, + acknowledge_instance_id: String, + acknowledge_dirty_usage_generation: u64, + ) -> Result { self.finalize_result( async { + let challenge = Uuid::new_v4(); let mut client = self .get_client() .await? .max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE) .max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE); - let response = client - .scanner_activity(Request::new(ScannerActivityRequest {})) - .await? - .into_inner(); - decode_scanner_activity(response) + let mut request = Request::new(ScannerActivityRequest { + challenge: challenge.as_bytes().to_vec().into(), + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id, + acknowledge_dirty_usage_generation, + }); + let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref()) + .map_err(|_| Error::other("scanner activity request is too large to authenticate"))?; + set_tonic_canonical_body_digest(&mut request, &canonical)?; + let response = client.scanner_activity(request).await?.into_inner(); + decode_scanner_activity(response, challenge.as_bytes()) } .await, ) .await } + pub async fn scanner_activity(&self) -> Result { + self.scanner_activity_request(String::new(), 0).await + } + + pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result { + self.scanner_activity_request(instance_id, generation).await + } + pub async fn get_metacache_listing(&self) -> Result<()> { warn!("get_metacache_listing is not implemented in PeerRestClient"); Err(Error::NotImplemented) @@ -1542,6 +1655,7 @@ impl PeerRestClient { #[cfg(test)] mod tests { use super::*; + use crate::config::com::STORAGE_CLASS_SUB_SYS; use serde_json::Value; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; @@ -1650,6 +1764,14 @@ mod tests { ) } + fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result { + decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| { + (proof == b"proof") + .then_some(()) + .ok_or_else(|| Error::other("peer returned an invalid scanner activity response proof")) + }) + } + #[test] fn build_clients_from_slots_preserves_missing_remote_topology_slots() { let slots = vec![ @@ -1682,13 +1804,89 @@ mod tests { #[test] fn scanner_activity_requires_restart_safe_peer_identity() { + let legacy = decode_test_scanner_activity(ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + topology_digest: Vec::new().into(), + data_movement_active: false, + response_proof: Vec::new().into(), + dirty_usage_generation: 0, + dirty_usage_pending: false, + }) + .expect("legacy peers should retain their activity generations during a rolling upgrade"); + assert_eq!( + legacy, + ScannerPeerActivity { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + topology_digest: None, + data_movement_active: None, + dirty_usage_generation: None, + dirty_usage_pending: None, + } + ); + + let previous = decode_test_scanner_activity(ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: true, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 0, + dirty_usage_pending: false, + }) + .expect("protocol v4 peers should remain observable during a rolling upgrade"); + assert_eq!( + previous, + ScannerPeerActivity { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + topology_digest: Some([7; 32]), + data_movement_active: Some(true), + dirty_usage_generation: None, + dirty_usage_pending: None, + } + ); + + let malformed_topology = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 31].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, + }; + assert!( + decode_test_scanner_activity(malformed_topology) + .expect_err("activity topology digests must have the protocol-defined length") + .to_string() + .contains("topology digest") + ); + let missing_instance = ScannerActivityResponse { instance_id: String::new(), namespace_generation: 7, maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, }; assert!( - decode_scanner_activity(missing_instance) + decode_test_scanner_activity(missing_instance) .expect_err("an empty instance ID is not restart safe") .to_string() .contains("instance ID") @@ -1698,18 +1896,30 @@ mod tests { instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(), namespace_generation: 7, maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, }; assert!( - decode_scanner_activity(malformed_instance) + decode_test_scanner_activity(malformed_instance) .expect_err("activity instance IDs must use the canonical lowercase hex form") .to_string() .contains("instance ID") ); - let activity = decode_scanner_activity(ScannerActivityResponse { + let activity = decode_test_scanner_activity(ScannerActivityResponse { instance_id: "0123456789abcdef0123456789abcdef".to_string(), namespace_generation: 7, maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: true, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, }) .expect("complete activity responses should be accepted"); assert_eq!( @@ -1718,8 +1928,123 @@ mod tests { instance_id: "0123456789abcdef0123456789abcdef".to_string(), namespace_generation: 7, maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: Some([7; 32]), + data_movement_active: Some(true), + dirty_usage_generation: Some(11), + dirty_usage_pending: Some(true), } ); + + let pending_without_generation = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 0, + dirty_usage_pending: true, + }; + assert!( + decode_test_scanner_activity(pending_without_generation) + .expect_err("pending dirty usage must carry a nonzero generation") + .to_string() + .contains("without a generation") + ); + + let previous_with_dirty_usage = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, + }; + assert!( + decode_test_scanner_activity(previous_with_dirty_usage) + .expect_err("protocol v4 responses must not claim unauthenticated dirty usage fields") + .to_string() + .contains("unauthenticated v5 fields") + ); + + let legacy_with_topology = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 0, + dirty_usage_pending: false, + }; + assert!( + decode_test_scanner_activity(legacy_with_topology) + .expect_err("legacy protocol responses must not claim extended fields") + .to_string() + .contains("unexpected extended fields") + ); + + let unsupported_protocol = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION + 1, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: b"proof".to_vec().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, + }; + assert!( + decode_test_scanner_activity(unsupported_protocol) + .expect_err("unknown activity protocols must fail closed") + .to_string() + .contains("unsupported scanner activity protocol") + ); + + let missing_proof = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: vec![7; 32].into(), + data_movement_active: false, + response_proof: Vec::new().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, + }; + assert!( + decode_test_scanner_activity(missing_proof) + .expect_err("unsigned scanner activity responses must fail closed") + .to_string() + .contains("response proof") + ); + } + + #[test] + fn dynamic_scanner_config_requires_versioned_peer_acknowledgement() { + for sub_system in [SCANNER_SUB_SYS, HEAL_SUB_SYS] { + let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, sub_system, 0) + .expect_err("an unversioned peer must not claim scanner config convergence"); + assert!(err.to_string().contains("does not support dynamic")); + validate_signal_service_protocol( + SERVICE_SIGNAL_RELOAD_DYNAMIC, + sub_system, + rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION, + ) + .expect("a current peer should support dynamic scanner config"); + } + + validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, 0) + .expect("unrelated dynamic config keeps its existing compatibility contract"); + validate_signal_service_protocol(SERVICE_SIGNAL_REFRESH_CONFIG, SCANNER_SUB_SYS, 0) + .expect("full refresh compatibility is guarded by its scanner preflight"); } #[test] diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index 3f4b5602b..17d42e84d 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -49,6 +49,20 @@ use tracing::{debug, info, warn}; type Client = Arc>; +#[derive(Clone, Debug)] +pub struct ScannerBucketListing { + pub buckets: Vec, + pub set_buckets: Vec, + pub topology_complete: bool, +} + +#[derive(Clone, Debug)] +pub struct ScannerSetBucketListing { + pub pool_index: usize, + pub set_index: usize, + pub buckets: Vec, +} + fn pool_participant_errors(clients: &[Client], errors: &[Option], pool_idx: usize) -> Vec> { clients .iter() @@ -216,6 +230,10 @@ impl S3PeerSys { Ok(()) } pub async fn list_bucket(&self, opts: &BucketOptions) -> Result> { + Ok(self.list_bucket_for_scanner(opts).await?.buckets) + } + + pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.list_bucket(opts)); @@ -239,9 +257,12 @@ impl S3PeerSys { } let mut result_map: HashMap<&String, BucketInfo> = HashMap::new(); + let mut topology_complete = true; for i in 0..self.pools_count { let per_pool_errs = pool_participant_errors(&self.clients, &errors, i); let quorum = pool_write_quorum(per_pool_errs.len()); + topology_complete &= + !per_pool_errs.is_empty() && per_pool_errs.iter().all(|participant_error| participant_error.is_none()); if let Some(pool_err) = reduce_pool_write_quorum_errs(&per_pool_errs) { tracing::error!("list_bucket per_pool_errs: {per_pool_errs:?}"); @@ -261,20 +282,17 @@ impl S3PeerSys { } for bucket in buckets.iter() { - if result_map.contains_key(&bucket.name) { - continue; - } - // incr bucket_map count create if not exists let count = bucket_map.entry(&bucket.name).or_insert(0usize); *count += 1; if *count >= quorum { - result_map.insert(&bucket.name, bucket.clone()); + result_map.entry(&bucket.name).or_insert_with(|| bucket.clone()); } } } } + topology_complete &= bucket_map.values().all(|count| *count >= quorum); // TODO: MRF } @@ -282,7 +300,11 @@ impl S3PeerSys { buckets.sort_by_key(|b| b.name.clone()); - Ok(buckets) + Ok(ScannerBucketListing { + buckets, + set_buckets: Vec::new(), + topology_complete, + }) } pub async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); @@ -1645,6 +1667,86 @@ mod tests { assert_eq!(buckets[0].name, bucket.name); } + #[tokio::test] + async fn scanner_bucket_listing_marks_quorum_result_incomplete_when_a_peer_is_missing() { + let bucket = BucketInfo { + name: "bucket-hidden-by-quorum".to_string(), + ..Default::default() + }; + let peer_sys = S3PeerSys { + clients: vec![ + test_peer_with_list_bucket(&[0], Ok(vec![bucket])), + test_peer_with_list_bucket(&[0], Ok(Vec::new())), + test_peer_with_list_bucket(&[0], Ok(Vec::new())), + test_peer_with_list_bucket(&[0], Err(Error::DiskAccessDenied)), + ], + pools_count: 1, + }; + + let listing = peer_sys + .list_bucket_for_scanner(&BucketOptions::default()) + .await + .expect("peer quorum should still produce a scanner candidate listing"); + + assert!(listing.buckets.is_empty()); + assert!(!listing.topology_complete); + } + + #[tokio::test] + async fn scanner_bucket_listing_marks_divergent_successful_peers_incomplete() { + let bucket = BucketInfo { + name: "bucket-below-quorum".to_string(), + ..Default::default() + }; + let peer_sys = S3PeerSys { + clients: vec![ + test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[0], Ok(vec![bucket])), + test_peer_with_list_bucket(&[0], Ok(Vec::new())), + test_peer_with_list_bucket(&[0], Ok(Vec::new())), + ], + pools_count: 1, + }; + + let listing = peer_sys + .list_bucket_for_scanner(&BucketOptions::default()) + .await + .expect("successful peer responses should still produce a scanner candidate listing"); + + assert!(listing.buckets.is_empty()); + assert!(!listing.topology_complete); + } + + #[tokio::test] + async fn scanner_bucket_listing_checks_same_bucket_in_every_pool() { + let bucket = BucketInfo { + name: "shared-bucket".to_string(), + ..Default::default() + }; + let peer_sys = S3PeerSys { + clients: vec![ + test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])), + test_peer_with_list_bucket(&[1], Ok(Vec::new())), + test_peer_with_list_bucket(&[1], Ok(Vec::new())), + ], + pools_count: 2, + }; + + let listing = peer_sys + .list_bucket_for_scanner(&BucketOptions::default()) + .await + .expect("a bucket visible in one pool should remain a scan candidate"); + + assert_eq!(listing.buckets.len(), 1); + assert_eq!(listing.buckets[0].name, bucket.name); + assert!(!listing.topology_complete); + } + #[tokio::test] async fn test_delete_bucket_fails_when_any_pool_misses_write_quorum() { let peer_sys = S3PeerSys { diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 642aca597..1a924016b 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -17,7 +17,8 @@ use crate::cluster::rpc::client::{ node_service_time_out_client_for_class, node_service_time_out_client_no_auth, }; use crate::cluster::rpc::internode_data_transport::{ - InternodeDataTransport, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest, + InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest, + WriteStreamRequest, }; use crate::disk::error::{Error, Result}; use crate::disk::{ @@ -81,6 +82,7 @@ const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20); const REMOTE_DISK_OPEN_READ_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_READ_RETRY_BACKOFF: Duration = Duration::from_millis(20); +const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); /// Base backoff for idempotent read-only RPC retries (grpc-optimization P3-3); doubles per attempt. const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50); const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ"; @@ -97,6 +99,17 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk"; const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health"; const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc"; +fn decode_volume_infos(volume_infos: Vec) -> Result> { + volume_infos + .into_iter() + .enumerate() + .map(|(index, json)| { + serde_json::from_str::(&json) + .map_err(|err| Error::other(format!("decode list volumes entry {index} failed: {err}"))) + }) + .collect() +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum BatchMetadataRpcMode { Off, @@ -264,6 +277,71 @@ fn spawn_control_channel_prewarm(addr: String) { } impl RemoteDisk { + pub(crate) async fn ns_scanner_server_epoch(&self) -> Result> { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest { + endpoint: self.endpoint.grid_host(), + }); + let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe) + .await + .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?; + match result { + Ok(server_epoch) => Ok(Some(server_epoch)), + // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3. + Err(DiskError::MethodNotAllowed) => Ok(None), + Err(err) + if matches!( + err.internode_http_error_kind(), + Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) + if matches!(status.as_u16(), 404 | 405) + ) => + { + Ok(None) + } + Err(err) + if matches!( + err.internode_http_error_kind(), + Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 426 + ) => + { + Ok(None) + } + Err(err) => Err(err), + } + } + + pub(crate) async fn open_ns_scanner_stream(&self, request: crate::disk::NsScannerOpenRequest) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + let crate::disk::NsScannerOpenRequest { + request_id, + server_epoch, + session_id, + session_sequence, + next_cycle, + leader_epoch, + body, + stall_timeout, + } = request; + self.data_transport + .open_ns_scanner(NsScannerStreamRequest { + endpoint: self.endpoint.grid_host(), + disk: self.disk_ref().await, + request_id, + server_epoch, + session_id, + session_sequence, + next_cycle, + leader_epoch, + body, + stall_timeout, + }) + .await + } + fn recovery_monitor_span(addr: &str, endpoint: &Endpoint) -> tracing::Span { tracing::info_span!( "recovery-monitor", @@ -1335,11 +1413,7 @@ impl DiskAPI for RemoteDisk { return Err(response.error.unwrap_or_default().into()); } - let infos = response - .volume_infos - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect(); + let infos = decode_volume_infos(response.volume_infos)?; Ok(infos) }, @@ -2722,6 +2796,19 @@ mod tests { static INIT: Once = Once::new(); + #[test] + fn list_volumes_decode_rejects_a_malformed_entry() { + let valid = serde_json::to_string(&VolumeInfo { + name: "bucket".to_string(), + created: None, + }) + .expect("volume info should serialize"); + let err = decode_volume_infos(vec![valid, "{".to_string()]) + .expect_err("a malformed volume entry must fail the complete response"); + + assert!(err.to_string().contains("entry 1")); + } + #[test] fn decoded_remote_metadata_rejects_default_like_delete_marker() { let forged = FileInfo { @@ -2794,14 +2881,31 @@ mod tests { Read(ReadStreamRequest), Write(WriteStreamRequest), WalkDir(WalkDirStreamRequest), + NsScanner(NsScannerStreamRequest), + NsScannerProbe(NsScannerCapabilityRequest), } #[derive(Debug, Clone, Default)] struct RecordingInternodeDataTransport { calls: Arc>>, + ns_scanner_probe_status: Arc>>, } impl RecordingInternodeDataTransport { + fn with_ns_scanner_probe_status(status: u16) -> Self { + Self { + calls: Arc::default(), + ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))), + } + } + + fn set_ns_scanner_probe_status(&self, status: Option) { + *self + .ns_scanner_probe_status + .lock() + .expect("namespace scanner probe status lock poisoned") = status; + } + fn calls(&self) -> Vec { self.calls.lock().expect("recorded transport calls lock poisoned").clone() } @@ -3369,6 +3473,26 @@ mod tests { Ok(Box::new(EmptyTestReader)) } + async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result { + self.record(RecordedTransportCall::NsScanner(request)); + Ok(Box::new(EmptyTestReader)) + } + + async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result { + self.record(RecordedTransportCall::NsScannerProbe(request)); + if let Some(status) = *self + .ns_scanner_probe_status + .lock() + .expect("namespace scanner probe status lock poisoned") + { + let status = reqwest::StatusCode::from_u16(status).expect("test status code should be valid"); + return Err( + rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)).into(), + ); + } + Ok(Uuid::from_u128(1)) + } + fn name(&self) -> &'static str { "recording" } @@ -3401,6 +3525,14 @@ mod tests { } } + async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result { + panic!("open_ns_scanner should not be used in walk_dir retry test"); + } + + async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result { + Ok(Uuid::from_u128(1)) + } + fn name(&self) -> &'static str { "retrying-walk-dir" } @@ -3429,6 +3561,14 @@ mod tests { panic!("open_walk_dir should not be used in open_write retry test"); } + async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result { + panic!("open_ns_scanner should not be used in open_write retry test"); + } + + async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result { + Ok(Uuid::from_u128(1)) + } + fn name(&self) -> &'static str { "retrying-open-write" } @@ -4092,6 +4232,139 @@ mod tests { } } + #[tokio::test] + async fn test_remote_disk_namespace_scanner_uses_configured_data_transport() { + let transport = RecordingInternodeDataTransport::default(); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await; + let expected_disk = remote_disk.disk_ref().await; + let expected_body = b"namespace-scanner-request".to_vec(); + let expected_request_id = Uuid::new_v4(); + let expected_server_epoch = Uuid::new_v4(); + let expected_session_id = Uuid::new_v4(); + + let _reader = remote_disk + .open_ns_scanner_stream(crate::disk::NsScannerOpenRequest { + request_id: expected_request_id, + server_epoch: expected_server_epoch, + session_id: expected_session_id, + session_sequence: 3, + next_cycle: 7, + leader_epoch: 9, + body: expected_body.clone(), + stall_timeout: Some(Duration::from_secs(15)), + }) + .await + .expect("namespace scanner stream should open"); + + let calls = transport.calls(); + assert_eq!(calls.len(), 1); + match &calls[0] { + RecordedTransportCall::NsScanner(request) => { + assert_eq!(request.endpoint, "http://remote-node:9000"); + assert_eq!(request.disk, expected_disk); + assert_eq!(request.request_id, expected_request_id); + assert_eq!(request.server_epoch, expected_server_epoch); + assert_eq!(request.session_id, expected_session_id); + assert_eq!(request.session_sequence, 3); + assert_eq!(request.next_cycle, 7); + assert_eq!(request.leader_epoch, 9); + assert_eq!(request.body, expected_body); + assert_eq!(request.stall_timeout, Some(Duration::from_secs(15))); + } + other => panic!("expected namespace scanner transport call, got {other:?}"), + } + } + + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_uses_configured_data_transport() { + let transport = RecordingInternodeDataTransport::default(); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await; + + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("namespace scanner capability probe should succeed"), + Some(Uuid::from_u128(1)) + ); + + let calls = transport.calls(); + assert_eq!(calls.len(), 1); + match &calls[0] { + RecordedTransportCall::NsScannerProbe(request) => { + assert_eq!(request.endpoint, "http://remote-node:9000"); + } + other => panic!("expected namespace scanner capability probe, got {other:?}"), + } + } + + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_rejects_old_and_incompatible_peers() { + for status in [404, 405, 426] { + let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(status); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await; + + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("unsupported namespace scanner response should be classified"), + None + ); + } + } + + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() { + let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await; + + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("legacy transport should be classified as unsupported"), + None + ); + } + + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_reprobes_after_peer_upgrade() { + let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(404); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await; + + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("old peer should be classified as unsupported"), + None + ); + transport.set_ns_scanner_probe_status(None); + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("upgraded peer should be re-probed"), + Some(Uuid::from_u128(1)) + ); + assert_eq!(transport.calls().len(), 2); + } + + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_propagates_transient_failure() { + let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(503); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await; + + let err = remote_disk + .ns_scanner_server_epoch() + .await + .expect_err("transient capability failure must not be reported as unsupported"); + assert!(matches!( + err.internode_http_error_kind(), + Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 503 + )); + } + #[tokio::test] async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() { let transport = RecordingInternodeDataTransport::default(); diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index d29d56e8d..0f3eb9ab5 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::config::{audit, notify, oidc, storageclass}; +use crate::config::{audit, heal, notify, oidc, scanner, storageclass}; use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; @@ -22,7 +22,7 @@ use crate::storage_api_contracts::{ admin::StorageAdminApi, heal::HealOperations, namespace::NamespaceLocking, - object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete}, + object::{DeletedObject, EcstoreObjectIO, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete}, range::HTTPRangeSpec, }; use crate::store::ECStore; @@ -41,14 +41,17 @@ use rustfs_config::notify::{ }; use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC}; use rustfs_config::server_config::{Config, KVS}; -use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION}; +use rustfs_config::{ + COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, HEAL_KEYS, HEAL_SUB_SYS, RUSTFS_REGION, SCANNER_KEYS, + SCANNER_SUB_SYS, +}; use rustfs_filemeta::FileInfo; use rustfs_utils::path::SLASH_SEPARATOR; use serde_json::{Map, Value}; use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; use std::sync::{Arc, RwLock}; -use tokio::sync::RwLock as AsyncRwLock; +use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; use tracing::{debug, error, info, instrument, warn}; pub const CONFIG_PREFIX: &str = "config"; @@ -56,7 +59,7 @@ const SERVER_CONFIG_OBJECT: &str = "config/config.json"; // Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock // for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. -static SERVER_CONFIG_LOCK: LazyLock> = LazyLock::new(|| AsyncRwLock::new(())); +static SERVER_CONFIG_LOCK: LazyLock>> = LazyLock::new(|| Arc::new(AsyncRwLock::new(()))); fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error { let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" }; @@ -231,6 +234,29 @@ struct TargetConfigDescriptor { valid_keys: &'static [&'static str], } +#[derive(Clone, Copy)] +struct ScalarConfigDescriptor { + subsystem_key: &'static str, + default_kvs: &'static LazyLock, + valid_keys: &'static [&'static str], +} + +fn scanner_config_descriptor() -> ScalarConfigDescriptor { + ScalarConfigDescriptor { + subsystem_key: SCANNER_SUB_SYS, + default_kvs: &scanner::DEFAULT_KVS, + valid_keys: SCANNER_KEYS, + } +} + +fn heal_config_descriptor() -> ScalarConfigDescriptor { + ScalarConfigDescriptor { + subsystem_key: HEAL_SUB_SYS, + default_kvs: &heal::DEFAULT_KVS, + valid_keys: HEAL_KEYS, + } +} + fn notify_target_descriptors() -> [TargetConfigDescriptor; 9] { [ TargetConfigDescriptor { @@ -566,7 +592,7 @@ fn new_server_config() -> Config { async fn new_and_save_server_config(api: Arc) -> Result where - S: EcstoreObjectIO + StorageAdminApi, + S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { let cfg = new_server_config(); save_server_config(api, &cfg).await?; @@ -584,6 +610,10 @@ where } fn get_config_file() -> String { + server_config_path() +} + +pub fn server_config_path() -> String { SERVER_CONFIG_OBJECT.to_string() } @@ -711,6 +741,100 @@ fn parse_target_scalar_value(key: &str, value: &Value) -> Option { } } +fn decode_scalar_config_object(config_obj: &Map, descriptor: ScalarConfigDescriptor) -> Result { + let mut overrides = KVS::new(); + for (key, value) in config_obj { + if !descriptor.valid_keys.contains(&key.as_str()) || key == COMMENT_KEY { + if key != "default" && key != DEFAULT_DELIMITER && key != COMMENT_KEY { + warn!(config_key = %format!("{}.{key}", descriptor.subsystem_key), "Ignoring unknown persisted scalar config key"); + } + continue; + } + + let parsed = parse_target_scalar_value(key, value).ok_or_else(|| { + Error::other(format!( + "invalid external {} config value for {key}: expected a scalar", + descriptor.subsystem_key + )) + })?; + overrides.insert(key.clone(), parsed); + } + + Ok(overrides) +} + +fn decode_scalar_config_value(config_value: &Value, descriptor: ScalarConfigDescriptor) -> Result { + match config_value { + Value::Object(config_obj) => { + let mut overrides = match config_obj.get("default").or_else(|| config_obj.get(DEFAULT_DELIMITER)) { + Some(value) => decode_scalar_config_value(value, descriptor)?, + None => KVS::new(), + }; + let direct = decode_scalar_config_object(config_obj, descriptor)?; + if !direct.is_empty() { + overrides.extend(direct); + } + Ok(overrides) + } + Value::Array(entries) => { + let mut overrides = KVS::new(); + for entry in entries { + let entry = entry.as_object().ok_or_else(|| { + Error::other(format!("invalid external {} KVS entry: expected an object", descriptor.subsystem_key)) + })?; + let key = entry.get("key").and_then(Value::as_str).ok_or_else(|| { + Error::other(format!("invalid external {} KVS entry: missing string key", descriptor.subsystem_key)) + })?; + if !descriptor.valid_keys.contains(&key) || key == COMMENT_KEY { + if key != COMMENT_KEY { + warn!(config_key = %format!("{}.{key}", descriptor.subsystem_key), "Ignoring unknown persisted scalar config key"); + } + continue; + } + let value = entry + .get("value") + .and_then(|value| parse_target_scalar_value(key, value)) + .ok_or_else(|| { + Error::other(format!( + "invalid external {} config value for {key}: expected a scalar", + descriptor.subsystem_key + )) + })?; + overrides.insert(key.to_string(), value); + } + Ok(overrides) + } + _ => Err(Error::other(format!( + "invalid external {} config shape: expected an object or KVS array", + descriptor.subsystem_key + ))), + } +} + +fn apply_external_scalar_config_map( + cfg: &mut Config, + root: &Map, + descriptor: ScalarConfigDescriptor, +) -> Result { + let Some(config_value) = root.get(descriptor.subsystem_key) else { + return Ok(false); + }; + let overrides = decode_scalar_config_value(config_value, descriptor)?; + + if overrides.is_empty() { + return Ok(false); + } + + let kvs = cfg + .0 + .entry(descriptor.subsystem_key.to_string()) + .or_default() + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(|| (**descriptor.default_kvs).clone()); + kvs.extend(overrides); + Ok(true) +} + fn decode_target_instance_object(instance: &Map, valid_keys: &[&str]) -> KVS { let mut kvs = KVS::new(); @@ -877,11 +1001,13 @@ fn decode_server_config_blob(data: &[u8]) -> Result { let mut cfg = Config::new(); let has_storage = apply_external_storage_class_map(&mut cfg, &root); + let has_scanner = apply_external_scalar_config_map(&mut cfg, &root, scanner_config_descriptor())?; + let has_heal = apply_external_scalar_config_map(&mut cfg, &root, heal_config_descriptor())?; let has_oidc = apply_external_oidc_map(&mut cfg, &root); let has_notify = apply_external_notify_map(&mut cfg, &root); let has_audit = apply_external_audit_map(&mut cfg, &root); let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential"); - if !has_storage && !has_oidc && !has_notify && !has_audit && !has_header { + if !has_storage && !has_scanner && !has_heal && !has_oidc && !has_notify && !has_audit && !has_header { return Err(Error::other("unrecognized external server config shape")); } Ok(cfg) @@ -911,6 +1037,105 @@ fn build_storageclass_object(cfg: &Config) -> Map { sc_obj } +fn build_scalar_config_object(cfg: &Config, descriptor: ScalarConfigDescriptor) -> Map { + let kvs = cfg.get_value(descriptor.subsystem_key, DEFAULT_DELIMITER).unwrap_or_default(); + build_target_instance_diff_object(&kvs, descriptor.default_kvs, descriptor.valid_keys, descriptor.default_kvs) +} + +fn rendered_scalar_config_kvs_entries(rendered: &HashMap) -> Vec { + let mut entries = Vec::with_capacity(rendered.len()); + for (key, value) in rendered { + let mut entry = Map::new(); + entry.insert("key".to_string(), Value::String(key.clone())); + entry.insert("value".to_string(), Value::String(value.clone())); + entry.insert("hidden_if_empty".to_string(), Value::Bool(false)); + entries.push(Value::Object(entry)); + } + entries +} + +fn sync_rendered_scalar_config_value( + existing: Option, + rendered: &Map, + descriptor: ScalarConfigDescriptor, +) -> Result> { + match existing { + Some(Value::Object(mut config_obj)) => { + let nested_key = if config_obj.contains_key("default") { + Some("default") + } else if config_obj.contains_key(DEFAULT_DELIMITER) { + Some(DEFAULT_DELIMITER) + } else { + None + }; + + let direct_had_known = config_obj.keys().any(|key| descriptor.valid_keys.contains(&key.as_str())); + for key in descriptor.valid_keys { + config_obj.remove(*key); + } + + if let Some(nested_key) = nested_key { + let nested = config_obj.remove(nested_key); + let synced = if direct_had_known { + sync_rendered_scalar_config_value(nested, &Map::new(), descriptor)? + } else { + sync_rendered_scalar_config_value(nested, rendered, descriptor)? + }; + if let Some(value) = synced { + config_obj.insert(nested_key.to_string(), value); + } + } + + if direct_had_known || nested_key.is_none() { + config_obj.extend(rendered.clone()); + } + + Ok((!config_obj.is_empty()).then_some(Value::Object(config_obj))) + } + Some(Value::Array(entries)) => { + let mut pending = rendered + .iter() + .map(|(key, value)| { + parse_target_scalar_value(key, value) + .map(|value| (key.clone(), value)) + .ok_or_else(|| { + Error::other(format!( + "{} config value for {key} cannot be represented as KVS", + descriptor.subsystem_key + )) + }) + }) + .collect::>>()?; + let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len())); + for entry in entries { + let Some(entry_obj) = entry.as_object() else { + updated.push(entry); + continue; + }; + let Some(key) = entry_obj.get("key").and_then(Value::as_str) else { + updated.push(entry); + continue; + }; + if !descriptor.valid_keys.contains(&key) { + updated.push(entry); + continue; + } + let Some(value) = pending.remove(key) else { + continue; + }; + let mut entry_obj = entry_obj.clone(); + entry_obj.insert("value".to_string(), Value::String(value)); + updated.push(Value::Object(entry_obj)); + } + updated.extend(rendered_scalar_config_kvs_entries(&pending)); + Ok((!updated.is_empty()).then_some(Value::Array(updated))) + } + Some(value) if rendered.is_empty() => Ok(Some(value)), + _ if rendered.is_empty() => Ok(None), + _ => Ok(Some(Value::Object(rendered.clone()))), + } +} + fn build_oidc_provider_object(kvs: &KVS) -> Map { let mut provider = Map::new(); @@ -986,6 +1211,73 @@ fn build_oidc_object(cfg: &Config) -> Map { oidc_obj } +fn sync_rendered_oidc_provider(existing: Value, rendered: Option<&Value>) -> Option { + match existing { + Value::Object(mut provider) => { + for key in IDENTITY_OPENID_KEYS { + provider.remove(*key); + } + if let Some(Value::Object(rendered)) = rendered { + provider.extend(rendered.clone()); + } + (!provider.is_empty()).then_some(Value::Object(provider)) + } + Value::Array(entries) => { + let mut pending = rendered + .and_then(Value::as_object) + .map(|rendered| { + rendered + .iter() + .filter_map(|(key, value)| parse_oidc_scalar_value(key, value).map(|value| (key.clone(), value))) + .collect::>() + }) + .unwrap_or_default(); + let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len())); + for entry in entries { + let Some(entry_obj) = entry.as_object() else { + updated.push(entry); + continue; + }; + let Some(key) = entry_obj.get("key").and_then(Value::as_str) else { + updated.push(entry); + continue; + }; + if !IDENTITY_OPENID_KEYS.contains(&key) { + updated.push(entry); + continue; + } + let Some(value) = pending.remove(key) else { + continue; + }; + let mut entry_obj = entry_obj.clone(); + entry_obj.insert("value".to_string(), Value::String(value)); + updated.push(Value::Object(entry_obj)); + } + updated.extend(rendered_scalar_config_kvs_entries(&pending)); + (!updated.is_empty()).then_some(Value::Array(updated)) + } + value if rendered.is_none() => Some(value), + _ => rendered.cloned(), + } +} + +fn sync_rendered_oidc_object(existing: Option, rendered: &Map) -> Option { + let existing = match existing { + Some(Value::Object(existing)) => existing, + _ => Map::new(), + }; + let mut merged = Map::new(); + for (provider_key, provider) in existing { + if let Some(provider) = sync_rendered_oidc_provider(provider, rendered.get(&provider_key)) { + merged.insert(provider_key, provider); + } + } + for (provider_key, provider) in rendered { + merged.entry(provider_key.clone()).or_insert_with(|| provider.clone()); + } + (!merged.is_empty()).then_some(Value::Object(merged)) +} + fn build_semantic_oidc_object(cfg: &Config) -> Map { let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else { return Map::new(); @@ -1210,15 +1502,20 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result v, _ => Map::new(), @@ -1267,6 +1564,9 @@ fn is_standard_object_server_config(data: &[u8]) -> bool { fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool { build_storageclass_object(lhs) == build_storageclass_object(rhs) + && build_scalar_config_object(lhs, scanner_config_descriptor()) + == build_scalar_config_object(rhs, scanner_config_descriptor()) + && build_scalar_config_object(lhs, heal_config_descriptor()) == build_scalar_config_object(rhs, heal_config_descriptor()) && build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs) && build_notify_object(lhs) == build_notify_object(rhs) && build_audit_object(lhs) == build_audit_object(rhs) @@ -1299,7 +1599,7 @@ where register_server_config_decrypt_fn(decrypt.clone()); } - let config_file = get_config_file(); + let config_file = server_config_path(); match api .get_object_info( RUSTFS_META_BUCKET, @@ -1390,7 +1690,7 @@ where /// Handle the situation where the configuration file does not exist, create and save a new configuration async fn handle_missing_config(api: Arc, context: &str, namespace_lock_held: bool) -> Result where - S: EcstoreObjectIO + StorageAdminApi, + S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { warn!("Configuration not found ({}): Start initializing new configuration", context); let cfg = if runtime_sources::first_cluster_node_is_local().await { @@ -1407,7 +1707,7 @@ where } /// Handle configuration file read errors -fn handle_config_read_error(err: Error, file_path: &str) -> Result { +fn handle_config_read_error(err: Error, file_path: &str) -> Result { error!("Read configuration failed (path: '{}'): {:?}", file_path, err); Err(err) } @@ -1423,6 +1723,7 @@ where GetObjectReader = GetObjectReader, PutObjectReader = PutObjReader, > + StorageAdminApi, + S: NamespaceLocking, { read_config_without_migrate_inner(api, false).await } @@ -1440,7 +1741,8 @@ where ObjectInfo = ObjectInfo, GetObjectReader = GetObjectReader, PutObjectReader = PutObjReader, - > + StorageAdminApi, + > + StorageAdminApi + + NamespaceLocking, { read_config_without_migrate_inner(api, true).await } @@ -1462,9 +1764,10 @@ where ObjectInfo = ObjectInfo, GetObjectReader = GetObjectReader, PutObjectReader = PutObjReader, - > + StorageAdminApi, + > + StorageAdminApi + + NamespaceLocking, { - let config_file = get_config_file(); + let config_file = server_config_path(); // Try to read the configuration file match read_config_no_lock(api.clone(), &config_file).await { @@ -1476,11 +1779,11 @@ where async fn read_server_config(api: Arc, data: &[u8], namespace_lock_held: bool) -> Result where - S: EcstoreObjectIO + StorageAdminApi, + S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { // If the provided data is empty, try to read from the file again if data.is_empty() { - let config_file = get_config_file(); + let config_file = server_config_path(); warn!("Received empty configuration data, try to reread from '{}'", config_file); // Try to read the configuration again @@ -1504,8 +1807,12 @@ where /// Decode the persisted server config blob, marking decode failures as /// deterministic corruption (see [`ServerConfigCorruptError`]). fn decode_persisted_server_config(data: &[u8]) -> Result { + decode_persisted_server_config_with_seed(data).map(|(config, _)| config) +} + +fn decode_persisted_server_config_with_seed(data: &[u8]) -> Result<(Config, Vec)> { match decode_server_config_blob(data) { - Ok(cfg) => Ok(cfg), + Ok(cfg) => Ok((cfg, data.to_vec())), Err(raw_decode_err) => { let Some(decrypt) = server_config_decrypt_fn() else { error!( @@ -1531,18 +1838,20 @@ fn decode_persisted_server_config(data: &[u8]) -> Result { return Err(Error::other(ServerConfigDecryptError(raw_decode_err.to_string()))); }; - decode_server_config_blob(&decrypted).map_err(|err| { - error!( - event = EVENT_SERVER_CONFIG_DECODE_FAILED, - component = LOG_COMPONENT_CONFIG, - subsystem = LOG_SUBSYSTEM_CONFIG, - size = decrypted.len(), - encrypted_size = data.len(), - error = %err, - "decrypted persisted server config cannot be decoded, object is corrupt" - ); - Error::other(ServerConfigCorruptError(err.to_string())) - }) + decode_server_config_blob(&decrypted) + .map(|config| (config, decrypted.clone())) + .map_err(|err| { + error!( + event = EVENT_SERVER_CONFIG_DECODE_FAILED, + component = LOG_COMPONENT_CONFIG, + subsystem = LOG_SUBSYSTEM_CONFIG, + size = decrypted.len(), + encrypted_size = data.len(), + error = %err, + "decrypted persisted server config cannot be decoded, object is corrupt" + ); + Error::other(ServerConfigCorruptError(err.to_string())) + }) } } } @@ -1561,14 +1870,17 @@ fn decode_persisted_server_config(data: &[u8]) -> Result { /// to the caller so the startup retry loop can wait for disks to come back. pub(crate) async fn read_config_without_migrate_with_recovery(api: Arc) -> Result where - S: EcstoreObjectIO + StorageAdminApi + HealOperations, + S: EcstoreObjectIO + + StorageAdminApi + + HealOperations + + NamespaceLocking, { let first_err = match read_config_without_migrate(api.clone()).await { Ok(cfg) => return Ok(cfg), Err(err) => err, }; - let config_file = get_config_file(); + let config_file = server_config_path(); warn!( event = EVENT_SERVER_CONFIG_READ_FAILED, component = LOG_COMPONENT_CONFIG, @@ -1710,9 +2022,149 @@ where ObjectInfo = ObjectInfo, GetObjectReader = GetObjectReader, PutObjectReader = PutObjReader, - >, + > + NamespaceLocking, { - save_server_config_inner(api, cfg, false).await + let snapshot = read_server_config_snapshot(api.clone()).await?; + save_server_config_snapshot(api, cfg, &snapshot).await.map(|_| ()) +} + +/// One serialized read of the persisted server config used as the seed and +/// compare-and-swap fence for an admin update. +#[derive(Debug)] +pub struct ServerConfigSnapshot { + pub config: Config, + raw: Option>, + seed: Option>, + etag: Option, + _local_guard: OwnedRwLockWriteGuard<()>, + _guard: rustfs_lock::NamespaceLockGuard, +} + +impl ServerConfigSnapshot { + pub fn ensure_lock_held(&self) -> Result<()> { + if self._guard.is_lock_lost() { + return Err(Error::other("server config transaction lock was lost")); + } + Ok(()) + } + + pub fn is_lock_lost(&self) -> bool { + self._guard.is_lock_lost() + } +} + +/// Read a server config transaction snapshot while holding the same local and +/// distributed write locks used by every other server-config writer. Internal +/// reads and the later conditional write use no-lock object I/O; the guards +/// remain live until the snapshot is dropped. +pub async fn read_server_config_snapshot(api: Arc) -> Result +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + NamespaceLocking, +{ + let config_file = server_config_path(); + let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await; + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &config_file).await?; + let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?; + let read_options = ObjectOptions { + no_lock: true, + ..Default::default() + }; + match read_config_with_metadata_inner(api, &config_file, &read_options, true).await { + Ok((raw, object_info)) => { + let (config, seed) = decode_persisted_server_config_with_seed(&raw)?; + Ok(ServerConfigSnapshot { + config: config.merge(), + raw: Some(raw), + seed: Some(seed), + etag: object_info.etag, + _local_guard: local_guard, + _guard: guard, + }) + } + Err(Error::ConfigNotFound) => Ok(ServerConfigSnapshot { + config: new_server_config(), + raw: None, + seed: None, + etag: None, + _local_guard: local_guard, + _guard: guard, + }), + Err(err) => handle_config_read_error(err, &config_file), + } +} + +/// Conditionally persist a server config derived from `snapshot`. +/// +/// Existing objects use `If-Match`; missing objects use `If-None-Match: *`. +/// The object layer evaluates the precondition while holding its own write +/// lock, so a concurrent update or transaction lease loss cannot commit an +/// unfenced overwrite. +pub async fn save_server_config_snapshot(api: Arc, cfg: &Config, snapshot: &ServerConfigSnapshot) -> Result +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + NamespaceLocking, +{ + snapshot.ensure_lock_held()?; + let config_file = server_config_path(); + if let Some(seed) = snapshot.seed.as_deref() + && is_standard_object_server_config(seed) + && configs_semantically_equal(&snapshot.config, cfg) + { + debug!("server config unchanged and already in standard object shape, skip write"); + return Ok(false); + } + + let data = encode_server_config_blob(cfg, snapshot.seed.as_deref())?; + if snapshot.raw.as_deref().is_some_and(|current| current == data.as_slice()) { + debug!("server config bytes unchanged after encode, skip write"); + return Ok(false); + } + + let http_preconditions = if snapshot.raw.is_some() { + let etag = snapshot + .etag + .clone() + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("persisted server config has no ETag for conditional update"))?; + HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + } + } else { + HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + } + }; + + save_config_with_opts( + api, + &config_file, + data, + &ObjectOptions { + max_parity: true, + no_lock: true, + http_preconditions: Some(http_preconditions), + ..Default::default() + }, + ) + .await?; + Ok(true) } /// Saves the server config while an upper layer holds the namespace write @@ -1847,11 +2299,12 @@ where #[cfg(test)] mod tests { use super::{ - apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob, - encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, - read_config_with_metadata, storage_class_kvs_mut, + SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error, + configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, + lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate, + read_server_config_snapshot, save_server_config, save_server_config_snapshot, server_config_path, storage_class_kvs_mut, }; - use crate::config::{audit, notify, oidc}; + use crate::config::{audit, heal, notify, oidc, scanner}; use crate::disk::endpoint::Endpoint; use crate::error::{Error, Result}; use crate::layout::endpoints::SetupType; @@ -1867,7 +2320,9 @@ mod tests { use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use rustfs_config::server_config::{Config, KV, KVS}; use rustfs_config::{ - DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, + DEFAULT_DELIMITER, ENABLE_KEY, EnableState, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, MYSQL_DSN_STRING, + MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, SCANNER_CYCLE, SCANNER_IDLE_MODE, SCANNER_KEYS, SCANNER_SPEED, + SCANNER_SUB_SYS, }; #[tokio::test] @@ -1890,9 +2345,10 @@ mod tests { use std::io::Cursor; use std::pin::Pin; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::task::{Context, Poll}; use time::OffsetDateTime; - use tokio::io::{AsyncRead, ReadBuf}; + use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio::sync::RwLock; #[derive(Debug, Default)] @@ -1937,6 +2393,67 @@ mod tests { } } + #[derive(Debug)] + struct RefreshControlledClient { + inner: LocalClient, + refresh_allowed: AtomicBool, + } + + impl RefreshControlledClient { + fn new(manager: Arc) -> Self { + Self { + inner: LocalClient::with_manager(manager), + refresh_allowed: AtomicBool::new(true), + } + } + + fn reject_refreshes(&self) { + self.refresh_allowed.store(false, Ordering::Release); + } + } + + #[async_trait::async_trait] + impl LockClient for RefreshControlledClient { + async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result { + self.inner.acquire_lock(request).await + } + + async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + self.inner.release(lock_id).await + } + + async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + if !self.refresh_allowed.load(Ordering::Acquire) { + return Ok(false); + } + self.inner.refresh(lock_id).await + } + + async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + self.inner.force_release(lock_id).await + } + + async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result> { + self.inner.check_status(lock_id).await + } + + async fn get_stats(&self) -> rustfs_lock::Result { + self.inner.get_stats().await + } + + async fn close(&self) -> rustfs_lock::Result<()> { + self.inner.close().await + } + + async fn is_online(&self) -> bool { + self.inner.is_online().await + } + + async fn is_local(&self) -> bool { + self.inner.is_local().await + } + } + struct GuardedCursor { inner: Cursor>, _guard: Option, @@ -2503,6 +3020,382 @@ mod tests { assert!(v.get("storage_class").is_none(), "should not write rustfs map shape"); } + fn config_with_scanner_cycle(cycle: &str) -> Config { + let mut cfg = Config::new(); + let mut kvs = scanner::DEFAULT_KVS.clone(); + kvs.insert(SCANNER_CYCLE.to_string(), cycle.to_string()); + cfg.0 + .entry(SCANNER_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + cfg + } + + fn config_with_heal_cycle(cycle: &str) -> Config { + let mut cfg = Config::new(); + let mut kvs = heal::DEFAULT_KVS.clone(); + kvs.insert(HEAL_BITROT_CYCLE.to_string(), cycle.to_string()); + cfg.0 + .entry(HEAL_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + cfg + } + + #[test] + fn test_external_scanner_config_decodes_with_defaults() { + let cfg = + decode_server_config_blob(br#"{"version":"33","storageclass":{"standard":"","rrs":""},"scanner":{"cycle":"61"}}"#) + .expect("external scanner config should decode"); + let scanner_kvs = cfg + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("scanner subsystem should be present"); + + assert_eq!(scanner_kvs.get(SCANNER_CYCLE), "61"); + assert_eq!(scanner_kvs.get(SCANNER_SPEED), scanner::DEFAULT_KVS.get(SCANNER_SPEED)); + } + + #[test] + fn test_scanner_config_roundtrip_in_external_shape() { + let cfg = config_with_scanner_cycle("61"); + let encoded = encode_server_config_blob(&cfg, None).expect("scanner config should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + assert_eq!( + value + .get(SCANNER_SUB_SYS) + .and_then(Value::as_object) + .and_then(|section| section.get(SCANNER_CYCLE)) + .and_then(Value::as_str), + Some("61") + ); + + let decoded = decode_server_config_blob(&encoded).expect("encoded scanner config should decode"); + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("scanner config should survive roundtrip") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[test] + fn test_heal_config_roundtrip_in_external_shape() { + let cfg = config_with_heal_cycle("off"); + let encoded = encode_server_config_blob(&cfg, None).expect("heal config should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + assert_eq!( + value + .get(HEAL_SUB_SYS) + .and_then(Value::as_object) + .and_then(|section| section.get(HEAL_BITROT_CYCLE)) + .and_then(Value::as_str), + Some("off") + ); + + let decoded = decode_server_config_blob(&encoded).expect("encoded heal config should decode"); + assert_eq!( + decoded + .get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER) + .expect("heal config should survive roundtrip") + .get(HEAL_BITROT_CYCLE), + "off" + ); + } + + #[test] + fn scanner_update_preserves_unknown_root_and_oidc_provider_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "future_root":{"mode":"keep"}, + "openid":{"default":{ + "config_url":"https://issuer.example/.well-known/openid-configuration", + "client_id":"console", + "future_provider_control":"keep" + }} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("seed config should decode"); + cfg.0 + .entry(SCANNER_SUB_SYS.to_string()) + .or_default() + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(|| scanner::DEFAULT_KVS.clone()) + .insert(SCANNER_CYCLE.to_string(), "61".to_string()); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("scanner update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + assert_eq!(value["future_root"]["mode"].as_str(), Some("keep")); + assert_eq!(value["openid"]["default"]["future_provider_control"].as_str(), Some("keep")); + assert_eq!(value["openid"]["default"]["client_id"].as_str(), Some("console")); + } + + #[test] + fn test_scanner_config_changes_are_semantically_significant() { + let baseline = Config::new(); + let scanner_changed = config_with_scanner_cycle("61"); + let heal_changed = config_with_heal_cycle("off"); + + assert!(!configs_semantically_equal(&baseline, &scanner_changed)); + assert!(!configs_semantically_equal(&baseline, &heal_changed)); + } + + #[test] + fn test_scanner_config_reset_removes_override_and_preserves_unknown_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "scanner":{"cycle":"61","future_control":"keep"} + }"#; + let cfg = config_with_scanner_cycle(""); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("reset scanner config should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let scanner = value + .get(SCANNER_SUB_SYS) + .and_then(Value::as_object) + .expect("unknown scanner fields should be preserved"); + + assert!(scanner.get(SCANNER_CYCLE).is_none(), "default cycle must not remain persisted"); + assert_eq!(scanner.get("future_control").and_then(Value::as_str), Some("keep")); + } + + #[test] + fn test_encode_decode_roundtrip_preserves_all_scanner_keys() { + let mut cfg = Config::new(); + let scanner_kvs = cfg + .0 + .entry(SCANNER_SUB_SYS.to_string()) + .or_default() + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(|| scanner::DEFAULT_KVS.clone()); + + for key in SCANNER_KEYS { + let value = match *key { + SCANNER_SPEED => "fast", + SCANNER_IDLE_MODE => "false", + _ => "1", + }; + scanner_kvs.insert((*key).to_string(), value.to_string()); + } + + let encoded = encode_server_config_blob(&cfg, None).expect("scanner config should encode"); + let external: Value = serde_json::from_slice(&encoded).expect("encoded scanner config should be valid json"); + let external_scanner = external + .get(SCANNER_SUB_SYS) + .and_then(Value::as_object) + .expect("encoded scanner config should exist"); + assert!(SCANNER_KEYS.iter().all(|key| external_scanner.contains_key(*key))); + + let decoded = decode_server_config_blob(&encoded).expect("scanner config should decode"); + let decoded_scanner = decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("decoded scanner config should exist"); + for key in SCANNER_KEYS { + let expected = match *key { + SCANNER_SPEED => "fast", + SCANNER_IDLE_MODE => "false", + _ => "1", + }; + assert_eq!(decoded_scanner.get(key), expected, "scanner key {key} should survive roundtrip"); + } + } + + #[test] + fn test_decode_accepts_scanner_only_external_config() { + let decoded = decode_server_config_blob(br#"{"scanner":{"cycle":"61"}}"#).expect("scanner-only config should decode"); + + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("decoded scanner config should exist") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[test] + fn test_decode_accepts_nested_scanner_config() { + let decoded = + decode_server_config_blob(br#"{"scanner":{"default":{"cycle":"61"}}}"#).expect("nested scanner config should decode"); + + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("decoded scanner config should exist") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[test] + fn test_decode_accepts_scanner_kvs_array() { + let decoded = decode_server_config_blob(br#"{"scanner":[{"key":"cycle","value":"61","hidden_if_empty":false}]}"#) + .expect("scanner KVS array should decode"); + + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("decoded scanner config should exist") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[test] + fn test_scanner_config_preserves_unknown_nested_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "scanner":{ + "default":{"cycle":"61","future_control":"keep"}, + "future_section":{"mode":"keep"} + } + }"#; + let cfg = config_with_scanner_cycle("62"); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("nested scanner config should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let scanner = value + .get(SCANNER_SUB_SYS) + .and_then(Value::as_object) + .expect("nested scanner config should remain an object"); + let default = scanner + .get("default") + .and_then(Value::as_object) + .expect("nested default scanner config should remain an object"); + + assert_eq!(default.get(SCANNER_CYCLE).and_then(Value::as_str), Some("62")); + assert_eq!(default.get("future_control").and_then(Value::as_str), Some("keep")); + assert_eq!( + scanner + .get("future_section") + .and_then(Value::as_object) + .and_then(|section| section.get("mode")) + .and_then(Value::as_str), + Some("keep") + ); + } + + #[test] + fn test_scanner_config_reset_clears_nested_override_without_losing_unknown_fields() { + let seed = br#"{ + "version":"33", + "scanner":{ + "default":{"cycle":"61","future_control":"keep"}, + "future_section":{"mode":"keep"} + } + }"#; + let cfg = config_with_scanner_cycle(""); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("nested scanner reset should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let scanner = value[SCANNER_SUB_SYS].as_object().expect("scanner object"); + let default = scanner["default"].as_object().expect("nested default scanner object"); + + assert!(default.get(SCANNER_CYCLE).is_none()); + assert_eq!(default.get("future_control").and_then(Value::as_str), Some("keep")); + assert_eq!(scanner["future_section"]["mode"].as_str(), Some("keep")); + } + + #[test] + fn test_direct_scanner_shape_clears_stale_nested_known_keys() { + let seed = br#"{ + "version":"33", + "scanner":{ + "cycle":"61", + "default":{"cycle":"59","future_control":"keep"} + } + }"#; + let cfg = config_with_scanner_cycle("62"); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("mixed scanner shape should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let scanner = value[SCANNER_SUB_SYS].as_object().expect("scanner object"); + + assert_eq!(scanner.get(SCANNER_CYCLE).and_then(Value::as_str), Some("62")); + assert!(scanner["default"].get(SCANNER_CYCLE).is_none()); + assert_eq!(scanner["default"]["future_control"].as_str(), Some("keep")); + let decoded = decode_server_config_blob(&encoded).expect("mixed scanner shape should decode"); + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("decoded scanner") + .get(SCANNER_CYCLE), + "62" + ); + } + + #[test] + fn test_scanner_config_preserves_unknown_kvs_entries() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "scanner":[ + {"key":"cycle","value":"61","hidden_if_empty":false,"future_attribute":"keep-cycle"}, + {"key":"future_control","value":"keep","future_attribute":"keep"} + ] + }"#; + let cfg = config_with_scanner_cycle("62"); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("scanner KVS array should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let scanner = value + .get(SCANNER_SUB_SYS) + .and_then(Value::as_array) + .expect("scanner KVS config should remain an array"); + let cycle = scanner + .iter() + .find(|entry| entry.get("key").and_then(Value::as_str) == Some(SCANNER_CYCLE)) + .expect("updated scanner cycle should remain in the KVS array"); + let future = scanner + .iter() + .find(|entry| entry.get("key").and_then(Value::as_str) == Some("future_control")) + .expect("unknown scanner KVS entry should be preserved"); + + assert_eq!(cycle.get("value").and_then(Value::as_str), Some("62")); + assert_eq!(cycle.get("hidden_if_empty").and_then(Value::as_bool), Some(false)); + assert_eq!(cycle.get("future_attribute").and_then(Value::as_str), Some("keep-cycle")); + assert_eq!(future.get("value").and_then(Value::as_str), Some("keep")); + assert_eq!(future.get("future_attribute").and_then(Value::as_str), Some("keep")); + } + + #[test] + fn test_decode_rejects_non_scalar_known_scanner_value() { + let err = decode_server_config_blob(br#"{"version":"33","scanner":{"cycle":{"seconds":61}}}"#) + .expect_err("non-scalar scanner values should be rejected"); + + assert!( + err.to_string() + .contains("invalid external scanner config value for cycle: expected a scalar"), + "unexpected scanner decode error: {err}" + ); + } + + #[test] + fn test_decode_rejects_scalar_scanner_section() { + let err = decode_server_config_blob(br#"{"version":"33","scanner":"cycle=61"}"#) + .expect_err("scalar scanner sections should be rejected"); + + assert!( + err.to_string() + .contains("invalid external scanner config shape: expected an object or KVS array"), + "unexpected scanner decode error: {err}" + ); + } + + #[test] + fn test_decode_rejects_malformed_known_scanner_kvs_value() { + let err = decode_server_config_blob(br#"{"scanner":[{"key":"cycle","value":{"seconds":61}}]}"#) + .expect_err("non-scalar scanner KVS values should be rejected"); + + assert!( + err.to_string() + .contains("invalid external scanner config value for cycle: expected a scalar"), + "unexpected scanner decode error: {err}" + ); + } + #[test] fn test_encode_server_config_writes_openid_object_shape() { let mut cfg = Config::new(); @@ -3173,7 +4066,6 @@ mod tests { }; use rustfs_common::heal_channel::HealOpts; use std::sync::Mutex; - use std::sync::atomic::{AtomicUsize, Ordering}; /// Bytes mirroring issue #4156: a bitrot-corrupted `config.json` whose /// shards decoded into garbage that is valid neither as the internal nor @@ -3241,7 +4133,12 @@ mod tests { state: Mutex, heal_replacement: Option>, heal_calls: AtomicUsize, + write_calls: AtomicUsize, + last_put_no_lock: AtomicBool, + revision: AtomicUsize, drive_counts: Vec, + lock_manager: Arc, + lock_resources: Mutex>, } impl RecoveryMockStore { @@ -3250,7 +4147,12 @@ mod tests { state: Mutex::new(state), heal_replacement, heal_calls: AtomicUsize::new(0), + write_calls: AtomicUsize::new(0), + last_put_no_lock: AtomicBool::new(false), + revision: AtomicUsize::new(1), drive_counts: vec![2], + lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()), + lock_resources: Mutex::new(Vec::new()), } } @@ -3309,6 +4211,7 @@ mod tests { let object_info = ObjectInfo { size: data.len() as i64, actual_size: data.len() as i64, + etag: Some(format!("config-{}", self.revision.load(Ordering::SeqCst))), ..Default::default() }; Ok(GetObjectReader { @@ -3323,13 +4226,169 @@ mod tests { &self, _bucket: &str, _object: &str, - _data: &mut PutObjReader, - _opts: &ObjectOptions, + data: &mut PutObjReader, + opts: &ObjectOptions, ) -> Result { - panic!("unused in test") + let current_etag = format!("config-{}", self.revision.load(Ordering::SeqCst)); + if let Some(preconditions) = &opts.http_preconditions + && (preconditions.if_match_value().is_some_and(|etag| etag != current_etag) + || preconditions.if_none_match_value() == Some("*")) + { + return Err(Error::PreconditionFailed); + } + let mut body = Vec::new(); + data.stream.read_to_end(&mut body).await?; + self.last_put_no_lock.store(opts.no_lock, Ordering::SeqCst); + self.write_calls.fetch_add(1, Ordering::SeqCst); + *self.state.lock().expect("state lock poisoned") = RecoveryReadState::Blob(body.clone()); + let revision = self.revision.fetch_add(1, Ordering::SeqCst) + 1; + Ok(ObjectInfo { + size: i64::try_from(body.len()).expect("test config should fit in i64"), + actual_size: i64::try_from(body.len()).expect("test config should fit in i64"), + etag: Some(format!("config-{revision}")), + ..Default::default() + }) } } + #[async_trait::async_trait] + impl crate::storage_api_contracts::namespace::NamespaceLocking for RecoveryMockStore { + type Error = Error; + type NamespaceLock = rustfs_lock::NamespaceLockWrapper; + + async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { + self.lock_resources + .lock() + .expect("lock resources mutex poisoned") + .push(object.to_string()); + Ok(rustfs_lock::NamespaceLockWrapper::new( + rustfs_lock::NamespaceLock::with_local_manager( + "server-config-recovery-mock".to_string(), + Arc::clone(&self.lock_manager), + ), + rustfs_lock::ObjectKey::new(bucket, object), + "server-config-recovery-mock".to_string(), + )) + } + } + + #[tokio::test] + async fn save_server_config_persists_scanner_only_changes() { + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + let cfg = config_with_scanner_cycle("61"); + + save_server_config(store.clone(), &cfg) + .await + .expect("scanner-only config change should be persisted"); + + assert_eq!(store.write_calls.load(Ordering::SeqCst), 1); + assert!(store.last_put_no_lock.load(Ordering::SeqCst)); + assert_eq!( + store.lock_resources.lock().expect("lock resources mutex poisoned").as_slice(), + &[server_config_path()] + ); + let decoded = read_config_without_migrate(store) + .await + .expect("persisted scanner config should reload"); + assert_eq!( + decoded + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("persisted config should contain scanner settings") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[tokio::test] + async fn stale_server_config_snapshot_cannot_overwrite_newer_update() { + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + let stale = read_server_config_snapshot(store.clone()) + .await + .expect("stale config snapshot"); + + let newer = encode_server_config_blob(&config_with_scanner_cycle("61"), None).expect("newer config should encode"); + *store.state.lock().expect("state lock") = RecoveryReadState::Blob(newer); + store.revision.fetch_add(1, Ordering::SeqCst); + let err = save_server_config_snapshot(store.clone(), &config_with_scanner_cycle("62"), &stale) + .await + .expect_err("stale conditional update must fail"); + drop(stale); + + assert_eq!(err, Error::PreconditionFailed); + let persisted = read_config_without_migrate(store) + .await + .expect("persisted config should reload"); + assert_eq!( + persisted + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("persisted scanner config") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[tokio::test] + async fn server_config_snapshot_serializes_read_modify_write_transactions() { + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + let first = read_server_config_snapshot(store.clone()) + .await + .expect("first config snapshot"); + + let blocked = + tokio::time::timeout(std::time::Duration::from_millis(20), read_server_config_snapshot(store.clone())).await; + assert!(blocked.is_err(), "a second config transaction must wait for the first snapshot guard"); + + drop(first); + let second = tokio::time::timeout(std::time::Duration::from_secs(1), read_server_config_snapshot(store)) + .await + .expect("second transaction should acquire after the first snapshot is dropped") + .expect("second config snapshot"); + assert!(configs_semantically_equal(&second.config, &Config::new())); + } + + #[tokio::test] + async fn server_config_snapshot_rejects_write_after_distributed_lease_loss() { + let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); + let client = Arc::new(RefreshControlledClient::new(manager)); + let lock = rustfs_lock::NamespaceLock::new("server-config-lease-loss".to_string(), client.clone()); + let guard = lock + .lock_guard( + rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_path()), + "server-config-lease-loss", + std::time::Duration::from_secs(1), + std::time::Duration::from_millis(120), + ) + .await + .expect("distributed config lock acquisition should not error") + .expect("distributed config lock should be acquired"); + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await; + let snapshot = ServerConfigSnapshot { + config: Config::new(), + raw: Some(baseline.clone()), + seed: None, + etag: Some("config-0".to_string()), + _local_guard: local_guard, + _guard: guard, + }; + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + + client.reject_refreshes(); + tokio::time::timeout(std::time::Duration::from_secs(2), snapshot._guard.lock_lost_notified()) + .await + .expect("heartbeat should report the removed backend lease"); + + let err = save_server_config_snapshot(store.clone(), &config_with_scanner_cycle("62"), &snapshot) + .await + .expect_err("a lost config lease must fence persistence"); + + assert!(err.to_string().contains("transaction lock was lost")); + assert_eq!(store.write_calls.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn read_config_preserve_empty_distinguishes_empty_object() { let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None)); diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index f52820393..5f045b8e3 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -560,17 +560,22 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?; - set.delete_object( - bucket, - cleanup_key.as_str(), - ObjectOptions { - delete_prefix: true, - delete_prefix_object: true, - no_lock: true, - ..Default::default() - }, - ) - .await + let result = set + .delete_object( + bucket, + cleanup_key.as_str(), + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + no_lock: true, + ..Default::default() + }, + ) + .await; + if result.is_ok() { + crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1); + } + result } fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool { diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 6b41c994e..73f8f1c1c 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -20,21 +20,21 @@ pub mod local_snapshot; use crate::storage_api_contracts::{ bucket::{BucketOperations as _, BucketOptions}, list::{ListOperations as _, StorageListObjectVersionsInfo}, - object::ObjectIO as _, + object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _}, }; use crate::{ bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys}, config::com::{read_config, read_config_preserve_empty}, - disk::DiskAPI, + disk::{DiskAPI, RUSTFS_META_BUCKET}, error::{Error, classify_system_path_failure_reason}, - object_api::ObjectInfo, + object_api::{ObjectInfo, ObjectOptions, PutObjReader}, runtime::sources as runtime_sources, store::{ECStore, list_objects::list_marker_key}, }; pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path}; use rustfs_data_usage::{ - BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, - SizeHistogram, SizeSummary, VersionsHistogram, + BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DataUsageCache, DataUsageEntry, + DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, VersionsHistogram, }; use rustfs_io_metrics::record_system_path_failure; use rustfs_utils::path::SLASH_SEPARATOR; @@ -53,12 +53,12 @@ use tracing::{debug, error, info, instrument}; // Data usage storage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; -const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json"; const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; const DATA_USAGE_CACHE_TTL_SECS: u64 = 30; const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024; +const DATA_USAGE_REMOVE_CAS_RETRIES: usize = 3; #[derive(Debug, Clone)] struct CachedBucketUsage { @@ -201,9 +201,15 @@ lazy_static::lazy_static! { pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, SLASH_SEPARATOR, - DATA_USAGE_OBJ_NAME + DATA_USAGE_OBJECT_NAME ); static ref DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + static ref LEGACY_DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", + crate::disk::BUCKET_META_PREFIX, + SLASH_SEPARATOR, + LEGACY_DATA_USAGE_OBJECT_NAME + ); + static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, SLASH_SEPARATOR, @@ -312,39 +318,240 @@ fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &s true } -fn merge_bucket_usage_removal(candidate: DataUsageInfo, existing: Option, bucket: &str) -> Option { - let mut data_usage_info = match existing { - Some(existing) if data_usage_info_updated_at(&existing) >= data_usage_info_updated_at(&candidate) => existing, - _ => candidate, - }; - - if remove_bucket_usage_from_info(&mut data_usage_info, bucket) { - Some(data_usage_info) - } else { - None - } -} - -async fn clear_bucket_usage_memory(bucket: &str) { +async fn clear_bucket_usage_memory(bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<(), Error> { if bucket.is_empty() { - return; + return Ok(()); } - memory_cache().write().await.remove(bucket); + let mut cache = memory_cache().write().await; + ensure_bucket_namespace_guard(guard, bucket, "data usage memory cleanup")?; + cache.remove(bucket); + Ok(()) } pub async fn remove_bucket_usage_from_backend(store: Arc, bucket: &str) -> Result<(), Error> { + remove_bucket_usage_for_namespace_change(store.as_ref(), bucket).await +} + +pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> { + prepare_bucket_usage_for_namespace_change(bucket, None).await?; + remove_bucket_usage_from_backend_with_guard(store, bucket, None).await +} + +pub(crate) async fn prepare_bucket_usage_for_namespace_change( + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> { + ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?; live_bucket_usage_cache().invalidate(bucket).await; - clear_bucket_usage_memory(bucket).await; + clear_bucket_usage_memory(bucket, guard).await?; - let data_usage_info = load_primary_data_usage_from_backend(store.clone()).await?; - let existing = load_primary_data_usage_from_backend(store.clone()).await.ok(); + let mut snapshot_cache = data_usage_snapshot_cache().write().await; + ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?; + *snapshot_cache = None; + Ok(()) +} - if let Some(data_usage_info) = merge_bucket_usage_removal(data_usage_info, existing, bucket) { - save_data_usage_in_backend(data_usage_info, store).await?; +pub(crate) async fn remove_bucket_usage_from_backend_with_guard( + store: &S, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await; + let mut snapshot_cache = data_usage_snapshot_cache().write().await; + ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?; + *snapshot_cache = None; + result +} + +async fn load_data_usage_for_bucket_removal(store: &S, object: &str) -> Result, Error> +where + S: EcstoreObjectIO + ?Sized, +{ + let mut reader = match store + .get_object_reader(RUSTFS_META_BUCKET, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + { + Ok(reader) => reader, + Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None), + Err(err) => return Err(err), + }; + let revision = reader + .object_info + .etag + .as_deref() + .filter(|etag| !etag.is_empty()) + .map(str::to_owned) + .ok_or_else(|| Error::other("data usage snapshot has no ETag"))?; + let data_usage_info = normalize_loaded_data_usage(parse_usage_snapshot(&reader.read_all().await?)?).await; + Ok(Some((data_usage_info, revision))) +} + +fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool { + data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket) +} + +fn advance_data_usage_last_update(data_usage_info: &mut DataUsageInfo) { + let now = SystemTime::now(); + data_usage_info.last_update = Some(match data_usage_info.last_update { + Some(last_update) if last_update >= now => last_update.checked_add(Duration::from_nanos(1)).unwrap_or(now), + _ => now, + }); +} + +#[cfg(test)] +async fn remove_bucket_usage_from_backend_with_store(store: &S, bucket: &str) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, None).await +} + +fn ensure_bucket_namespace_guard( + guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, + operation: &'static str, +) -> Result<(), Error> { + if guard.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost) { + return Err(Error::other(format!("bucket namespace lock was lost before {operation}: {bucket}"))); + } + Ok(()) +} + +async fn remove_bucket_usage_from_backend_with_store_and_guard( + store: &S, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?; + let primary_seed = load_data_usage_seed_for_missing_primary(store).await?; + remove_bucket_usage_from_object_with_retries( + store, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + bucket, + DATA_USAGE_REMOVE_CAS_RETRIES, + Some(&primary_seed), + guard, + ) + .await?; + + ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?; + let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await? + .map_or(primary_seed, |(data_usage_info, _)| data_usage_info); + remove_bucket_usage_from_object_with_retries( + store, + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + bucket, + DATA_USAGE_REMOVE_CAS_RETRIES, + Some(&backup_seed), + guard, + ) + .await?; + + for object in [ + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), + LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + ] { + remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?; + } + Ok(()) +} + +async fn load_data_usage_seed_for_missing_primary(store: &S) -> Result +where + S: EcstoreObjectIO + ?Sized, +{ + for object in [ + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), + LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + ] { + if let Some((data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? { + return Ok(data_usage_info); + } + } + Ok(DataUsageInfo::default()) +} + +async fn remove_bucket_usage_from_object_with_retries( + store: &S, + object: &str, + bucket: &str, + cas_retries: usize, + missing_seed: Option<&DataUsageInfo>, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + for attempt in 0..=cas_retries { + ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?; + let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? { + Some((data_usage_info, revision)) => (data_usage_info, Some(revision)), + None => match missing_seed { + Some(data_usage_info) => (data_usage_info.clone(), None), + None => return Ok(()), + }, + }; + remove_bucket_usage_from_info(&mut data_usage_info, bucket); + advance_data_usage_last_update(&mut data_usage_info); + + let data = serde_json::to_vec(&data_usage_info) + .map_err(|err| Error::other(format!("Failed to serialize data usage info: {err}")))?; + let mut put_data = PutObjReader::from_vec(data); + let http_preconditions = match revision.as_ref() { + Some(revision) => HTTPPreconditions { + if_match: Some(revision.clone()), + ..Default::default() + }, + None => HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + }; + ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?; + let save_result = store + .put_object( + RUSTFS_META_BUCKET, + object, + &mut put_data, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(http_preconditions), + ..Default::default() + }, + ) + .await; + match save_result { + Ok(_) => return Ok(()), + Err(err) => { + if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? { + let revision_advanced = revision.as_ref().is_none_or(|revision| revision != &observed_revision); + if revision_advanced && !data_usage_contains_bucket(&observed, bucket) { + return Ok(()); + } + if attempt < cas_retries + && (err == Error::PreconditionFailed || revision.as_ref() != Some(&observed_revision)) + { + ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot retry")?; + continue; + } + } else if attempt < cas_retries && err == Error::PreconditionFailed { + continue; + } + return Err(err); + } + } } - Ok(()) + Err(Error::other("data usage bucket removal CAS retries exhausted")) } fn record_usage_snapshot_failure(operation: &'static str, object: &str, e: &Error) { @@ -390,7 +597,7 @@ fn parse_usage_snapshot(buf: &[u8]) -> Result { }) } -/// Decide which usage snapshot to serve: the primary `.usage.json` or its +/// Decide which usage snapshot to serve: a primary usage object or its /// `.bkp` backup. The backup future is polled only when the primary is unusable, /// so the healthy path performs a single read. /// @@ -398,21 +605,23 @@ fn parse_usage_snapshot(buf: &[u8]) -> Result { /// the backup are `ConfigNotFound` — a genuine "no snapshot yet". A real /// primary failure with a missing backup propagates as an error instead of /// being rendered as confirmed all-zero stats. -async fn resolve_loaded_snapshot_with_source( +async fn resolve_loaded_snapshot_pair_with_source( primary: Result, Error>, backup: impl Future, Error>>, + primary_path: &str, + backup_path: &str, ) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> { let primary_failure = match primary { Ok(buf) => match parse_usage_snapshot(&buf) { Ok(info) => return Ok((info, UsageSnapshotSource::Primary)), Err(e) => { - record_usage_snapshot_decode_failure("parse_primary", DATA_USAGE_OBJ_NAME_PATH.as_str(), &e); + record_usage_snapshot_decode_failure("parse_primary", primary_path, &e); e } }, Err(e) => { if e != Error::ConfigNotFound { - record_usage_snapshot_failure("read_primary", DATA_USAGE_OBJ_NAME_PATH.as_str(), &e); + record_usage_snapshot_failure("read_primary", primary_path, &e); } e } @@ -421,7 +630,7 @@ async fn resolve_loaded_snapshot_with_source( Ok(buf) => match parse_usage_snapshot(&buf) { Ok(info) => Ok((info, UsageSnapshotSource::Backup)), Err(e) => { - record_usage_snapshot_decode_failure("parse_backup", &DATA_USAGE_OBJ_BACKUP_PATH, &e); + record_usage_snapshot_decode_failure("parse_backup", backup_path, &e); Err(e) } }, @@ -430,7 +639,7 @@ async fn resolve_loaded_snapshot_with_source( } Err(Error::ConfigNotFound) => Err(primary_failure), Err(e) => { - record_usage_snapshot_failure("read_backup", &DATA_USAGE_OBJ_BACKUP_PATH, &e); + record_usage_snapshot_failure("read_backup", backup_path, &e); Err(e) } } @@ -440,21 +649,60 @@ async fn resolve_loaded_snapshot( primary: Result, Error>, backup: impl Future, Error>>, ) -> Result { - Ok(resolve_loaded_snapshot_with_source(primary, backup).await?.0) + Ok(resolve_loaded_snapshot_pair_with_source( + primary, + backup, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + ) + .await? + .0) } -async fn load_data_usage_snapshot(store: Arc) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> { - let primary = read_config_preserve_empty(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await; - resolve_loaded_snapshot_with_source( +#[cfg(test)] +async fn resolve_loaded_snapshot_with_source( + primary: Result, Error>, + backup: impl Future, Error>>, +) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> { + resolve_loaded_snapshot_pair_with_source( primary, - async move { read_config_preserve_empty(store, &DATA_USAGE_OBJ_BACKUP_PATH).await }, + backup, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), ) .await } -async fn load_primary_data_usage_from_backend(store: Arc) -> Result { - let buf = read_config_preserve_empty(store, &DATA_USAGE_OBJ_NAME_PATH).await?; - Ok(normalize_loaded_data_usage(parse_usage_snapshot(&buf)?).await) +async fn load_data_usage_snapshot_from_store( + store: Arc, +) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> { + let primary = read_config_preserve_empty(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await; + let authoritative = resolve_loaded_snapshot_pair_with_source( + primary, + { + let store = store.clone(); + async move { read_config_preserve_empty(store, &DATA_USAGE_OBJ_BACKUP_PATH).await } + }, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + ) + .await?; + if authoritative.1 != UsageSnapshotSource::Missing { + return Ok(authoritative); + } + + let legacy_primary = read_config_preserve_empty(store.clone(), &LEGACY_DATA_USAGE_OBJ_NAME_PATH).await; + resolve_loaded_snapshot_pair_with_source( + legacy_primary, + async move { read_config_preserve_empty(store, &LEGACY_DATA_USAGE_OBJ_BACKUP_PATH).await }, + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), + LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + ) + .await +} + +async fn load_data_usage_snapshot(store: Arc) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> { + load_data_usage_snapshot_from_store(store).await } /// Load data usage info from backend storage @@ -1103,8 +1351,23 @@ pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) { record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await; } -/// Get bucket usage from in-memory cache +/// Get bucket usage from the authoritative cache for this topology. +async fn get_persisted_bucket_usage(bucket: &str) -> Option { + let store = runtime_sources::object_store_handle()?; + let data_usage_info = load_data_usage_from_backend_cached(store).await.ok()?; + data_usage_info.buckets_usage.get(bucket).map(|usage| usage.size) +} + pub async fn get_bucket_usage_memory(bucket: &str) -> Option { + // A process-local mutation cache is authoritative only when every request + // is handled by this process. In a distributed deployment it contains an + // absolute count derived from one node's requests, so applying it as the + // cluster total can replace a complete snapshot with roughly one node's + // share of the bucket. + if runtime_sources::setup_is_dist_erasure().await { + return get_persisted_bucket_usage(bucket).await; + } + update_usage_cache_if_needed().await; let cache = memory_cache().read().await; @@ -1215,7 +1478,11 @@ pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageIn *cache = next_cache; } -pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) { +async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) { + if !authoritative { + return; + } + let cache = memory_cache().read().await; if cache.is_empty() { return; @@ -1240,6 +1507,11 @@ pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageIn } } +pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) { + let authoritative = !runtime_sources::setup_is_dist_erasure().await; + apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await; +} + /// Sync memory cache with backend data (called by scanner) pub async fn sync_memory_cache_with_backend() -> Result<(), Error> { if let Some(store) = runtime_sources::object_store_handle() { @@ -1562,7 +1834,213 @@ pub async fn init_compression_total_memory_from_backend(store: Arc) { mod tests { use super::*; use rustfs_data_usage::BucketUsageInfo; + use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; use serial_test::serial; + use std::io::Cursor; + use std::sync::Arc; + use tokio::{io::AsyncReadExt, sync::Mutex}; + + #[derive(Debug, Default)] + struct UsageCasState { + object: Option<(Vec, u64)>, + backup_object: Option<(Vec, u64)>, + legacy_object: Option<(Vec, u64)>, + legacy_backup_object: Option<(Vec, u64)>, + interleaving_snapshot: Option>, + interleaving_backup_snapshot: Option>, + interleaving_legacy_snapshot: Option>, + interleaving_legacy_backup_snapshot: Option>, + error_after_commit_put: Option, + advance_time_on_put: Option, + advance_time_after_get: Option<(UsageObjectSlot, Duration)>, + advance_time_before_put: Option<(usize, Duration)>, + advance_time_after_put: Option<(usize, Duration)>, + put_count: usize, + } + + #[derive(Debug, Default)] + struct UsageCasStore { + state: Mutex, + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum UsageObjectSlot { + Primary, + Backup, + LegacyPrimary, + LegacyBackup, + } + + #[async_trait::async_trait] + impl crate::storage_api_contracts::object::ObjectIO for UsageCasStore { + type Error = Error; + type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = crate::object_api::GetObjectReader; + type PutObjectReader = PutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> Result { + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + let slot = match object { + object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary, + object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup, + object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary, + object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup, + _ => return Err(Error::FileNotFound), + }; + let mut state = self.state.lock().await; + let stored = match slot { + UsageObjectSlot::Primary => &state.object, + UsageObjectSlot::Backup => &state.backup_object, + UsageObjectSlot::LegacyPrimary => &state.legacy_object, + UsageObjectSlot::LegacyBackup => &state.legacy_backup_object, + }; + let (data, revision) = stored.clone().ok_or(Error::FileNotFound)?; + let advance = match state.advance_time_after_get { + Some((expected_slot, duration)) if expected_slot == slot => { + state.advance_time_after_get = None; + Some(duration) + } + _ => None, + }; + drop(state); + if let Some(duration) = advance { + tokio::time::advance(duration).await; + } + Ok(crate::object_api::GetObjectReader { + stream: Box::new(Cursor::new(data)), + object_info: ObjectInfo { + etag: Some(format!("usage-{revision}")), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> Result { + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + let slot = match object { + object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary, + object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup, + object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary, + object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup, + _ => return Err(Error::FileNotFound), + }; + let mut buf = Vec::new(); + data.stream.read_to_end(&mut buf).await?; + + let mut state = self.state.lock().await; + state.put_count += 1; + let put_count = state.put_count; + if let Some(duration) = state.advance_time_on_put.take() { + drop(state); + tokio::time::advance(duration).await; + state = self.state.lock().await; + } + if let Some((expected_put, duration)) = state.advance_time_before_put + && expected_put == put_count + { + state.advance_time_before_put = None; + drop(state); + tokio::time::advance(duration).await; + state = self.state.lock().await; + } + if slot == UsageObjectSlot::Primary + && let Some(interleaving) = state.interleaving_snapshot.take() + { + let revision = state.object.as_ref().map_or(1, |(_, revision)| revision + 1); + state.object = Some((interleaving, revision)); + } + if slot == UsageObjectSlot::Backup + && let Some(interleaving) = state.interleaving_backup_snapshot.take() + { + let revision = state.backup_object.as_ref().map_or(1, |(_, revision)| revision + 1); + state.backup_object = Some((interleaving, revision)); + } + if slot == UsageObjectSlot::LegacyPrimary + && let Some(interleaving) = state.interleaving_legacy_snapshot.take() + { + let revision = state.legacy_object.as_ref().map_or(1, |(_, revision)| revision + 1); + state.legacy_object = Some((interleaving, revision)); + } + if slot == UsageObjectSlot::LegacyBackup + && let Some(interleaving) = state.interleaving_legacy_backup_snapshot.take() + { + let revision = state.legacy_backup_object.as_ref().map_or(1, |(_, revision)| revision + 1); + state.legacy_backup_object = Some((interleaving, revision)); + } + let current_revision = match slot { + UsageObjectSlot::Primary => state.object.as_ref(), + UsageObjectSlot::Backup => state.backup_object.as_ref(), + UsageObjectSlot::LegacyPrimary => state.legacy_object.as_ref(), + UsageObjectSlot::LegacyBackup => state.legacy_backup_object.as_ref(), + } + .map(|(_, revision)| *revision); + if let Some(preconditions) = &opts.http_preconditions { + if preconditions + .if_none_match + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + && current_revision.is_some() + { + return Err(Error::PreconditionFailed); + } + if let Some(expected) = preconditions + .if_match + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let actual = current_revision.map(|revision| format!("usage-{revision}")); + if actual.as_deref() != Some(expected.trim_matches('"')) { + return Err(Error::PreconditionFailed); + } + } + } + + let revision = current_revision.unwrap_or_default() + 1; + match slot { + UsageObjectSlot::Primary => state.object = Some((buf, revision)), + UsageObjectSlot::Backup => state.backup_object = Some((buf, revision)), + UsageObjectSlot::LegacyPrimary => state.legacy_object = Some((buf, revision)), + UsageObjectSlot::LegacyBackup => state.legacy_backup_object = Some((buf, revision)), + } + if state.error_after_commit_put == Some(put_count) { + return Err(Error::other("injected post-commit failure")); + } + if let Some((expected_put, duration)) = state.advance_time_after_put + && expected_put == put_count + { + state.advance_time_after_put = None; + drop(state); + tokio::time::advance(duration).await; + } + Ok(ObjectInfo { + etag: Some(format!("usage-{revision}")), + ..Default::default() + }) + } + } async fn clear_usage_memory_cache_for_test() { memory_cache().write().await.clear(); @@ -1833,6 +2311,33 @@ mod tests { assert!(info.buckets_usage.is_empty()); } + #[tokio::test] + async fn authoritative_usage_loader_migrates_only_when_v2_is_absent() { + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + legacy_object: Some((snapshot_bytes("legacy-bucket"), 1)), + ..Default::default() + }), + }); + + let (legacy, _) = load_data_usage_snapshot_from_store(store.clone()) + .await + .expect("a legacy snapshot should seed an upgrade when v2 is absent"); + assert_snapshot_bucket(&legacy, "legacy-bucket"); + + store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 1)); + let (authoritative, _) = load_data_usage_snapshot_from_store(store.clone()) + .await + .expect("a v2 snapshot should be authoritative"); + assert_snapshot_bucket(&authoritative, "v2-bucket"); + + store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 2)); + let err = load_data_usage_snapshot_from_store(store) + .await + .expect_err("corrupt v2 state must not fall back to a legacy writer"); + assert!(err.to_string().contains("deserialize")); + } + #[tokio::test] async fn compute_usage_preserves_same_object_across_1000_entry_page_boundary() { let first_page = UsageVersionPage { @@ -2151,6 +2656,36 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn distributed_usage_does_not_apply_a_process_local_absolute_count() { + clear_usage_memory_cache_for_test().await; + + let persisted = data_usage_info_for_test("bucket-a", 100, 1_000, SystemTime::now() - Duration::from_secs(10)); + replace_bucket_usage_memory_from_info(&persisted).await; + record_bucket_object_delete_memory("bucket-a", 42, true).await; + + let mut distributed_response = persisted.clone(); + apply_bucket_usage_memory_overlay_if_authoritative(&mut distributed_response, false).await; + assert_eq!( + distributed_response + .buckets_usage + .get("bucket-a") + .map(|usage| (usage.objects_count, usage.size)), + Some((100, 1_000)) + ); + + let mut single_process_response = persisted; + apply_bucket_usage_memory_overlay_if_authoritative(&mut single_process_response, true).await; + assert_eq!( + single_process_response + .buckets_usage + .get("bucket-a") + .map(|usage| (usage.objects_count, usage.size)), + Some((99, 958)) + ); + } + #[test] fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() { let last_update = SystemTime::now(); @@ -2184,12 +2719,16 @@ mod tests { ); } - #[test] - fn merge_bucket_usage_removal_preserves_current_snapshot() { + #[tokio::test] + async fn remove_bucket_usage_retries_against_newer_scanner_snapshot() { let now = SystemTime::now(); - let candidate = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); - let mut existing = data_usage_info_for_test("bucket-a", 4, 168, now); - existing.buckets_usage.insert( + let mut initial = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); + initial.scanner_epoch = Some(8); + initial.scanner_cycle = Some(12); + let mut scanner_winner = data_usage_info_for_test("bucket-a", 4, 168, now); + scanner_winner.scanner_epoch = Some(9); + scanner_winner.scanner_cycle = Some(13); + scanner_winner.buckets_usage.insert( "bucket-c".to_string(), BucketUsageInfo { objects_count: 5, @@ -2198,20 +2737,34 @@ mod tests { ..Default::default() }, ); - existing.bucket_sizes.insert("bucket-c".to_string(), 210); - existing.buckets_count = 2; - existing.calculate_totals(); + scanner_winner.bucket_sizes.insert("bucket-c".to_string(), 210); + scanner_winner.buckets_count = 2; + scanner_winner.calculate_totals(); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&initial).expect("initial usage snapshot should encode"), 1)), + interleaving_snapshot: Some(serde_json::to_vec(&scanner_winner).expect("scanner winner snapshot should encode")), + ..Default::default() + }), + }); - let merged = merge_bucket_usage_removal(candidate, Some(existing), "bucket-a") - .expect("bucket-a should be removed from the current snapshot"); + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should reconcile the scanner CAS winner"); - assert_eq!(merged.last_update, Some(now)); - assert_eq!(merged.buckets_count, 1); - assert_eq!(merged.objects_total_count, 5); - assert_eq!(merged.objects_total_size, 210); - assert!(!merged.buckets_usage.contains_key("bucket-a")); + let state = store.state.lock().await; + let saved = serde_json::from_slice::(&state.object.as_ref().expect("usage snapshot should remain").0) + .expect("saved usage snapshot should decode"); + assert_eq!(state.put_count, 3); + assert!(saved.last_update.is_some_and(|last_update| last_update > now)); + assert_eq!(saved.scanner_epoch, Some(9)); + assert_eq!(saved.scanner_cycle, Some(13)); + assert_eq!(saved.buckets_count, 1); + assert_eq!(saved.objects_total_count, 5); + assert_eq!(saved.objects_total_size, 210); + assert!(!saved.buckets_usage.contains_key("bucket-a")); assert_eq!( - merged + saved .buckets_usage .get("bucket-c") .map(|usage| (usage.objects_count, usage.size)), @@ -2219,6 +2772,693 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_stops_retry_after_namespace_lock_loss() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-cleanup-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket-a", ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let now = SystemTime::now(); + let initial = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); + let scanner_winner = data_usage_info_for_test("bucket-a", 4, 168, now); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&initial).expect("initial usage snapshot should encode"), 1)), + interleaving_snapshot: Some(serde_json::to_vec(&scanner_winner).expect("scanner winner should encode")), + advance_time_on_put: Some(ttl + Duration::from_millis(1)), + ..Default::default() + }), + }); + + let err = remove_bucket_usage_from_backend_with_store_and_guard(store.as_ref(), "bucket-a", Some(&guard)) + .await + .expect_err("namespace lock loss must stop the cleanup CAS retry"); + assert!(err.to_string().contains("namespace lock was lost")); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 1, "the stale cleanup must not issue a second CAS write"); + let saved = serde_json::from_slice::(&state.object.as_ref().expect("scanner winner should remain").0) + .expect("scanner winner should decode"); + assert_eq!(saved.buckets_usage.get("bucket-a").map(|usage| usage.objects_count), Some(4)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_stops_before_put_after_namespace_lock_loss() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-pre-put-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket-a", ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + let encoded = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded, 1)), + advance_time_after_get: Some((UsageObjectSlot::Primary, ttl + Duration::from_millis(1))), + ..Default::default() + }), + }); + + let err = remove_bucket_usage_from_backend_with_store_and_guard(store.as_ref(), "bucket-a", Some(&guard)) + .await + .expect_err("namespace lock loss after the snapshot read must fence the write"); + assert!(err.to_string().contains("namespace lock was lost")); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 0, "a cleanup that lost its lease must not start a snapshot write"); + let saved = serde_json::from_slice::(&state.object.as_ref().expect("usage snapshot should remain").0) + .expect("usage snapshot should decode"); + assert!(saved.buckets_usage.contains_key("bucket-a")); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_stops_before_backup_after_namespace_lock_loss() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-backup-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket-a", ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded, 1)), + advance_time_after_put: Some((1, ttl + Duration::from_millis(1))), + ..Default::default() + }), + }); + + let err = remove_bucket_usage_from_backend_with_store_and_guard(store.as_ref(), "bucket-a", Some(&guard)) + .await + .expect_err("namespace lock loss must stop cleanup before the backup write"); + assert!(err.to_string().contains("namespace lock was lost")); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 1, "the cleanup must not write the backup after lock loss"); + let backup = serde_json::from_slice::( + &state.backup_object.as_ref().expect("new-generation backup should remain").0, + ) + .expect("backup should decode"); + assert_eq!(backup.buckets_usage.get("bucket-a").map(|usage| usage.objects_count), Some(2)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_stops_backup_retry_after_namespace_lock_loss() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-backup-retry-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket-a", ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let now = SystemTime::now(); + let initial = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); + let scanner_winner = data_usage_info_for_test("bucket-a", 4, 168, now); + let encoded = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded, 1)), + interleaving_backup_snapshot: Some(serde_json::to_vec(&scanner_winner).expect("scanner winner should encode")), + advance_time_before_put: Some((2, ttl + Duration::from_millis(1))), + ..Default::default() + }), + }); + + let err = remove_bucket_usage_from_backend_with_store_and_guard(store.as_ref(), "bucket-a", Some(&guard)) + .await + .expect_err("namespace lock loss must stop the backup CAS retry"); + assert!(err.to_string().contains("namespace lock was lost")); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 2, "the stale cleanup must not issue a second backup CAS write"); + let backup = serde_json::from_slice::( + &state.backup_object.as_ref().expect("scanner winner backup should remain").0, + ) + .expect("backup should decode"); + assert_eq!(backup.buckets_usage.get("bucket-a").map(|usage| usage.objects_count), Some(4)); + } + + async fn assert_legacy_cleanup_stops_retry_after_lock_loss(backup: bool) { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new( + format!("usage-legacy-{}-retry-lock-loss-test", if backup { "backup" } else { "primary" }), + Arc::new(LocalClient::new()), + ); + let request = LockRequest::new(ObjectKey::new("bucket-a", ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let now = SystemTime::now(); + let initial = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); + let scanner_winner = data_usage_info_for_test("bucket-a", 4, 168, now); + let encoded = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); + let winner = serde_json::to_vec(&scanner_winner).expect("scanner winner should encode"); + let conflict_put = if backup { 4 } else { 3 }; + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded.clone(), 1)), + legacy_object: Some((encoded.clone(), 1)), + legacy_backup_object: Some((encoded, 1)), + interleaving_legacy_snapshot: (!backup).then_some(winner.clone()), + interleaving_legacy_backup_snapshot: backup.then_some(winner), + advance_time_before_put: Some((conflict_put, ttl + Duration::from_millis(1))), + ..Default::default() + }), + }); + + let err = remove_bucket_usage_from_backend_with_store_and_guard(store.as_ref(), "bucket-a", Some(&guard)) + .await + .expect_err("namespace lock loss must stop the legacy snapshot CAS retry"); + assert!(err.to_string().contains("namespace lock was lost")); + + let state = store.state.lock().await; + assert_eq!( + state.put_count, conflict_put, + "the stale cleanup must not retry the conflicted legacy snapshot" + ); + let saved = if backup { + &state + .legacy_backup_object + .as_ref() + .expect("scanner winner legacy backup should remain") + .0 + } else { + &state + .legacy_object + .as_ref() + .expect("scanner winner legacy primary should remain") + .0 + }; + let saved = serde_json::from_slice::(saved).expect("scanner winner should decode"); + assert_eq!(saved.buckets_usage.get("bucket-a").map(|usage| usage.objects_count), Some(4)); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_stops_legacy_retries_after_namespace_lock_loss() { + assert_legacy_cleanup_stops_retry_after_lock_loss(false).await; + assert_legacy_cleanup_stops_retry_after_lock_loss(true).await; + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn prepare_bucket_usage_preserves_successor_memory_after_lock_loss() { + const BUCKET: &str = "successor-bucket"; + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-memory-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new(BUCKET, ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = Arc::new( + lock.acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"), + ); + + let mut cache = memory_cache().write().await; + cache.insert( + BUCKET.to_string(), + cached_bucket_usage_now(BucketUsageInfo { + objects_count: 7, + versions_count: 7, + size: 294, + ..Default::default() + }), + ); + let guard_for_prepare = guard.clone(); + let prepare = + tokio::spawn( + async move { prepare_bucket_usage_for_namespace_change(BUCKET, Some(guard_for_prepare.as_ref())).await }, + ); + tokio::task::yield_now().await; + + tokio::time::advance(ttl + Duration::from_millis(1)).await; + drop(cache); + let err = prepare + .await + .expect("prepare task should join") + .expect_err("lock loss while waiting for the cache must stop cleanup"); + assert!(err.to_string().contains("namespace lock was lost")); + + let mut cache = memory_cache().write().await; + assert_eq!( + cache.get(BUCKET).map(|entry| entry.usage.objects_count), + Some(7), + "fresh successor memory usage must not be erased after namespace lock loss" + ); + cache.remove(BUCKET); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn prepare_bucket_usage_preserves_successor_snapshot_cache_after_lock_loss() { + const BUCKET: &str = "successor-snapshot-bucket"; + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-snapshot-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new(BUCKET, ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = Arc::new( + lock.acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"), + ); + let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); + let mut snapshot_cache = data_usage_snapshot_cache().write().await; + *snapshot_cache = Some(CachedDataUsageSnapshot { + info: Some(successor), + loaded_at: tokio::time::Instant::now(), + }); + memory_cache() + .write() + .await + .insert(BUCKET.to_string(), cached_bucket_usage_now(BucketUsageInfo::default())); + + let guard_for_prepare = guard.clone(); + let prepare = + tokio::spawn( + async move { prepare_bucket_usage_for_namespace_change(BUCKET, Some(guard_for_prepare.as_ref())).await }, + ); + loop { + if !memory_cache().read().await.contains_key(BUCKET) { + break; + } + tokio::task::yield_now().await; + } + + tokio::time::advance(ttl + Duration::from_millis(1)).await; + drop(snapshot_cache); + let err = prepare + .await + .expect("prepare task should join") + .expect_err("lock loss while waiting for the snapshot cache must stop cleanup"); + assert!(err.to_string().contains("namespace lock was lost")); + + let snapshot_cache = data_usage_snapshot_cache().read().await; + assert_eq!( + snapshot_cache + .as_ref() + .and_then(|cached| cached.info.as_ref()) + .and_then(|info| info.buckets_usage.get(BUCKET)) + .map(|usage| usage.objects_count), + Some(7), + "fresh successor snapshot cache must not be erased after namespace lock loss" + ); + drop(snapshot_cache); + *data_usage_snapshot_cache().write().await = None; + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remove_bucket_usage_preserves_successor_snapshot_cache_after_lock_loss() { + const BUCKET: &str = "successor-final-snapshot-bucket"; + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("usage-final-snapshot-lock-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new(BUCKET, ""), LockType::Exclusive, "cleanup-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = Arc::new( + lock.acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"), + ); + let store = Arc::new(UsageCasStore::default()); + let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); + let mut snapshot_cache = data_usage_snapshot_cache().write().await; + *snapshot_cache = Some(CachedDataUsageSnapshot { + info: Some(successor), + loaded_at: tokio::time::Instant::now(), + }); + + let store_for_cleanup = store.clone(); + let guard_for_cleanup = guard.clone(); + let cleanup = tokio::spawn(async move { + remove_bucket_usage_from_backend_with_guard(store_for_cleanup.as_ref(), BUCKET, Some(guard_for_cleanup.as_ref())) + .await + }); + loop { + if store.state.lock().await.put_count >= 2 { + break; + } + tokio::task::yield_now().await; + } + + tokio::time::advance(ttl + Duration::from_millis(1)).await; + drop(snapshot_cache); + let err = cleanup + .await + .expect("cleanup task should join") + .expect_err("lock loss while waiting for final cache invalidation must stop cleanup"); + assert!(err.to_string().contains("namespace lock was lost")); + + let snapshot_cache = data_usage_snapshot_cache().read().await; + assert_eq!( + snapshot_cache + .as_ref() + .and_then(|cached| cached.info.as_ref()) + .and_then(|info| info.buckets_usage.get(BUCKET)) + .map(|usage| usage.objects_count), + Some(7), + "fresh successor snapshot cache must not be erased by final invalidation after lock loss" + ); + drop(snapshot_cache); + *data_usage_snapshot_cache().write().await = None; + } + + #[tokio::test] + #[serial] + async fn remove_bucket_usage_invalidates_snapshot_cache_after_backend_error() { + const BUCKET: &str = "backend-error-cache-bucket"; + let store = UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((b"{".to_vec(), 1)), + ..Default::default() + }), + }; + let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); + *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot { + info: Some(stale), + loaded_at: tokio::time::Instant::now(), + }); + + remove_bucket_usage_from_backend_with_guard(&store, BUCKET, None) + .await + .expect_err("the malformed backend snapshot should be reported"); + + assert!( + data_usage_snapshot_cache().read().await.is_none(), + "a failed cleanup must still invalidate cached predecessor usage" + ); + } + + #[tokio::test] + async fn remove_bucket_usage_writes_fences_when_bucket_is_already_absent() { + let last_update = SystemTime::now() - Duration::from_secs(10); + let snapshot = data_usage_info_for_test("bucket-b", 3, 126, last_update); + let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded, 1)), + ..Default::default() + }), + }); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should fence both snapshots even when the bucket is absent"); + + let state = store.state.lock().await; + assert_eq!(state.object.as_ref().map(|(_, revision)| *revision), Some(2)); + assert_eq!(state.backup_object.as_ref().map(|(_, revision)| *revision), Some(2)); + for (data, _) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { + let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); + assert!(saved.last_update.is_some_and(|updated| updated > last_update)); + assert_eq!(saved.buckets_usage.get("bucket-b").map(|usage| usage.objects_count), Some(3)); + } + } + + #[tokio::test] + async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() { + let store = Arc::new(UsageCasStore::default()); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should create both usage fences"); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 2); + for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { + let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); + assert_eq!(*revision, 1); + assert!(saved.last_update.is_some()); + assert!(!data_usage_contains_bucket(&saved, "bucket-a")); + } + } + + #[tokio::test] + async fn remove_bucket_usage_migrates_legacy_snapshot_without_hiding_other_buckets() { + let mut legacy = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + legacy.buckets_usage.insert( + "bucket-b".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 3, + size: 126, + ..Default::default() + }, + ); + legacy.bucket_sizes.insert("bucket-b".to_string(), 126); + legacy.buckets_count = 2; + legacy.calculate_totals(); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + legacy_object: Some((serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"), 1)), + ..Default::default() + }), + }); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should migrate the legacy usage baseline"); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 3); + for (data, _) in [ + state.object.as_ref(), + state.backup_object.as_ref(), + state.legacy_object.as_ref(), + ] + .into_iter() + .flatten() + { + let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); + assert!(!data_usage_contains_bucket(&saved, "bucket-a")); + assert_eq!( + saved + .buckets_usage + .get("bucket-b") + .map(|usage| (usage.objects_count, usage.size)), + Some((3, 126)) + ); + } + } + + #[tokio::test] + async fn remove_bucket_usage_seeds_missing_backup_from_updated_primary() { + let mut primary = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + primary.buckets_usage.insert( + "bucket-b".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 3, + size: 126, + ..Default::default() + }, + ); + primary.bucket_sizes.insert("bucket-b".to_string(), 126); + primary.buckets_count = 2; + primary.calculate_totals(); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&primary).expect("primary usage snapshot should encode"), 1)), + ..Default::default() + }), + }); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should seed a missing backup from the updated primary"); + + let state = store.state.lock().await; + assert_eq!(state.put_count, 2); + for (data, _) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { + let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); + assert!(!data_usage_contains_bucket(&saved, "bucket-a")); + assert_eq!( + saved + .buckets_usage + .get("bucket-b") + .map(|usage| (usage.objects_count, usage.size)), + Some((3, 126)) + ); + } + } + + #[tokio::test] + async fn remove_bucket_usage_fences_reject_stale_primary_and_backup_writes() { + let stale = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now() - Duration::from_secs(10)); + let stale_data = serde_json::to_vec(&stale).expect("stale usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((stale_data.clone(), 1)), + ..Default::default() + }), + }); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should write both usage fences"); + + let mut stale_primary = PutObjReader::from_vec(stale_data.clone()); + let primary_err = store + .put_object( + RUSTFS_META_BUCKET, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + &mut stale_primary, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some("usage-1".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("the pre-delete primary baseline must be fenced"); + assert_eq!(primary_err, Error::PreconditionFailed); + + let mut stale_backup = PutObjReader::from_vec(stale_data); + let backup_err = store + .put_object( + RUSTFS_META_BUCKET, + DATA_USAGE_OBJ_BACKUP_PATH.as_str(), + &mut stale_backup, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("the missing pre-delete backup baseline must be fenced"); + assert_eq!(backup_err, Error::PreconditionFailed); + } + + #[tokio::test] + async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() { + let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&initial).expect("initial usage snapshot should encode"), 1)), + error_after_commit_put: Some(1), + ..Default::default() + }), + }); + + remove_bucket_usage_from_object_with_retries( + store.as_ref(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + "bucket-a", + 0, + None, + None, + ) + .await + .expect("final-attempt read-back should confirm the ambiguous committed removal"); + + let state = store.state.lock().await; + let saved = serde_json::from_slice::(&state.object.as_ref().expect("usage snapshot should remain").0) + .expect("saved usage snapshot should decode"); + assert_eq!(state.put_count, 1); + assert!(!saved.buckets_usage.contains_key("bucket-a")); + assert!(!saved.bucket_sizes.contains_key("bucket-a")); + } + + #[tokio::test] + async fn remove_bucket_usage_updates_primary_and_backup_snapshots() { + let now = SystemTime::now(); + let primary = data_usage_info_for_test("bucket-a", 2, 84, now); + let mut backup = primary.clone(); + backup.buckets_usage.insert( + "bucket-b".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 3, + size: 126, + ..Default::default() + }, + ); + backup.bucket_sizes.insert("bucket-b".to_string(), 126); + backup.calculate_totals(); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&primary).expect("primary usage snapshot should encode"), 1)), + backup_object: Some((serde_json::to_vec(&backup).expect("backup usage snapshot should encode"), 1)), + ..Default::default() + }), + }); + + remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") + .await + .expect("bucket removal should update both usage snapshots"); + + let state = store.state.lock().await; + let primary = serde_json::from_slice::(&state.object.as_ref().expect("primary snapshot should remain").0) + .expect("primary snapshot should decode"); + let backup = + serde_json::from_slice::(&state.backup_object.as_ref().expect("backup snapshot should remain").0) + .expect("backup snapshot should decode"); + assert_eq!(state.put_count, 2); + assert!(!data_usage_contains_bucket(&primary, "bucket-a")); + assert!(!data_usage_contains_bucket(&backup, "bucket-a")); + assert_eq!( + backup + .buckets_usage + .get("bucket-b") + .map(|usage| (usage.objects_count, usage.size)), + Some((3, 126)) + ); + } + #[tokio::test] #[serial] async fn clear_bucket_usage_memory_prevents_deleted_bucket_overlay() { @@ -2227,7 +3467,9 @@ mod tests { let persisted = data_usage_info_for_test("bucket-a", 1, 42, SystemTime::now() - Duration::from_secs(10)); replace_bucket_usage_memory_from_info(&persisted).await; record_bucket_object_delete_memory("bucket-a", 42, true).await; - clear_bucket_usage_memory("bucket-a").await; + clear_bucket_usage_memory("bucket-a", None) + .await + .expect("bucket usage memory cleanup should succeed"); let mut response = DataUsageInfo { last_update: Some(SystemTime::now()), diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index bb6c07b5f..9d15dd573 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -231,6 +231,13 @@ impl DiskError { } } + pub fn is_internode_http_status(&self, status: u16) -> bool { + matches!( + self.internode_http_error_kind(), + Some(InternodeHttpErrorKind::HttpStatus(actual)) if actual.as_u16() == status + ) + } + // /// If all errors are of the same fatal disk error type, returns the corresponding error. // /// Otherwise, returns Ok. // pub fn check_disk_fatal_errs(errs: &[Option]) -> Result<()> { @@ -981,6 +988,17 @@ mod tests { assert!(!disk_error.contains_io_error_kind(std::io::ErrorKind::TimedOut)); } + #[test] + fn test_internode_http_status_classification() { + let too_many_requests = DiskError::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::TOO_MANY_REQUESTS), + )); + + assert!(too_many_requests.is_internode_http_status(429)); + assert!(!too_many_requests.is_internode_http_status(500)); + assert!(!DiskError::FileNotFound.is_internode_http_status(429)); + } + #[test] fn test_metacache_output_stream_closed_classification_survives_clone() { let disk_error = DiskError::metacache_output_stream_closed(); diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 2f4dec3d4..d683e860f 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -52,7 +52,7 @@ use local::LocalDisk; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_madmin::info_commands::DiskMetrics; use serde::{Deserialize, Serialize}; -use std::{fmt::Debug, path::PathBuf, sync::Arc}; +use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration}; use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; @@ -453,6 +453,20 @@ impl DiskAPI for Disk { } impl Disk { + pub async fn ns_scanner_server_epoch(&self) -> Result> { + match self { + Disk::Local(_) => Ok(None), + Disk::Remote(remote_disk) => remote_disk.ns_scanner_server_epoch().await, + } + } + + pub async fn open_ns_scanner_stream(&self, request: NsScannerOpenRequest) -> Result { + match self { + Disk::Remote(remote_disk) => remote_disk.open_ns_scanner_stream(request).await, + Disk::Local(_) => Err(Error::other("namespace scanner stream requires a remote disk")), + } + } + pub fn runtime_state(&self) -> RuntimeDriveHealthState { match self { Disk::Local(local_disk) => local_disk.runtime_state(), @@ -498,6 +512,18 @@ impl Disk { } } +#[derive(Debug)] +pub struct NsScannerOpenRequest { + pub request_id: Uuid, + pub server_epoch: Uuid, + pub session_id: Uuid, + pub session_sequence: u64, + pub next_cycle: u64, + pub leader_epoch: u64, + pub body: Vec, + pub stall_timeout: Option, +} + impl Disk { /// Reset drive health so `connect_load_init_formats` retries are not blocked by a prior /// transient mark-faulty (same disk handles are reused across retries). diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 367e88a72..30c60fa1a 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -471,6 +471,14 @@ impl PutObjReader { } } + pub fn from_prehashed_bytes(data: Bytes, sha256hex: Option) -> std::io::Result { + let content_length = + i64::try_from(data.len()).map_err(|_| std::io::Error::other("prehashed object payload exceeds i64 length"))?; + Ok(PutObjReader { + stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false)?, + }) + } + pub fn size(&self) -> i64 { self.stream.size() } diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index e5a5210d5..451c67f47 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -154,6 +154,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo last_cycle_replication_checks: metrics.last_cycle_replication_checks, last_cycle_usage_saves: metrics.last_cycle_usage_saves, failed_cycles: metrics.failed_cycles, + superseded_cycles: metrics.superseded_cycles, partial_cycles_unknown: metrics.partial_cycles_unknown, partial_cycles_runtime: metrics.partial_cycles_runtime, partial_cycles_objects: metrics.partial_cycles_objects, @@ -796,6 +797,7 @@ mod test { last_cycle_replication_checks: 27, last_cycle_usage_saves: 28, failed_cycles: 29, + superseded_cycles: 30, partial_cycles_unknown: 30, partial_cycles_runtime: 31, partial_cycles_objects: 32, @@ -927,6 +929,7 @@ mod test { assert_eq!(scanner.last_cycle_replication_checks, 27); assert_eq!(scanner.last_cycle_usage_saves, 28); assert_eq!(scanner.failed_cycles, 29); + assert_eq!(scanner.superseded_cycles, 30); assert_eq!(scanner.partial_cycles_unknown, 30); assert_eq!(scanner.partial_cycles_runtime, 31); assert_eq!(scanner.partial_cycles_objects, 32); diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 107f45ff2..890e71a6f 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -29,7 +29,7 @@ use rustfs_madmin::metrics::RealtimeMetrics; use rustfs_madmin::net::NetInfo; use rustfs_madmin::{ItemState, ServerProperties, StorageInfo}; use rustfs_utils::XHost; -use std::collections::hash_map::DefaultHasher; +use std::collections::{HashMap, hash_map::DefaultHasher}; use std::future::Future; use std::hash::{Hash, Hasher}; use std::sync::{Arc, Mutex, OnceLock}; @@ -1041,6 +1041,40 @@ impl NotificationSys { Ok(generations) } + pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result { + let mut by_host = HashMap::with_capacity(acknowledgements.len()); + for (host, instance_id, generation) in acknowledgements { + if by_host.insert(host.clone(), (instance_id, generation)).is_some() { + return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}"))); + } + } + + let clients = self + .peer_clients + .iter() + .flatten() + .map(|client| (client.grid_host.clone(), client.clone())) + .collect::>(); + let mut failures = Vec::new(); + let mut futures = Vec::with_capacity(by_host.len()); + for (host, (instance_id, generation)) in by_host { + let Some(client) = clients.get(&host).cloned() else { + failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable")); + continue; + }; + futures.push(async move { + let result = scanner_activity_with_timeout( + SCANNER_ACTIVITY_PROBE_TIMEOUT, + &host, + client.acknowledge_scanner_dirty_usage(instance_id, generation), + ) + .await; + (host, result) + }); + } + aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures) + } + pub async fn reload_site_replication_config(&self) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter() { @@ -1591,6 +1625,23 @@ fn aggregate_notification_failures(operation: &str, failures: Vec) -> Re ))) } +fn aggregate_scanner_dirty_usage_acknowledgement_results( + results: Vec<(String, Result)>, + mut failures: Vec, +) -> Result { + let mut dirty_usage_pending = false; + for (host, result) in results { + match result { + Ok(activity) => { + dirty_usage_pending |= activity.dirty_usage_pending != Some(false); + } + Err(err) => failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: {err}")), + } + } + aggregate_notification_failures("acknowledge_scanner_dirty_usage", failures)?; + Ok(dirty_usage_pending) +} + #[cfg(test)] mod tests { use super::*; @@ -1842,6 +1893,78 @@ mod tests { assert!(err.to_string().contains("peer-1")); } + #[tokio::test] + async fn scanner_dirty_usage_acknowledgement_rejects_missing_and_duplicate_targets() { + let sys = NotificationSys { + peer_clients: Vec::new(), + all_peer_clients: Vec::new(), + peer_admin_caches: Vec::new(), + peer_topology_hosts: Vec::new(), + }; + let missing = sys + .acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)]) + .await + .expect_err("a missing acknowledgement target must remain pending"); + assert!(missing.to_string().contains("peer is not reachable")); + + let duplicate = sys + .acknowledge_scanner_dirty_usage(vec![ + ("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7), + ("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7), + ]) + .await + .expect_err("duplicate acknowledgement targets must be rejected"); + assert!( + duplicate + .to_string() + .contains("duplicate scanner dirty usage acknowledgement target") + ); + } + + #[test] + fn scanner_dirty_usage_acknowledgement_preserves_newer_pending_work() { + let activity = |dirty_usage_pending| ScannerPeerActivity { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 1, + maintenance_generation: 1, + protocol_version: crate::storage_api_contracts::internode::SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: Some([0; 32]), + data_movement_active: Some(false), + dirty_usage_generation: Some(2), + dirty_usage_pending, + }; + + let pending = aggregate_scanner_dirty_usage_acknowledgement_results( + vec![ + ("peer-1".to_string(), Ok(activity(Some(false)))), + ("peer-2".to_string(), Ok(activity(Some(true)))), + ], + Vec::new(), + ) + .expect("successful acknowledgements should return their pending state"); + assert!(pending, "new dirty usage reported by an acknowledged peer must remain pending"); + + let cleared = aggregate_scanner_dirty_usage_acknowledgement_results( + vec![("peer-1".to_string(), Ok(activity(Some(false))))], + Vec::new(), + ) + .expect("a cleared acknowledgement should succeed"); + assert!(!cleared, "an explicitly cleared peer must not remain pending"); + + let unknown = + aggregate_scanner_dirty_usage_acknowledgement_results(vec![("peer-1".to_string(), Ok(activity(None)))], Vec::new()) + .expect("an acknowledgement without a pending field should remain retryable"); + assert!(unknown, "a peer that cannot prove its dirty state is clear must remain pending"); + + let err = aggregate_scanner_dirty_usage_acknowledgement_results( + vec![("peer-1".to_string(), Err(Error::other("injected acknowledgement failure")))], + Vec::new(), + ) + .expect_err("a reachable peer acknowledgement failure must be reported"); + assert!(err.to_string().contains("peer-1")); + assert!(err.to_string().contains("injected acknowledgement failure")); + } + #[tokio::test] async fn load_bucket_metadata_reports_unreachable_peers() { let sys = NotificationSys { diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index abf567f49..9d07d8310 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3862,6 +3862,24 @@ impl SetDisks { Ok(removed) } + fn write_precondition_lookup_error( + error: StorageError, + http_preconditions: &HTTPPreconditions, + bucket: &str, + object: &str, + ) -> Option { + match error { + StorageError::VersionNotFound(_, _, _) | StorageError::ObjectNotFound(_, _) => { + if http_preconditions.if_match_value().is_some() { + Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string())) + } else { + None + } + } + error => Some(error), + } + } + pub(in crate::set_disk) async fn check_write_precondition( &self, bucket: &str, @@ -3892,19 +3910,8 @@ impl SetDisks { } } - Err(StorageError::VersionNotFound(_, _, _)) - | Err(StorageError::ObjectNotFound(_, _)) - | Err(StorageError::ErasureReadQuorum) => { - // When the object is not found, - // - if If-Match is set, we should return 404 NotFound - // - if If-None-Match is set, we should be able to proceed with the request - if http_preconditions.if_match_value().is_some() { - return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string())); - } - } - - Err(e) => { - return Some(e); + Err(error) => { + return Self::write_precondition_lookup_error(error, &http_preconditions, bucket, object); } } @@ -4407,6 +4414,41 @@ mod tests { use tempfile::TempDir; use tokio::io::AsyncReadExt; + #[test] + fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() { + let create_only = HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }; + let replace_only = HTTPPreconditions { + if_match: Some("etag".to_string()), + ..Default::default() + }; + + assert!(matches!( + SetDisks::write_precondition_lookup_error(StorageError::ErasureReadQuorum, &create_only, "bucket", "object",), + Some(StorageError::ErasureReadQuorum) + )); + assert!( + SetDisks::write_precondition_lookup_error( + StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()), + &create_only, + "bucket", + "object", + ) + .is_none() + ); + assert!(matches!( + SetDisks::write_precondition_lookup_error( + StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()), + &replace_only, + "bucket", + "object", + ), + Some(StorageError::ObjectNotFound(_, _)) + )); + } + fn metadata_test_fileinfo(object: &str) -> FileInfo { let mut fi = FileInfo::new(object, 2, 2); fi.volume = "bucket".to_string(); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 8ba420268..4c7308a81 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -9275,6 +9275,48 @@ mod tests { )); } + #[tokio::test] + async fn set_level_if_none_match_fails_closed_without_read_quorum() { + let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await; + let bucket = "bucket-write-precondition-quorum"; + let object = "existing-object.txt"; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created before disk loss"); + let mut reader = PutObjReader::from_vec(b"existing object body".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("object should be written before disk loss"); + { + let mut disks = set_disks.disks.write().await; + disks[1..].fill(None); + } + + let create_only = ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }; + let result = set_disks.check_write_precondition(bucket, object, &create_only).await; + assert!( + matches!(result, Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))), + "expected read-quorum failure, got {result:?}" + ); + } + #[tokio::test] async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() { let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await; diff --git a/crates/ecstore/src/set_disk/ops/bucket.rs b/crates/ecstore/src/set_disk/ops/bucket.rs index 0d66aed89..904e59e19 100644 --- a/crates/ecstore/src/set_disk/ops/bucket.rs +++ b/crates/ecstore/src/set_disk/ops/bucket.rs @@ -21,6 +21,71 @@ use super::super::*; +impl SetDisks { + pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec, bool)> { + let disks = self.disk_inventory().await; + let write_quorum = (disks.len() / 2) + 1; + + let mut futures = Vec::with_capacity(disks.len()); + for disk in disks { + futures.push(async move { + match disk { + Some(disk) => disk.list_volumes().await, + None => Err(DiskError::DiskNotFound), + } + }); + } + + let results = join_all(futures).await; + let mut topology_complete = results.iter().all(|result| result.is_ok()); + let mut infos = Vec::with_capacity(results.len()); + let mut errs = Vec::with_capacity(results.len()); + for result in results { + match result { + Ok(volumes) => { + infos.push(Some(volumes)); + errs.push(None); + } + Err(err) => { + infos.push(None); + errs.push(Some(err)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) { + return Err(err.into()); + } + + let mut counts: HashMap = HashMap::new(); + for volumes in infos.into_iter().flatten() { + for volume in volumes { + if is_reserved_or_invalid_bucket(&volume.name, false) { + continue; + } + + let entry = counts.entry(volume.name.clone()).or_insert(( + 0, + BucketInfo { + name: volume.name.clone(), + created: volume.created, + ..Default::default() + }, + )); + entry.0 += 1; + } + } + + topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum); + let mut buckets = counts + .into_values() + .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket)) + .collect::>(); + buckets.sort_by(|left, right| left.name.cmp(&right.name)); + Ok((buckets, topology_complete)) + } +} + #[async_trait::async_trait] impl BucketOperations for SetDisks { type Error = Error; @@ -117,65 +182,8 @@ impl BucketOperations for SetDisks { } #[tracing::instrument(skip(self))] - async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { - let disks = self.disk_inventory().await; - let write_quorum = (disks.len() / 2) + 1; - - let mut futures = Vec::with_capacity(disks.len()); - for disk in disks { - futures.push(async move { - match disk { - Some(disk) => disk.list_volumes().await, - None => Err(DiskError::DiskNotFound), - } - }); - } - - let results = join_all(futures).await; - let mut infos = Vec::with_capacity(results.len()); - let mut errs = Vec::with_capacity(results.len()); - for result in results { - match result { - Ok(volumes) => { - infos.push(Some(volumes)); - errs.push(None); - } - Err(err) => { - infos.push(None); - errs.push(Some(err)); - } - } - } - - if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) { - return Err(err.into()); - } - - let mut counts: HashMap = HashMap::new(); - for volumes in infos.into_iter().flatten() { - for volume in volumes { - if is_reserved_or_invalid_bucket(&volume.name, false) { - continue; - } - - let entry = counts.entry(volume.name.clone()).or_insert(( - 0, - BucketInfo { - name: volume.name.clone(), - created: volume.created, - ..Default::default() - }, - )); - entry.0 += 1; - } - } - - let mut buckets = counts - .into_values() - .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket)) - .collect::>(); - buckets.sort_by(|left, right| left.name.cmp(&right.name)); - Ok(buckets) + async fn list_bucket(&self, opts: &BucketOptions) -> Result> { + Ok(self.list_bucket_for_scanner(opts).await?.0) } #[tracing::instrument(skip(self))] diff --git a/crates/ecstore/src/storage_api_contracts/mod.rs b/crates/ecstore/src/storage_api_contracts/mod.rs index dbdb52d82..d061c257b 100644 --- a/crates/ecstore/src/storage_api_contracts/mod.rs +++ b/crates/ecstore/src/storage_api_contracts/mod.rs @@ -23,7 +23,12 @@ pub(crate) mod heal { pub(crate) mod internode { pub(crate) use rustfs_storage_api::{ - WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, + NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, + NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, + NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, + NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, + WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, }; } diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 0e25ea824..8112b7223 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -17,12 +17,21 @@ use crate::bucket::{ metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix}, utils::is_meta_bucketname, }; +use crate::error::is_err_bucket_not_found; use crate::runtime::sources as runtime_sources; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::bucket::SRBucketDeleteOp; use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use futures::stream::{self, StreamExt}; +use std::collections::BTreeMap; +use std::future::Future; const DELETED_BUCKETS_PREFIX: &str = ".deleted"; +const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4; + +fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize { + set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY) +} fn should_override_created_from_metadata(created: OffsetDateTime) -> bool { created != OffsetDateTime::UNIX_EPOCH @@ -78,6 +87,49 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String { format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket)) } +async fn await_bucket_namespace_operation( + guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, + operation: &'static str, + future: F, +) -> Result +where + F: Future>, +{ + let Some(guard) = guard else { + return future.await; + }; + if guard.is_lock_lost() { + return Err(StorageError::other(format!( + "bucket namespace lock was lost before {operation}: {bucket}" + ))); + } + tokio::select! { + biased; + _ = guard.lock_lost_notified() => Err(StorageError::other(format!( + "bucket namespace lock was lost during {operation}: {bucket}" + ))), + result = future => result, + } +} + +async fn run_bucket_usage_cleanup(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()> +where + F: Future>, +{ + await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await +} + +async fn run_physical_bucket_deletion(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()> +where + F: Future>, +{ + // Fence before polling deletion: the physical namespace may become + // invisible at any await point inside the storage operation. + list_objects::observe_scanner_namespace_mutations(bucket, 1); + await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await +} + impl ECStore { async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> { let marker_volume = bucket_deleted_marker_volume(bucket); @@ -96,21 +148,84 @@ impl ECStore { Ok(()) } - async fn cleanup_deleted_bucket_metadata(&self, bucket: &str, include_deleted_marker: bool) -> Result<()> { + async fn cleanup_deleted_bucket_metadata( + &self, + bucket: &str, + include_deleted_marker: bool, + guard: Option<&rustfs_lock::NamespaceLockGuard>, + ) -> Result<()> { for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) { - self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()).await?; + await_bucket_namespace_operation( + guard, + bucket, + "deleted bucket metadata cleanup", + self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()), + ) + .await?; } if include_deleted_marker { let marker_prefix = bucket_deleted_marker_prefix(bucket); - self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()).await?; + await_bucket_namespace_operation( + guard, + bucket, + "deleted bucket marker cleanup", + self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()), + ) + .await?; } - metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?; + await_bucket_namespace_operation( + guard, + bucket, + "deleted bucket metadata cache cleanup", + metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket), + ) + .await?; runtime_sources::delete_bucket_monitor_entry(bucket); Ok(()) } + async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> { + run_bucket_usage_cleanup(guard, bucket, async { + crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?; + crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await + }) + .await + } + + async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) { + if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await { + warn!( + bucket = %bucket, + error = ?err, + "bucket data usage cleanup deferred to scanner reconciliation" + ); + } + } + + async fn rollback_failed_bucket_creation(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) { + let rollback_opts = DeleteBucketOptions { + no_lock: true, + no_recreate: true, + ..Default::default() + }; + if let Err(err) = await_bucket_namespace_operation(guard, bucket, "failed bucket creation rollback", async { + self.peer_sys + .delete_bucket(bucket, &rollback_opts) + .await + .map_err(|rollback_err| to_object_err(rollback_err.into(), vec![bucket])) + }) + .await + { + warn!( + bucket = %bucket, + error = ?err, + "failed bucket creation rollback did not remove every physical bucket volume" + ); + } + } + #[instrument(skip(self))] pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { if !is_meta_bucketname(bucket) @@ -119,7 +234,7 @@ impl ECStore { return Err(StorageError::BucketNameInvalid(err.to_string())); } - let _ns_guard = if !opts.no_lock { + let ns_guard = if !opts.no_lock { let ns_lock = self.new_ns_lock(bucket, bucket).await?; Some( ns_lock @@ -142,8 +257,34 @@ impl ECStore { None }; - if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await { - let err = to_object_err(err.into(), vec![bucket]); + let confirmed_missing = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await { + Ok(_) => false, + Err(err) => { + let err: StorageError = err.into(); + if is_err_bucket_not_found(&err) { + true + } else { + return Err(to_object_err(err, vec![bucket])); + } + } + }; + + if confirmed_missing && !is_meta_bucketname(bucket) { + // Fence every scanner cycle that could have observed the namespace + // before physical creation. Creation may become visible even when a + // later metadata write or namespace-lock check fails. + crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1); + self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?; + } + + if let Err(err) = await_bucket_namespace_operation(ns_guard.as_ref(), bucket, "physical bucket creation", async { + self.peer_sys + .make_bucket(bucket, opts) + .await + .map_err(|err| to_object_err(err.into(), vec![bucket])) + }) + .await + { if is_err_bucket_exists(&err) && let Err(heal_err) = self .handle_heal_bucket( @@ -157,18 +298,9 @@ impl ECStore { { warn!("best-effort bucket heal after BucketExists failed: {heal_err}"); } - if !is_err_bucket_exists(&err) { + if !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) { error!("make bucket failed: {err}"); - let _ = self - .delete_bucket( - bucket, - &DeleteBucketOptions { - no_lock: true, - no_recreate: true, - ..Default::default() - }, - ) - .await; + self.rollback_failed_bucket_creation(bucket, ns_guard.as_ref()).await; } return Err(err); }; @@ -186,7 +318,13 @@ impl ECStore { meta.versioning_config_xml = crate::bucket::utils::serialize::(&enableVersioningConfig)?; } - metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await?; + await_bucket_namespace_operation( + ns_guard.as_ref(), + bucket, + "bucket metadata initialization", + metadata_sys::set_bucket_metadata_in(&self.ctx, meta), + ) + .await?; Ok(()) } @@ -224,6 +362,80 @@ impl ECStore { Ok(buckets) } + pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result { + let sets = self + .pools + .iter() + .flat_map(|pool| { + pool.disk_set + .iter() + .map(|set| (set.pool_index, set.set_index, Arc::clone(set))) + }) + .collect::>(); + let set_count = sets.len(); + let deleted = opts.deleted; + let cached = opts.cached; + let no_metadata = opts.no_metadata; + let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| { + let opts = BucketOptions { + deleted, + cached, + no_metadata, + }; + async move { + set.list_bucket_for_scanner(&opts) + .await + .map(|(buckets, complete)| (pool_index, set_index, buckets, complete)) + } + })) + .buffer_unordered(scanner_bucket_list_set_concurrency(set_count)); + let mut topology_complete = set_count != 0; + let mut bucket_map = BTreeMap::::new(); + let mut scoped_buckets = Vec::with_capacity(set_count); + while let Some(set_listing) = set_listings.next().await { + let (pool_index, set_index, buckets, set_complete) = set_listing?; + topology_complete &= set_complete; + for bucket in &buckets { + bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone()); + } + scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing { + pool_index, + set_index, + buckets, + }); + } + scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index)); + let mut listing = crate::cluster::rpc::ScannerBucketListing { + buckets: bucket_map.into_values().collect(), + set_buckets: scoped_buckets, + topology_complete, + }; + + if !opts.no_metadata { + for bucket in &mut listing.buckets { + if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await + && should_override_created_from_metadata(created) + { + bucket.created = Some(created); + } + } + let created_by_bucket = listing + .buckets + .iter() + .map(|bucket| (bucket.name.as_str(), bucket.created)) + .collect::>(); + for scope in &mut listing.set_buckets { + for bucket in &mut scope.buckets { + if let Some(created) = created_by_bucket.get(bucket.name.as_str()) { + bucket.created = *created; + } + } + } + } + + Ok(listing) + } + #[instrument(skip(self))] pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { if is_meta_bucketname(bucket) { @@ -234,7 +446,7 @@ impl ECStore { return Err(StorageError::BucketNameInvalid(err.to_string())); } - let _ns_guard = if !opts.no_lock { + let ns_guard = if !opts.no_lock { let ns_lock = self.new_ns_lock(bucket, bucket).await?; Some( ns_lock @@ -299,17 +511,40 @@ impl ECStore { } if sr_mark_delete { - self.mark_bucket_deleted(bucket).await?; + await_bucket_namespace_operation( + ns_guard.as_ref(), + bucket, + "bucket delete marker creation", + self.mark_bucket_deleted(bucket), + ) + .await?; } - if let Err(err) = self.peer_sys.delete_bucket(bucket, &delete_opts).await { - let storage_err = to_object_err(err.into(), vec![bucket]); - if !sr_delete || !is_err_strict_volume_not_found(&storage_err) { - return Err(storage_err); - } + let delete_result = run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async { + self.peer_sys + .delete_bucket(bucket, &delete_opts) + .await + .map_err(|err| to_object_err(err.into(), vec![bucket])) + }) + .await; + if let Err(err) = delete_result + && (!sr_delete || !is_err_strict_volume_not_found(&err)) + { + return Err(err); } - self.cleanup_deleted_bucket_metadata(bucket, sr_purge).await?; + self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await; + + if let Err(err) = self + .cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref()) + .await + { + warn!( + bucket = %bucket, + error = ?err, + "physical bucket deletion succeeded but metadata cleanup remains pending" + ); + } Ok(()) } } @@ -317,8 +552,9 @@ impl ECStore { #[cfg(test)] mod tests { use super::{ - bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume, - should_override_created_from_metadata, validate_table_bucket_delete_allowed, + SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, + bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion, + scanner_bucket_list_set_concurrency, should_override_created_from_metadata, validate_table_bucket_delete_allowed, }; use crate::bucket::metadata::table_bucket_catalog_metadata_prefix; use crate::bucket::metadata_sys; @@ -335,16 +571,146 @@ mod tests { disk::endpoint::Endpoint, layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, }; + use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo}; + use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; use serial_test::serial; use std::path::{Path, PathBuf}; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::{Duration, SystemTime}; use time::OffsetDateTime; - use tokio::sync::OnceCell; + use tokio::sync::{Notify, OnceCell}; use tokio_util::sync::CancellationToken; use uuid::Uuid; static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec, Arc)> = OnceCell::const_new(); + #[tokio::test(start_paused = true)] + async fn bucket_namespace_operation_fails_closed_after_lease_expiry() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + tokio::time::advance(ttl + Duration::from_millis(1)).await; + + let operation_ran = Arc::new(AtomicBool::new(false)); + let operation_ran_for_future = operation_ran.clone(); + let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move { + operation_ran_for_future.store(true, Ordering::SeqCst); + Ok(()) + }) + .await; + + assert!(result.is_err(), "an expired namespace lease must fence the operation"); + assert!( + !operation_ran.load(Ordering::SeqCst), + "a fenced namespace operation must not poll its mutation future" + ); + } + + #[tokio::test] + #[serial] + async fn physical_bucket_delete_fences_scanner_before_polling_storage() { + let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation(); + let storage_polled = Arc::new(AtomicBool::new(false)); + let storage_polled_for_future = storage_polled.clone(); + + run_physical_bucket_deletion(None, "generation-order-bucket", async move { + assert!( + crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before, + "scanner generation must advance before physical deletion is polled" + ); + storage_polled_for_future.store(true, Ordering::SeqCst); + Ok(()) + }) + .await + .expect("synthetic physical deletion should succeed"); + + assert!(storage_polled.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn bucket_usage_cleanup_stops_after_parent_cancellation() { + let started = Arc::new(Notify::new()); + let started_wait = started.notified(); + let release = Arc::new(Notify::new()); + let completed = Arc::new(AtomicBool::new(false)); + let started_for_cleanup = started.clone(); + let release_for_cleanup = release.clone(); + let completed_for_cleanup = completed.clone(); + let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move { + started_for_cleanup.notify_one(); + release_for_cleanup.notified().await; + completed_for_cleanup.store(true, Ordering::SeqCst); + Ok(()) + })); + started_wait.await; + + parent.abort(); + let _ = parent.await; + tokio::task::yield_now().await; + release.notify_waiters(); + tokio::task::yield_now().await; + assert!( + !completed.load(Ordering::SeqCst), + "a cancelled cleanup future must not continue in a detached task" + ); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() { + let ttl = Duration::from_millis(20); + let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new())); + let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner") + .with_acquire_timeout(Duration::from_secs(1)) + .with_ttl(ttl) + .with_refresh_interval(ttl); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not fail") + .expect("namespace lock should be acquired"); + + let operation_started = Arc::new(Notify::new()); + let started_wait = operation_started.notified(); + let operation_release = Arc::new(Notify::new()); + let operation_completed = Arc::new(AtomicBool::new(false)); + let started_for_operation = operation_started.clone(); + let release_for_operation = operation_release.clone(); + let completed_for_operation = operation_completed.clone(); + let task = tokio::spawn(async move { + await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move { + started_for_operation.notify_one(); + release_for_operation.notified().await; + completed_for_operation.store(true, Ordering::SeqCst); + Ok(()) + }) + .await + }); + started_wait.await; + + tokio::time::advance(ttl + Duration::from_millis(1)).await; + let err = task + .await + .expect("operation task should join") + .expect_err("an operation still running after lease loss must be fenced"); + assert!(err.to_string().contains("namespace lock was lost during test operation")); + operation_release.notify_waiters(); + tokio::task::yield_now().await; + assert!( + !operation_completed.load(Ordering::SeqCst), + "lock loss must stop the old owner before a successor can acquire the namespace" + ); + } + async fn setup_bucket_delete_test_env() -> (Vec, Arc) { BUCKET_DELETE_TEST_ENV .get_or_init(|| async { @@ -413,6 +779,58 @@ mod tests { .clone() } + async fn setup_multi_pool_scanner_listing_test_env() -> (tempfile::TempDir, Arc) { + let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created"); + let mut pools = Vec::new(); + for pool_index in 0..2 { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("multi-pool scanner test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pools.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: format!("scanner-listing-pool-{pool_index}"), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }); + } + + let endpoint_pools = EndpointServerPools(pools); + let instance_ctx = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("multi-pool local disks should initialize"); + let ecstore = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address"), + endpoint_pools, + CancellationToken::new(), + instance_ctx, + ) + .await + .expect("multi-pool ECStore should initialize"); + let storage_class = + crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4]) + .expect("multi-pool storage class should match both four-disk pools"); + for pool in &ecstore.pools { + for set in &pool.disk_set { + set.set_test_storage_class_config(storage_class.clone()); + } + } + + (temp_dir, ecstore) + } + async fn create_bucket_with_object(ecstore: &Arc, bucket: &str, object: &str) { let generation_before_make = ecstore.scanner_namespace_mutation_generation(); ecstore @@ -515,6 +933,98 @@ mod tests { ); } + #[test] + fn scanner_bucket_listing_bounds_set_fanout() { + assert_eq!(scanner_bucket_list_set_concurrency(0), 1); + assert_eq!(scanner_bucket_list_set_concurrency(2), 2); + assert_eq!(scanner_bucket_list_set_concurrency(100), SCANNER_BUCKET_LIST_SET_CONCURRENCY); + } + + #[tokio::test] + #[serial] + async fn scanner_bucket_listing_unions_every_erasure_set() { + let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await; + let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple()); + ecstore.pools[1].disk_set[0] + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in the second pool only"); + + let listing = ecstore + .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("scanner should enumerate every pool and set"); + + assert!(listing.topology_complete); + assert!(listing.buckets.iter().any(|entry| entry.name == bucket)); + assert_eq!(listing.set_buckets.len(), 2); + assert!( + listing + .set_buckets + .iter() + .find(|scope| scope.pool_index == 0 && scope.set_index == 0) + .is_some_and(|scope| scope.buckets.is_empty()) + ); + assert!( + listing + .set_buckets + .iter() + .find(|scope| scope.pool_index == 1 && scope.set_index == 0) + .is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket)) + ); + } + + #[tokio::test] + async fn scanner_bucket_listing_marks_degraded_set_incomplete() { + let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await; + let bucket = format!("degraded-set-{}", Uuid::new_v4().simple()); + let set = &ecstore.pools[0].disk_set[0]; + set.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created before a disk is removed"); + set.disks.write().await[0] = None; + + let listing = ecstore + .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("a degraded set with quorum should still return candidate buckets"); + + assert!(listing.buckets.iter().any(|entry| entry.name == bucket)); + assert!(!listing.topology_complete); + } + + #[tokio::test] + async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() { + let (temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await; + let bucket = format!("divergent-set-{}", Uuid::new_v4().simple()); + ecstore.pools[0].disk_set[0] + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created before disk views diverge"); + for disk_index in 0..2 { + tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket)) + .await + .expect("test bucket directory should be removed from a minority disk view"); + } + + let listing = ecstore + .list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("responsive disks should still produce a scanner candidate listing"); + + assert!(listing.buckets.iter().all(|entry| entry.name != bucket)); + assert!(!listing.topology_complete); + } + // These tests share one isolated instance and mutate its bucket metadata; // serialize them so their assertions cannot observe each other's operations. #[tokio::test] @@ -736,4 +1246,210 @@ mod tests { "failed default S3 DeleteBucket must keep metadata cache" ); } + + #[tokio::test] + #[serial] + async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() { + let (_, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple()); + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut snapshot = DataUsageInfo { + last_update: Some(SystemTime::now()), + buckets_count: 1, + ..Default::default() + }; + snapshot.buckets_usage.insert( + bucket.clone(), + BucketUsageInfo { + size: 42, + objects_count: 1, + versions_count: 1, + ..Default::default() + }, + ); + snapshot.bucket_sizes.insert(bucket.clone(), 42); + snapshot.calculate_totals(); + crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone()) + .await + .expect("usage snapshot should be stored"); + + ecstore + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("empty bucket should be deleted"); + let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone()) + .await + .expect("usage snapshot should remain readable"); + assert!(!deleted.buckets_usage.contains_key(&bucket)); + + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("same bucket name should be recreated after delete returns"); + crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await; + + let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone()) + .await + .expect("recreated bucket usage base should load"); + crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await; + assert_eq!( + recreated + .buckets_usage + .get(&bucket) + .map(|usage| (usage.objects_count, usage.versions_count, usage.size)), + Some((1, 1, 84)) + ); + } + + #[tokio::test] + #[serial] + async fn bucket_create_removes_stale_usage_before_physical_creation() { + let (_, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple()); + let mut snapshot = DataUsageInfo { + last_update: Some(SystemTime::now()), + buckets_count: 1, + ..Default::default() + }; + snapshot.buckets_usage.insert( + bucket.clone(), + BucketUsageInfo { + size: 42, + objects_count: 1, + versions_count: 1, + ..Default::default() + }, + ); + snapshot.bucket_sizes.insert(bucket.clone(), 42); + snapshot.calculate_totals(); + crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone()) + .await + .expect("stale usage fixture should be stored"); + + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("CreateBucket should succeed"); + + let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone()) + .await + .expect("usage snapshot should remain readable"); + assert!( + !persisted.buckets_usage.contains_key(&bucket), + "a newly created bucket must not inherit the predecessor generation's usage" + ); + assert!( + ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(), + "physical creation should happen after the usage fence succeeds" + ); + } + + #[tokio::test] + #[serial] + async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() { + let (_, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple()); + ecstore + .peer_sys + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("the partial-create fixture should expose a physical bucket"); + + let mut snapshot = DataUsageInfo { + last_update: Some(SystemTime::now()), + buckets_count: 1, + ..Default::default() + }; + snapshot.buckets_usage.insert( + bucket.clone(), + BucketUsageInfo { + size: 42, + objects_count: 1, + versions_count: 1, + ..Default::default() + }, + ); + snapshot.bucket_sizes.insert(bucket.clone(), 42); + snapshot.calculate_totals(); + crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone()) + .await + .expect("the usage fixture should be stored"); + + ecstore.rollback_failed_bucket_creation(&bucket, None).await; + + assert!( + ecstore + .peer_sys + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .is_err(), + "failed-create rollback should remove the partial physical bucket" + ); + let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone()) + .await + .expect("the usage snapshot should remain readable"); + assert!( + persisted.buckets_usage.contains_key(&bucket), + "failed-create rollback must not start an unfenced usage cleanup" + ); + + crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone()) + .await + .expect("the rollback usage fixture should be cleared"); + } + + #[tokio::test] + #[serial] + async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() { + let (_, ecstore) = setup_bucket_delete_test_env().await; + let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple()); + ecstore + .make_bucket(&deleted_bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created before corrupting usage"); + + let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}"); + crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec()) + .await + .expect("corrupt usage fixture should be stored"); + + ecstore + .delete_bucket(&deleted_bucket, &DeleteBucketOptions::default()) + .await + .expect("usage snapshot corruption must not block DeleteBucket"); + assert!( + ecstore + .get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .is_err(), + "successful DeleteBucket must remove the physical bucket" + ); + + let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple()); + let create_err = ecstore + .make_bucket(&new_bucket, &MakeBucketOptions::default()) + .await + .expect_err("MakeBucket must fail when stale usage cannot be fenced"); + assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced"); + assert!( + ecstore + .get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .is_err(), + "failed MakeBucket must not expose a new physical bucket" + ); + + let restored = serde_json::to_vec(&DataUsageInfo { + last_update: Some(SystemTime::now()), + ..Default::default() + }) + .expect("restored usage fixture should encode"); + crate::config::com::save_config(ecstore, &usage_path, restored) + .await + .expect("usage fixture should be restored after the failure-path test"); + } } diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 11f50c148..8a1697d8f 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -619,7 +619,7 @@ pub(super) fn scanner_namespace_mutation_generation() -> u64 { SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire) } -pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { +pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { if bucket == RUSTFS_META_BUCKET { return; } diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 4f8e8e7cb..f706aef15 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -322,6 +322,11 @@ impl ECStore { pub fn scanner_namespace_mutation_generation(&self) -> u64 { list_objects::scanner_namespace_mutation_generation() } + + pub async fn scanner_data_movement_active(&self) -> bool { + let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started()); + decommission || rebalance + } } // impl Clone for ECStore { @@ -440,11 +445,7 @@ impl BucketOperations for ECStore { #[instrument(skip(self))] async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { - let result = self.handle_make_bucket(bucket, opts).await; - if result.is_ok() { - list_objects::observe_scanner_namespace_mutations(bucket, 1); - } - result + Box::pin(self.handle_make_bucket(bucket, opts)).await } #[instrument(skip(self))] @@ -457,11 +458,7 @@ impl BucketOperations for ECStore { } #[instrument(skip(self))] async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { - let result = self.handle_delete_bucket(bucket, opts).await; - if result.is_ok() { - list_objects::observe_scanner_namespace_mutations(bucket, 1); - } - result + Box::pin(self.handle_delete_bucket(bucket, opts)).await } } @@ -871,9 +868,9 @@ mod tests { // Build a minimal ECStore carrying an explicit instance context. Empty // pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`. - fn build_store_with_ctx(ctx: Arc) -> ECStore { + fn build_store_with_ctx(ctx: Arc) -> Arc { let endpoint_pools = EndpointServerPools::default(); - ECStore { + Arc::new(ECStore { id: uuid::Uuid::new_v4(), disk_map: std::collections::HashMap::new(), pools: Vec::new(), @@ -884,7 +881,7 @@ mod tests { start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::new(()), ctx, - } + }) } // The object graph is the isolation carrier: two ECStore instances holding diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 2c0985a56..55e2ca99e 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -23,6 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; pub const INTERNODE_OPERATION_READ_FILE_STREAM: &str = "read_file_stream"; pub const INTERNODE_OPERATION_PUT_FILE_STREAM: &str = "put_file_stream"; pub const INTERNODE_OPERATION_WALK_DIR: &str = "walk_dir"; +pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner"; pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all"; pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all"; pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple"; diff --git a/crates/lock/src/distributed_lock.rs b/crates/lock/src/distributed_lock.rs index b25b6f97f..d6fb896f6 100644 --- a/crates/lock/src/distributed_lock.rs +++ b/crates/lock/src/distributed_lock.rs @@ -18,16 +18,20 @@ use crate::{ error::{LockError, Result}, types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType}, }; -use futures::future::join_all; +use futures::{ + future::join_all, + stream::{FuturesUnordered, StreamExt}, +}; use rustfs_io_metrics::{ record_lock_refresh_quorum_lost, record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release, }; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::sync::Notify; use tokio::task::{JoinHandle, JoinSet}; +use tokio::time::Instant; use tracing::{debug, warn}; use uuid::Uuid; @@ -80,10 +84,10 @@ fn is_unrecoverable_quorum_error(error: &str) -> bool { /// Deliberately avoids `Duration::clamp` (which asserts `min <= max` and would panic for /// sub-second ttls where `ttl - 1s` underflows to zero). Returns `None` for degenerate cases so /// no heartbeat is spawned: -/// - `entries_len <= 1`: single/degenerate path (matches a local, non-distributed lock), +/// - `entries_len == 0`: no acquired backend lease to renew, /// - `interval.is_zero()` or `interval >= ttl`: too small a ttl / too large an interval to renew. fn derive_refresh_interval(entries_len: usize, ttl: Duration, injected: Option) -> Option { - if entries_len <= 1 { + if entries_len == 0 { return None; } let interval = injected.unwrap_or(ttl / 3); @@ -130,10 +134,29 @@ fn should_warn_lock_failure(error: &str) -> bool { #[derive(Debug, Default)] pub struct LockLostSignal { lost: AtomicBool, + valid_until: Mutex>, notify: Notify, } impl LockLostSignal { + fn set_valid_until(&self, valid_until: Instant) { + if self.lost.load(Ordering::SeqCst) { + return; + } + match self.valid_until.lock() { + Ok(mut deadline) => { + if deadline.is_some_and(|current| Instant::now() >= current) { + drop(deadline); + self.mark_lost(); + return; + } + *deadline = Some(valid_until); + self.notify.notify_waiters(); + } + Err(_) => self.mark_lost(), + } + } + fn mark_lost(&self) { self.lost.store(true, Ordering::SeqCst); self.notify.notify_waiters(); @@ -141,16 +164,81 @@ impl LockLostSignal { /// Whether refresh quorum has been lost for the associated guard. pub fn is_lost(&self) -> bool { - self.lost.load(Ordering::SeqCst) + if self.lost.load(Ordering::SeqCst) { + return true; + } + let expired = match self.valid_until.lock() { + Ok(deadline) => deadline.is_some_and(|valid_until| Instant::now() >= valid_until), + Err(_) => true, + }; + if expired { + self.mark_lost(); + } + expired } /// Resolves once the lock is declared lost (immediately if already lost). pub async fn notified(&self) { - if self.is_lost() { - return; - } - self.notify.notified().await; + self.notified_after_registration(|| {}).await; } + + async fn notified_after_registration(&self, after_registration: impl FnOnce()) { + let mut after_registration = Some(after_registration); + loop { + let notified = self.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if let Some(after_registration) = after_registration.take() { + after_registration(); + } + if self.is_lost() { + return; + } + + let valid_until = match self.valid_until.lock() { + Ok(deadline) => *deadline, + Err(_) => return, + }; + match valid_until { + Some(valid_until) => { + tokio::select! { + _ = tokio::time::sleep_until(valid_until) => {} + _ = &mut notified => {} + } + } + None => notified.await, + } + } + } +} + +#[derive(Clone, Debug)] +struct HeldLockEntry { + lock_id: LockId, + client: Arc, + valid_until: Instant, +} + +impl HeldLockEntry { + fn release_entry(&self) -> (LockId, Arc) { + (self.lock_id.clone(), self.client.clone()) + } +} + +#[derive(Clone, Debug)] +struct LockHeartbeatConfig { + ttl: Duration, + interval: Option, + refresh_quorum: usize, + owner: String, + resource: ObjectKey, +} + +#[derive(Clone, Copy, Debug, Default)] +struct LockRefreshStats { + refreshed: usize, + not_found: usize, + refresh_errors: usize, } /// A RAII guard for distributed locks that releases the lock asynchronously when dropped. @@ -159,14 +247,12 @@ pub struct DistributedLockGuard { /// The public-facing lock id. For multi-client scenarios this is typically /// an aggregate id; for single-client it is the only id. lock_id: LockId, - /// All underlying (LockId, client) entries that should be released when the - /// guard is dropped. - entries: Vec<(LockId, Arc)>, + /// All underlying leases that should be refreshed and released with the guard. + entries: Vec, lock_type: LockType, /// If true, Drop will not try to release (used if user manually released). disarmed: bool, - /// Background heartbeat task that periodically refreshes the per-client leases. - /// `None` for degenerate/single-client paths that do not need renewal. + /// Background lease task for renewable leases. refresh_task: Option>, /// Lock-loss signal, shared with the heartbeat task. lock_lost: Arc, @@ -176,31 +262,22 @@ impl DistributedLockGuard { /// Create a new guard. /// /// - `lock_id` is the id returned to the caller (`lock_id()`). - /// - `entries` is the full list of underlying (LockId, client) pairs - /// that should be released when this guard is dropped. - /// - `refresh_interval`: `Some` spawns a heartbeat that refreshes every entry on that - /// cadence; `None` (degenerate/single-client, or interval derived away) spawns nothing. - /// - `refresh_quorum`: minimum refreshes that must keep succeeding; if `not_found` exceeds - /// `entries.len() - refresh_quorum` the guard is declared lost. - /// - `owner`/`resource`: diagnostics only. - pub(crate) fn new( - lock_id: LockId, - entries: Vec<(LockId, Arc)>, - lock_type: LockType, - refresh_interval: Option, - refresh_quorum: usize, - owner: String, - resource: ObjectKey, - ) -> Self { + /// - `entries` is the full list of underlying leases that should be refreshed and released + /// with this guard. + /// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics. + fn new(lock_id: LockId, entries: Vec, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self { record_lock_held_acquire(lock_type); let lock_lost = Arc::new(LockLostSignal::default()); - let refresh_task = refresh_interval.and_then(|interval| { - // Only spawn when a tokio runtime is available; guard construction off-runtime - // (e.g. some tests) simply skips the heartbeat. + match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) { + Some(valid_until) => lock_lost.set_valid_until(valid_until), + None => lock_lost.mark_lost(), + } + // Non-renewable guards are fenced directly by LockLostSignal's deadline checks. + let refresh_task = heartbeat.interval.and_then(|_| { tokio::runtime::Handle::try_current().ok().map(|handle| { let entries = entries.clone(); let lock_lost = lock_lost.clone(); - handle.spawn(Self::run_heartbeat(entries, interval, refresh_quorum, lock_lost, owner, resource)) + handle.spawn(Self::run_heartbeat(entries, heartbeat, lock_lost)) }) }); Self { @@ -213,59 +290,107 @@ impl DistributedLockGuard { } } - /// Heartbeat loop: every `interval`, refresh all entries and classify the outcomes. - /// `Ok(true)` = refreshed, `Ok(false)` = not_found, `Err` = RPC jitter (ignored, absorbed by - /// the ttl > interval margin and retried next tick). Declares the lock lost when - /// `not_found > entries.len() - refresh_quorum`. Phase 1 does not release on loss: the guarded - /// operation is still running, and tearing the lock down here would only widen the window. - async fn run_heartbeat( - entries: Vec<(LockId, Arc)>, - interval: Duration, - refresh_quorum: usize, - lock_lost: Arc, - owner: String, - resource: ObjectKey, + fn quorum_valid_until(entries: &[HeldLockEntry], refresh_quorum: usize) -> Option { + if refresh_quorum == 0 || entries.len() < refresh_quorum { + return None; + } + + let mut deadlines: Vec<_> = entries.iter().map(|entry| entry.valid_until).collect(); + deadlines.sort_unstable(); + deadlines.get(deadlines.len() - refresh_quorum).copied() + } + + fn signal_refresh_quorum_lost( + lock_lost: &LockLostSignal, + config: &LockHeartbeatConfig, + entries: usize, + stats: LockRefreshStats, ) { - let tolerable_not_found = entries.len().saturating_sub(refresh_quorum); - let mut ticker = tokio::time::interval(interval); + warn!( + resource = %config.resource, + owner = config.owner, + refreshed = stats.refreshed, + not_found = stats.not_found, + refresh_errors = stats.refresh_errors, + entries, + refresh_quorum = config.refresh_quorum, + "lock refresh lost quorum" + ); + record_lock_refresh_quorum_lost(); + lock_lost.mark_lost(); + } + + /// Refresh every tracked lease while a quorum is still known to be valid. + /// + /// A successful refresh extends only that entry's conservative local deadline. RPC errors + /// retain the previous deadline, so transient jitter is tolerated but an unconfirmed lease + /// can never remain valid beyond its backend TTL. Waiting for a slow refresh is also fenced + /// by the current quorum deadline. + async fn run_heartbeat(mut entries: Vec, config: LockHeartbeatConfig, lock_lost: Arc) { + let Some(interval) = config.interval else { + return; + }; + + let now = Instant::now(); + let first_refresh = now.checked_add(interval).unwrap_or(now); + let mut ticker = tokio::time::interval_at(first_refresh, interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - // The first tick fires immediately; skip it so the first refresh lands one interval - // after acquisition rather than right away. - ticker.tick().await; + loop { - ticker.tick().await; + let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else { + Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default()); + return; + }; + lock_lost.set_valid_until(valid_until); - let results = join_all(entries.iter().map(|(lock_id, client)| { - let lock_id = lock_id.clone(); - let client = client.clone(); - async move { client.refresh(&lock_id).await } - })) - .await; - - let mut refreshed = 0usize; - let mut not_found = 0usize; - for result in &results { - match result { - Ok(true) => refreshed += 1, - Ok(false) => not_found += 1, - // RPC jitter: count as neither; the ttl > interval margin covers a transient - // dip and the next tick retries. - Err(_) => {} + tokio::select! { + biased; + _ = tokio::time::sleep_until(valid_until) => { + Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default()); + return; } + _ = ticker.tick() => {} } - if not_found > tolerable_not_found { - warn!( - resource = %resource, - owner = %owner, - refreshed, - not_found, - entries = entries.len(), - refresh_quorum, - "lock refresh lost quorum" - ); - record_lock_refresh_quorum_lost(); - lock_lost.mark_lost(); + let mut stats = LockRefreshStats::default(); + let mut pending = FuturesUnordered::new(); + for (idx, entry) in entries.iter().enumerate() { + let refresh_started = Instant::now(); + let lock_id = entry.lock_id.clone(); + let client = entry.client.clone(); + pending.push(async move { (idx, refresh_started, client.refresh(&lock_id).await) }); + } + + while !pending.is_empty() { + let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else { + Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats); + return; + }; + lock_lost.set_valid_until(valid_until); + + let next = tokio::select! { + biased; + _ = tokio::time::sleep_until(valid_until) => { + Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats); + return; + } + next = pending.next() => next + }; + let Some((idx, refresh_started, result)) = next else { + break; + }; + + match result { + Ok(true) => { + stats.refreshed += 1; + entries[idx].valid_until = refresh_started.checked_add(config.ttl).unwrap_or(refresh_started); + } + Ok(false) => { + stats.not_found += 1; + entries[idx].valid_until = Instant::now(); + } + Err(_) => stats.refresh_errors += 1, + } } } } @@ -326,7 +451,7 @@ impl DistributedLockGuard { return true; } - let entries = self.entries.clone(); + let entries = self.entries.iter().map(HeldLockEntry::release_entry).collect(); DistributedLock::spawn_release_cleanup(entries, "distributed_lock_guard_release"); // Disarm to prevent double-release on drop @@ -361,7 +486,7 @@ type LockAcquireTaskResult = (usize, Result); struct LockAcquireQuorumResult { response: LockResponse, - individual_locks: Vec<(LockId, Arc)>, + individual_locks: Vec, failure_kind: Option, quorum_impossible: bool, } @@ -438,15 +563,18 @@ impl DistributedLock { // Heartbeat operates on the per-client individual locks (their per-client ids), never // the aggregate id, so refreshes round-trip to the exact backend entries. - let refresh_interval = derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval); + let heartbeat = LockHeartbeatConfig { + ttl: request.ttl, + interval: derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval), + refresh_quorum: required_quorum, + owner: request.owner.clone(), + resource: request.resource.clone(), + }; Ok(Some(DistributedLockGuard::new( aggregate_lock_id, individual_locks, request.lock_type, - refresh_interval, - required_quorum, - request.owner.clone(), - request.resource.clone(), + heartbeat, ))) } else { // Check if it's a timeout or quorum failure @@ -738,7 +866,7 @@ impl DistributedLock { fn lock_acquire_attempt_timeout_result( timeout: Duration, - individual_locks: Vec<(LockId, Arc)>, + individual_locks: Vec, last_failure: Option, last_failure_kind: Option, ) -> LockAcquireQuorumResult { @@ -770,19 +898,23 @@ impl DistributedLock { /// Returns the LockResponse with aggregate lock_id and individual lock mappings. async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result { let required_quorum = self.required_quorum(request.lock_type); + let attempt_started = Instant::now(); let mut pending = self.spawn_lock_requests(request); - let mut individual_locks: Vec<(LockId, Arc)> = Vec::new(); + let mut individual_locks: Vec = Vec::new(); let fallback_lock_id = request.lock_id.clone(); let mut last_failure = None; let mut last_failure_kind = None; let mut last_hard_failure_kind = None; let mut hard_failures = 0usize; - let start = tokio::time::Instant::now(); + let start = Instant::now(); while !pending.is_empty() { let remaining = request.acquire_timeout.saturating_sub(start.elapsed()); if remaining.is_zero() { - Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout"); + Self::spawn_release_cleanup( + individual_locks.iter().map(HeldLockEntry::release_entry).collect(), + "distributed_lock_attempt_timeout", + ); Self::spawn_pending_cleanup( pending, self.clients.clone(), @@ -801,7 +933,10 @@ impl DistributedLock { Ok(Some(join_result)) => join_result, Ok(None) => break, Err(_) => { - Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout"); + Self::spawn_release_cleanup( + individual_locks.iter().map(HeldLockEntry::release_entry).collect(), + "distributed_lock_attempt_timeout", + ); Self::spawn_pending_cleanup( pending, self.clients.clone(), @@ -827,7 +962,11 @@ impl DistributedLock { .unwrap_or_else(|| fallback_lock_id.clone()); if let Some(client) = self.clients.get(idx) { - individual_locks.push((lock_id, client.clone())); + individual_locks.push(HeldLockEntry { + lock_id, + client: client.clone(), + valid_until: attempt_started.checked_add(request.ttl).unwrap_or(attempt_started), + }); } else { tracing::warn!("Missing lock client at index {} while recording success", idx); } @@ -862,7 +1001,10 @@ impl DistributedLock { if self.clients.len().saturating_sub(hard_failures) < required_quorum { let rollback_count = individual_locks.len(); - Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback"); + Self::spawn_release_cleanup( + individual_locks.iter().map(HeldLockEntry::release_entry).collect(), + "distributed_lock_quorum_rollback", + ); if !pending.is_empty() { Self::spawn_pending_cleanup( pending, @@ -933,7 +1075,10 @@ impl DistributedLock { if individual_locks.len() + pending.len() < required_quorum { let rollback_count = individual_locks.len(); - Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback"); + Self::spawn_release_cleanup( + individual_locks.iter().map(HeldLockEntry::release_entry).collect(), + "distributed_lock_quorum_rollback", + ); if !pending.is_empty() { Self::spawn_pending_cleanup( pending, @@ -967,7 +1112,10 @@ impl DistributedLock { } let rollback_count = individual_locks.len(); - Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback"); + Self::spawn_release_cleanup( + individual_locks.iter().map(HeldLockEntry::release_entry).collect(), + "distributed_lock_quorum_rollback", + ); let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"); if let Some(last_failure) = &last_failure { error.push_str("; last failure: "); @@ -1006,7 +1154,7 @@ fn record_lock_held_release(lock_type: LockType) { #[cfg(test)] mod tests { use super::{ - DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, is_remote_lock_rpc_failure, + DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, LockLostSignal, is_remote_lock_rpc_failure, should_warn_lock_failure, }; use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient}; @@ -1020,9 +1168,10 @@ mod tests { #[derive(Clone, Copy, Debug)] enum RefreshOutcome { - Alive, // Ok(true) refresh succeeded - NotFound, // Ok(false) remote already lost the lock (reclaimed / never held) - RpcError, // Err RPC jitter + Alive, // Ok(true) refresh succeeded + SlowAlive(Duration), // Ok(true) after a delayed response + NotFound, // Ok(false) remote already lost the lock (reclaimed / never held) + RpcError, // Err RPC jitter } /// Counting test client: acquires successfully and echoes back request.lock_id as @@ -1071,6 +1220,10 @@ mod tests { self.refresh_calls.fetch_add(1, Ordering::SeqCst); match self.outcome { RefreshOutcome::Alive => Ok(true), + RefreshOutcome::SlowAlive(delay) => { + tokio::time::sleep(delay).await; + Ok(true) + } RefreshOutcome::NotFound => Ok(false), RefreshOutcome::RpcError => Err(LockError::internal("refresh rpc jitter")), } @@ -1140,19 +1293,37 @@ mod tests { drop(guard); } - // A2 -- single client (degenerate path) spawns no heartbeat (matches localLockInstance). + // A2 -- a quorum-one distributed lease still requires renewal. #[tokio::test] - async fn single_client_guard_spawns_no_heartbeat() { + async fn single_client_guard_keeps_backend_lease_refreshed() { let (clients, counters) = counting_clients(&[RefreshOutcome::Alive]); let lock = DistributedLock::new("test".to_string(), clients, 1); let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner") .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let guard = lock.acquire_guard(&request).await.unwrap().unwrap(); - tokio::time::sleep(Duration::from_millis(90)).await; + let guard = lock + .acquire_guard(&request) + .await + .expect("single-client lock acquisition should not error") + .expect("single-client lock quorum should be reached"); + assert!( + guard.refresh_task.is_some(), + "single-client distributed guard must spawn a background task" + ); + tokio::time::timeout(Duration::from_secs(2), async { + while counters[0].load(Ordering::SeqCst) < 2 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("single-client distributed guard should renew before the test deadline"); - assert_eq!(counters[0].load(Ordering::SeqCst), 0, "single-client guard must not run a heartbeat"); + assert!( + counters[0].load(Ordering::SeqCst) >= 2, + "single-client distributed guard must renew beyond the original backend TTL" + ); + assert!(!guard.is_lock_lost(), "successful quorum-one refresh must preserve the lease"); drop(guard); } @@ -1170,7 +1341,11 @@ mod tests { .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let guard = lock.acquire_guard(&request).await.unwrap().unwrap(); + let guard = lock + .acquire_guard(&request) + .await + .expect("distributed lock acquisition should not error") + .expect("distributed lock quorum should be reached"); tokio::time::sleep(Duration::from_millis(50)).await; drop(guard); // should abort the heartbeat first let after_drop: Vec = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect(); @@ -1200,7 +1375,11 @@ mod tests { .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let guard = lock.acquire_guard(&request).await.unwrap().unwrap(); + let guard = lock + .acquire_guard(&request) + .await + .expect("distributed lock acquisition should not error") + .expect("distributed lock quorum should be reached"); tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick assert!( @@ -1214,6 +1393,52 @@ mod tests { drop(guard); } + #[tokio::test] + async fn lock_lost_notification_cannot_be_missed_after_waiter_registration() { + let signal = LockLostSignal::default(); + + tokio::time::timeout(Duration::from_secs(2), signal.notified_after_registration(|| signal.mark_lost())) + .await + .expect("loss between waiter registration and state recheck must still resolve"); + } + + #[tokio::test] + async fn expired_lease_is_fenced_without_waiting_for_monitor_scheduling() { + let signal = LockLostSignal::default(); + signal.set_valid_until(tokio::time::Instant::now()); + + assert!(signal.is_lost(), "an elapsed backend deadline must fence the owner immediately"); + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("an elapsed backend deadline must resolve lock-loss waiters immediately"); + } + + #[tokio::test] + async fn deadline_notification_permanently_fences_the_signal() { + let signal = LockLostSignal::default(); + signal.set_valid_until(tokio::time::Instant::now() + Duration::from_millis(10)); + + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("the backend deadline must resolve lock-loss waiters"); + + assert!( + signal.lost.load(Ordering::SeqCst), + "deadline notification must persist the lock-loss fence" + ); + } + + #[test] + fn expired_lease_cannot_be_revived_by_a_later_deadline() { + let signal = LockLostSignal::default(); + signal.set_valid_until(tokio::time::Instant::now()); + assert!(signal.is_lost(), "an elapsed lease must become permanently fenced"); + + signal.set_valid_until(tokio::time::Instant::now() + Duration::from_secs(60)); + + assert!(signal.is_lost(), "a later heartbeat must not revive an expired lease"); + } + // Phase 2 passthrough: NamespaceLockGuard::Standard must forward the distributed // guard's lock-loss signal so the write path can fence its commit on it. #[tokio::test] @@ -1229,7 +1454,11 @@ mod tests { .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let guard = lock.acquire_guard(&request).await.unwrap().unwrap(); + let guard = lock + .acquire_guard(&request) + .await + .expect("namespace lock acquisition should not error") + .expect("namespace lock quorum should be reached"); tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick -> lost let ns_guard = crate::namespace::NamespaceLockGuard::Standard(guard); @@ -1239,9 +1468,10 @@ mod tests { ); } - // A5 -- refresh RPC jitter (Err) is not counted as not_found; the lock is not declared lost. + // A5 -- refresh RPC jitter is tolerated within the lease, but an unconfirmed lease cannot + // remain valid after its backend TTL. #[tokio::test] - async fn heartbeat_rpc_error_not_counted_as_lock_lost() { + async fn heartbeat_rpc_error_expires_at_backend_ttl() { let (clients, _counters) = counting_clients(&[ RefreshOutcome::RpcError, RefreshOutcome::RpcError, @@ -1253,17 +1483,53 @@ mod tests { .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let guard = lock.acquire_guard(&request).await.unwrap().unwrap(); + let guard = lock + .acquire_guard(&request) + .await + .expect("distributed lock acquisition should not error") + .expect("distributed lock quorum should be reached"); tokio::time::sleep(Duration::from_millis(60)).await; assert!( !guard.is_lock_lost(), - "RPC jitter (Err) must NOT be counted as not_found; lock must not be declared lost" + "transient RPC jitter before the backend TTL must not immediately signal lock loss" + ); + + let signal = guard.lock_lost(); + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("unconfirmed leases must signal lock loss when their backend TTL expires"); + assert!(guard.is_lock_lost(), "the guard must fail closed once refresh quorum expires"); + drop(guard); + } + + // A6 -- a refresh call that remains in flight cannot keep an expired quorum alive. + #[tokio::test] + async fn heartbeat_slow_refresh_is_fenced_by_backend_ttl() { + let (clients, _counters) = + counting_clients(&[RefreshOutcome::SlowAlive(Duration::from_millis(200)), RefreshOutcome::Alive]); + let lock = DistributedLock::new("test".to_string(), clients, 2); + let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner") + .with_ttl(Duration::from_millis(120)) + .with_refresh_interval(Duration::from_millis(20)); + + let guard = lock + .acquire_guard(&request) + .await + .expect("distributed lock acquisition should not error") + .expect("distributed lock quorum should be reached"); + let signal = guard.lock_lost(); + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("an in-flight refresh must not extend an unconfirmed lease past its TTL"); + assert!( + guard.is_lock_lost(), + "slow refresh must fail closed at the last confirmed quorum deadline" ); drop(guard); } - // A6 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl + // A7 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl // (fixes clamp panic; aligns with the 50ms-ttl distributed path in namespace tests). #[tokio::test] async fn subsecond_ttl_guard_does_not_panic_and_skips_heartbeat() { @@ -1286,15 +1552,26 @@ mod tests { let request2 = LockRequest::new(ObjectKey::new("bucket", "object2"), LockType::Exclusive, "owner") .with_ttl(Duration::from_millis(50)) .with_refresh_interval(Duration::from_millis(50)); // interval >= ttl -> None - let guard2 = lock2.acquire_guard(&request2).await.unwrap().unwrap(); - tokio::time::sleep(Duration::from_millis(60)).await; + let guard2 = lock2 + .acquire_guard(&request2) + .await + .expect("subsecond lock acquisition should not error") + .expect("subsecond lock quorum should be reached"); + assert!( + guard2.refresh_task.is_none(), + "non-renewable subsecond guard must not spawn a background task" + ); + tokio::time::timeout(Duration::from_secs(2), guard2.lock_lost().notified()) + .await + .expect("non-renewable subsecond guard must signal loss at its backend TTL"); for c in &counters2 { - assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not spawn heartbeat"); + assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not refresh the backend lease"); } + assert!(guard2.is_lock_lost(), "interval>=ttl guard must fail closed after backend TTL"); drop(guard2); } - // A7 -- disarm() must also stop the heartbeat (public API gap). + // A8 -- disarm() must also stop the heartbeat (public API gap). #[tokio::test] async fn disarm_stops_heartbeat() { let (clients, counters) = counting_clients(&[ @@ -1308,7 +1585,11 @@ mod tests { .with_ttl(Duration::from_millis(120)) .with_refresh_interval(Duration::from_millis(20)); - let mut guard = lock.acquire_guard(&request).await.unwrap().unwrap(); + let mut guard = lock + .acquire_guard(&request) + .await + .expect("distributed lock acquisition should not error") + .expect("distributed lock quorum should be reached"); tokio::time::sleep(Duration::from_millis(50)).await; guard.disarm(); // should abort the heartbeat let after_disarm: Vec = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect(); diff --git a/crates/lock/src/namespace/mod.rs b/crates/lock/src/namespace/mod.rs index 51296134c..f0195bd46 100644 --- a/crates/lock/src/namespace/mod.rs +++ b/crates/lock/src/namespace/mod.rs @@ -128,6 +128,19 @@ impl NamespaceLockGuard { Self::Fast(_) => false, } } + + /// Resolves when a distributed guard loses refresh quorum. + /// + /// Local fast locks cannot lose distributed quorum, so their future remains pending. + pub async fn lock_lost_notified(&self) { + match self { + Self::Standard(guard) => { + let signal = guard.lock_lost(); + signal.notified().await; + } + Self::Fast(_) => std::future::pending().await, + } + } } /// Namespace lock for managing locks by resource namespaces diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 4322ea21a..f186922f1 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -640,6 +640,8 @@ pub struct ScannerMetrics { pub last_cycle_usage_saves: u64, #[serde(rename = "failed_cycles", default)] pub failed_cycles: u64, + #[serde(rename = "superseded_cycles", default)] + pub superseded_cycles: u64, #[serde(rename = "partial_cycles_unknown", default)] pub partial_cycles_unknown: u64, #[serde(rename = "partial_cycles_runtime", default)] @@ -820,6 +822,7 @@ impl ScannerMetrics { .saturating_add(other.last_cycle_replication_checks); self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves); self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles); + self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles); self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown); self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime); self.partial_cycles_objects = self.partial_cycles_objects.saturating_add(other.partial_cycles_objects); @@ -1862,6 +1865,7 @@ mod tests { last_cycle_replication_checks: 7, last_cycle_usage_saves: 8, failed_cycles: 1, + superseded_cycles: 2, partial_cycles_unknown: 2, partial_cycles_runtime: 3, partial_cycles_objects: 4, @@ -1933,6 +1937,7 @@ mod tests { last_cycle_replication_checks: 70, last_cycle_usage_saves: 80, failed_cycles: 10, + superseded_cycles: 20, partial_cycles_unknown: 20, partial_cycles_runtime: 30, partial_cycles_objects: 40, @@ -2014,6 +2019,7 @@ mod tests { assert_eq!(scanner.last_cycle_replication_checks, 77); assert_eq!(scanner.last_cycle_usage_saves, 88); assert_eq!(scanner.failed_cycles, 11); + assert_eq!(scanner.superseded_cycles, 22); assert_eq!(scanner.partial_cycles_unknown, 22); assert_eq!(scanner.partial_cycles_runtime, 33); assert_eq!(scanner.partial_cycles_objects, 44); diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index f8e2abc55..6a25ca524 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -44,8 +44,9 @@ use crate::metrics::schema::scanner::{ SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD, - SCANNER_PARTIAL_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, - SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD, + SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, + SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, + SCANNER_YIELD_EVERY_N_OBJECTS_MD, }; /// Scanner statistics. @@ -139,7 +140,7 @@ pub struct ScannerStats { pub current_cycle_usage_saves: u64, /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan pub current_scan_mode: u64, - /// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial + /// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded pub last_cycle_result: u64, /// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories pub last_cycle_partial_reason: u64, @@ -177,6 +178,8 @@ pub struct ScannerStats { pub last_cycle_usage_saves: u64, /// Number of scanner cycles that failed since server start pub failed_cycles: u64, + /// Number of scanner cycles superseded by concurrent namespace activity + pub superseded_cycles: u64, /// Number of scanner cycles stopped by runtime budget since server start pub partial_cycles: u64, /// Number of scanner cycles stopped by an unknown budget reason @@ -318,6 +321,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64), PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64), + PrometheusMetric::from_descriptor(&SCANNER_SUPERSEDED_CYCLES_MD, stats.superseded_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_MD, stats.partial_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_unknown as f64) .with_label("reason", "unknown"), @@ -405,6 +409,7 @@ mod tests { last_cycle_replication_checks: 12, last_cycle_usage_saves: 9, failed_cycles: 3, + superseded_cycles: 5, partial_cycles: 10, partial_cycles_unknown: 1, partial_cycles_runtime: 2, @@ -415,7 +420,7 @@ mod tests { let metrics = collect_scanner_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 68); + assert_eq!(metrics.len(), 69); let objects = metrics.iter().find(|m| m.value == 1000000.0); assert!(objects.is_some()); @@ -708,6 +713,11 @@ mod tests { .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name()); assert_eq!(failed_cycles.map(|m| m.value), Some(3.0)); + let superseded_cycles = metrics + .iter() + .find(|m| m.name == SCANNER_SUPERSEDED_CYCLES_MD.get_full_metric_name()); + assert_eq!(superseded_cycles.map(|m| m.value), Some(5.0)); + let partial_cycles = metrics .iter() .find(|m| m.name == SCANNER_PARTIAL_CYCLES_MD.get_full_metric_name()); @@ -743,7 +753,7 @@ mod tests { let stats = ScannerStats::default(); let metrics = collect_scanner_metrics(&stats); - assert_eq!(metrics.len(), 68); + assert_eq!(metrics.len(), 69); for metric in &metrics { assert_eq!(metric.value, 0.0); if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() { diff --git a/crates/obs/src/metrics/schema/entry/metric_name.rs b/crates/obs/src/metrics/schema/entry/metric_name.rs index 9c0cbf2a2..a240acb03 100644 --- a/crates/obs/src/metrics/schema/entry/metric_name.rs +++ b/crates/obs/src/metrics/schema/entry/metric_name.rs @@ -337,6 +337,7 @@ pub enum MetricName { ScannerLastCycleReplicationChecks, ScannerLastCycleUsageSaves, ScannerFailedCycles, + ScannerSupersededCycles, ScannerPartialCycles, ScannerPartialCyclesByReason, @@ -745,6 +746,7 @@ impl MetricName { Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(), Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(), Self::ScannerFailedCycles => "failed_cycles".to_string(), + Self::ScannerSupersededCycles => "superseded_cycles".to_string(), Self::ScannerPartialCycles => "partial_cycles".to_string(), Self::ScannerPartialCyclesByReason => "partial_cycles_by_reason".to_string(), diff --git a/crates/obs/src/metrics/schema/scanner.rs b/crates/obs/src/metrics/schema/scanner.rs index 7f3ac768a..b79ee6e15 100644 --- a/crates/obs/src/metrics/schema/scanner.rs +++ b/crates/obs/src/metrics/schema/scanner.rs @@ -416,7 +416,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock = LazyLock:: pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerLastCycleResult, - "Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial.", + "Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.", &[], subsystems::SCANNER, ) @@ -584,6 +584,15 @@ pub static SCANNER_FAILED_CYCLES_MD: LazyLock = LazyLock::new( ) }); +pub static SCANNER_SUPERSEDED_CYCLES_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::ScannerSupersededCycles, + "Total number of clean scanner cycles superseded by concurrent cluster, configuration, topology, or namespace activity since server start.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_PARTIAL_CYCLES_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerPartialCycles, diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index a0c6c19c4..9d6cec219 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -1135,6 +1135,7 @@ pub async fn collect_scanner_metric_stats() -> Option { last_cycle_replication_checks: metrics.last_cycle_replication_checks, last_cycle_usage_saves: metrics.last_cycle_usage_saves, failed_cycles: metrics.failed_cycles, + superseded_cycles: metrics.superseded_cycles, partial_cycles: metrics.partial_cycles, partial_cycles_unknown: metrics.partial_cycles_unknown, partial_cycles_runtime: metrics.partial_cycles_runtime, diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 0c3b8cb1d..6c7fc7fd5 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1076,9 +1076,20 @@ pub struct SignalServiceResponse { pub success: bool, #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint32, tag = "3")] + pub protocol_version: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerActivityRequest { + #[prost(bytes = "bytes", tag = "1")] + pub challenge: ::prost::bytes::Bytes, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(string, tag = "3")] + pub acknowledge_instance_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub acknowledge_dirty_usage_generation: u64, } -#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] -pub struct ScannerActivityRequest {} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerActivityResponse { #[prost(string, tag = "1")] @@ -1087,6 +1098,18 @@ pub struct ScannerActivityResponse { pub namespace_generation: u64, #[prost(uint64, tag = "3")] pub maintenance_generation: u64, + #[prost(uint32, tag = "4")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "5")] + pub topology_digest: ::prost::bytes::Bytes, + #[prost(bool, tag = "6")] + pub data_movement_active: bool, + #[prost(bytes = "bytes", tag = "7")] + pub response_proof: ::prost::bytes::Bytes, + #[prost(uint64, tag = "8")] + pub dirty_usage_generation: u64, + #[prost(bool, tag = "9")] + pub dirty_usage_pending: bool, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusRequest {} diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 669145d2b..985331d56 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -169,6 +169,7 @@ pub fn internode_rpc_max_message_size() -> usize { pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024; pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2; +pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1; pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0"; pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024; pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024; @@ -353,6 +354,230 @@ pub fn canonical_tier_mutation_rpc_response_body( Ok(body) } +/// Builds the stable byte representation authenticated for a scanner activity request. +pub fn canonical_scanner_activity_request_body( + request: &proto_gen::node_service::ScannerActivityRequest, +) -> Result, std::num::TryFromIntError> { + const DOMAIN: &[u8] = b"rustfs-scanner-activity-request-v1\0"; + + let challenge = request.challenge.as_ref(); + let acknowledge_instance_id = request.acknowledge_instance_id.as_bytes(); + let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + acknowledge_instance_id.len() + 4 + 8 * 3); + body.extend_from_slice(DOMAIN); + body.extend_from_slice(&request.protocol_version.to_be_bytes()); + body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes()); + body.extend_from_slice(challenge); + body.extend_from_slice(&u64::try_from(acknowledge_instance_id.len())?.to_be_bytes()); + body.extend_from_slice(acknowledge_instance_id); + body.extend_from_slice(&request.acknowledge_dirty_usage_generation.to_be_bytes()); + Ok(body) +} + +/// Builds the protocol-v4 byte representation authenticated for a scanner activity response. +pub fn canonical_scanner_activity_v4_response_body( + challenge: &[u8], + response: &proto_gen::node_service::ScannerActivityResponse, +) -> Result, std::num::TryFromIntError> { + const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v1\0"; + + let instance_id = response.instance_id.as_bytes(); + let topology_digest = response.topology_digest.as_ref(); + let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 5 + 1); + body.extend_from_slice(DOMAIN); + body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes()); + body.extend_from_slice(challenge); + body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes()); + body.extend_from_slice(instance_id); + body.extend_from_slice(&response.namespace_generation.to_be_bytes()); + body.extend_from_slice(&response.maintenance_generation.to_be_bytes()); + body.extend_from_slice(&response.protocol_version.to_be_bytes()); + body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes()); + body.extend_from_slice(topology_digest); + body.push(u8::from(response.data_movement_active)); + Ok(body) +} + +/// Builds the stable byte representation authenticated for a scanner activity response. +pub fn canonical_scanner_activity_response_body( + challenge: &[u8], + response: &proto_gen::node_service::ScannerActivityResponse, +) -> Result, std::num::TryFromIntError> { + const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v2\0"; + + let instance_id = response.instance_id.as_bytes(); + let topology_digest = response.topology_digest.as_ref(); + let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 6 + 2); + body.extend_from_slice(DOMAIN); + body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes()); + body.extend_from_slice(challenge); + body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes()); + body.extend_from_slice(instance_id); + body.extend_from_slice(&response.namespace_generation.to_be_bytes()); + body.extend_from_slice(&response.maintenance_generation.to_be_bytes()); + body.extend_from_slice(&response.protocol_version.to_be_bytes()); + body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes()); + body.extend_from_slice(topology_digest); + body.push(u8::from(response.data_movement_active)); + body.extend_from_slice(&response.dirty_usage_generation.to_be_bytes()); + body.push(u8::from(response.dirty_usage_pending)); + Ok(body) +} + +#[cfg(test)] +mod scanner_activity_tests { + use super::{ + canonical_scanner_activity_request_body, canonical_scanner_activity_response_body, + canonical_scanner_activity_v4_response_body, + proto_gen::node_service::{ScannerActivityRequest, ScannerActivityResponse}, + }; + + #[test] + fn canonical_scanner_activity_request_binds_every_field() { + let request = ScannerActivityRequest { + challenge: vec![1; 16].into(), + protocol_version: 5, + acknowledge_instance_id: "0123456789abcdef0123456789abcdef".to_string(), + acknowledge_dirty_usage_generation: 11, + }; + let baseline = canonical_scanner_activity_request_body(&request).expect("scanner activity request should encode"); + + let variants = [ + ScannerActivityRequest { + challenge: vec![2; 16].into(), + ..request.clone() + }, + ScannerActivityRequest { + protocol_version: 4, + ..request.clone() + }, + ScannerActivityRequest { + acknowledge_instance_id: "1123456789abcdef0123456789abcdef".to_string(), + ..request.clone() + }, + ScannerActivityRequest { + acknowledge_dirty_usage_generation: 12, + ..request + }, + ]; + for variant in variants { + assert_ne!( + baseline, + canonical_scanner_activity_request_body(&variant).expect("scanner activity request variant should encode") + ); + } + } + + #[test] + fn canonical_scanner_activity_response_binds_challenge_and_every_status_field() { + let response = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: 4, + topology_digest: vec![9; 32].into(), + data_movement_active: true, + response_proof: Vec::new().into(), + dirty_usage_generation: 11, + dirty_usage_pending: true, + }; + let baseline = + canonical_scanner_activity_response_body(&[1; 16], &response).expect("scanner activity response should encode"); + + let variants = [ + ScannerActivityResponse { + instance_id: "1123456789abcdef0123456789abcdef".to_string(), + ..response.clone() + }, + ScannerActivityResponse { + namespace_generation: 8, + ..response.clone() + }, + ScannerActivityResponse { + maintenance_generation: 4, + ..response.clone() + }, + ScannerActivityResponse { + protocol_version: 5, + ..response.clone() + }, + ScannerActivityResponse { + topology_digest: vec![8; 32].into(), + ..response.clone() + }, + ScannerActivityResponse { + data_movement_active: false, + ..response.clone() + }, + ScannerActivityResponse { + dirty_usage_generation: 12, + ..response.clone() + }, + ScannerActivityResponse { + dirty_usage_pending: false, + ..response.clone() + }, + ]; + for variant in variants { + assert_ne!( + baseline, + canonical_scanner_activity_response_body(&[1; 16], &variant) + .expect("scanner activity response variant should encode") + ); + } + assert_ne!( + baseline, + canonical_scanner_activity_response_body(&[2; 16], &response) + .expect("scanner activity response with a different challenge should encode") + ); + } + + #[test] + fn scanner_activity_v4_response_canonicalization_ignores_v5_fields() { + let response = ScannerActivityResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + namespace_generation: 7, + maintenance_generation: 3, + protocol_version: 4, + topology_digest: vec![9; 32].into(), + data_movement_active: true, + response_proof: Vec::new().into(), + dirty_usage_generation: 0, + dirty_usage_pending: false, + }; + let baseline = + canonical_scanner_activity_v4_response_body(&[1; 16], &response).expect("scanner activity v4 response should encode"); + let expected = [ + b"rustfs-scanner-activity-response-v1\0".as_slice(), + 16u64.to_be_bytes().as_slice(), + [1u8; 16].as_slice(), + 32u64.to_be_bytes().as_slice(), + b"0123456789abcdef0123456789abcdef".as_slice(), + 7u64.to_be_bytes().as_slice(), + 3u64.to_be_bytes().as_slice(), + 4u32.to_be_bytes().as_slice(), + 32u64.to_be_bytes().as_slice(), + [9u8; 32].as_slice(), + [1u8].as_slice(), + ] + .concat(); + assert_eq!( + baseline, expected, + "protocol v4 response bytes must remain stable during rolling upgrades" + ); + let extended = ScannerActivityResponse { + dirty_usage_generation: 11, + dirty_usage_pending: true, + ..response + }; + + assert_eq!( + baseline, + canonical_scanner_activity_v4_response_body(&[1; 16], &extended) + .expect("scanner activity v4 response should ignore v5 fields") + ); + } +} + #[cfg(test)] mod heal_control_tests { use super::{ diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 92b4e74a7..3a0fc66ed 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -756,14 +756,26 @@ message SignalServiceRequest { message SignalServiceResponse { bool success = 1; optional string error_info = 2; + uint32 protocol_version = 3; } -message ScannerActivityRequest {} +message ScannerActivityRequest { + bytes challenge = 1; + uint32 protocol_version = 2; + string acknowledge_instance_id = 3; + uint64 acknowledge_dirty_usage_generation = 4; +} message ScannerActivityResponse { string instance_id = 1; uint64 namespace_generation = 2; uint64 maintenance_generation = 3; + uint32 protocol_version = 4; + bytes topology_digest = 5; + bool data_movement_active = 6; + bytes response_proof = 7; + uint64 dirty_usage_generation = 8; + bool dirty_usage_pending = 9; } message BackgroundHealStatusRequest {} diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 103dccc2e..967cf8f02 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -19,7 +19,8 @@ use http::{HeaderMap, Version}; use pin_project_lite::pin_project; use reqwest::{Certificate, Client, Identity, Method, RequestBuilder}; use rustfs_io_metrics::internode_metrics::{ - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, + INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_OPERATION_WALK_DIR, }; use rustfs_tls_runtime::load_cert_bundle_der_bytes; use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize}; @@ -43,6 +44,7 @@ use tracing::{error, warn}; const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream"; const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir"; +const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner"; const HTTP_VERSION_09_LABEL: &str = "http/0.9"; const HTTP_VERSION_10_LABEL: &str = "http/1.0"; const HTTP_VERSION_11_LABEL: &str = "http/1.1"; @@ -1120,6 +1122,7 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> { READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM), PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM), WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR), + NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER), _ => None, } } @@ -1770,6 +1773,10 @@ mod tests { internode_rpc_operation(&format!("http://node:9000{WALK_DIR_PATH}?disk=d")), Some(INTERNODE_OPERATION_WALK_DIR) ); + assert_eq!( + internode_rpc_operation(&format!("http://node:9000{NS_SCANNER_PATH}?disk=d")), + Some(INTERNODE_OPERATION_NS_SCANNER) + ); assert_eq!(internode_rpc_operation("http://node:9000/rustfs/rpc/unknown"), None); assert_eq!( internode_rpc_operation("http://node:9000/rustfs/rpc/unknown?next=/rustfs/rpc/read_file_stream"), diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 95dcd1f3c..490a1d9c6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -32,6 +32,7 @@ workspace = true [dependencies] rustfs-config = { workspace = true, features = ["server-config-model"] } rustfs-common = { workspace = true } +rustfs-credentials = { workspace = true } rustfs-utils = { workspace = true } tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] } tracing = { workspace = true } @@ -43,6 +44,8 @@ futures = { workspace = true } time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] } chrono = { workspace = true, features = ["serde"] } rmp-serde = { workspace = true } +hmac = { workspace = true } +sha2 = { workspace = true } rustfs-filemeta = { workspace = true } tokio-util = { workspace = true, features = ["io", "compat"] } rustfs-ecstore = { workspace = true } @@ -52,11 +55,15 @@ rand = { workspace = true, features = ["serde"] } s3s = { workspace = true, features = ["minio"] } metrics = { workspace = true } rustfs-data-usage = { workspace = true } +uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] } +bytes.workspace = true +hex-simd.workspace = true [dev-dependencies] tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } serial_test = { workspace = true } temp-env = { workspace = true } +tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index bf5fa83ca..0812e1de7 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -13,7 +13,7 @@ // limitations under the License. use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, ser::SerializeMap}; use std::{ collections::{HashMap, HashSet}, future::Future, @@ -27,34 +27,37 @@ use rustfs_common::heal_channel::HealScanMode; #[cfg(test)] use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ - BucketTargetUsageInfo, BucketUsageInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, hash_path, + BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageEntry, DataUsageHash, DataUsageHashMap, + DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, hash_path, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; -use tracing::warn; +use tracing::{debug, warn}; use crate::ScannerObjectIO; +use crate::storage_api::owner::HTTPPreconditions; use crate::{ BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config, - storageclass, + save_config_with_preconditions, storageclass, }; // Data usage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; -const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; - const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; const DATA_USAGE_CACHE_SAVE_RETRIES: u32 = 2; const DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX: u64 = 5; const DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES: u32 = 0; +const DATA_USAGE_CACHE_SAVE_RETRY_BACKOFF_MAX: Duration = Duration::from_millis(350); +const DATA_USAGE_CACHE_PERSISTENCE_MARGIN: Duration = Duration::from_secs(5); const METRIC_CACHE_SAVE_ATTEMPT_TOTAL: &str = "rustfs_scanner_cache_save_attempt_total"; const METRIC_CACHE_SAVE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cache_save_timeout_total"; const METRIC_CACHE_SAVE_RETRY_TOTAL: &str = "rustfs_scanner_cache_save_retry_total"; const METRIC_CACHE_SAVE_DURATION_SECONDS: &str = "rustfs_scanner_cache_save_duration_seconds"; +const METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL: &str = "rustfs_scanner_cache_backup_revision_failure_total"; const LOG_COMPONENT_SCANNER: &str = "scanner"; const LOG_SUBSYSTEM_CACHE: &str = "cache"; const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state"; @@ -63,12 +66,114 @@ static CACHE_SAVE_METRICS_ONCE: Once = Once::new(); pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1; +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum DataUsageCacheRevision { + Missing, + Etag(String), +} + +impl DataUsageCacheRevision { + pub(crate) fn preconditions(&self) -> HTTPPreconditions { + match self { + Self::Missing => HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + Self::Etag(etag) => HTTPPreconditions { + if_match: Some(etag.clone()), + ..Default::default() + }, + } + } +} + +pub(crate) async fn read_config_with_revision( + store: Arc, + path: &str, +) -> StorageResult<(Option>, DataUsageCacheRevision)> { + match store + .get_object_reader( + RUSTFS_META_BUCKET, + path, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(mut reader) => { + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag) + .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?; + Ok((Some(reader.read_all().await?), revision)) + } + Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { + Ok((None, DataUsageCacheRevision::Missing)) + } + Err(err) => Err(err), + } +} + +#[derive(Clone, Debug)] +pub(crate) struct DataUsageCacheRevisions { + main: DataUsageCacheRevision, + backup: Option, +} + +enum DataUsageCacheLoadAttempt { + Loaded { + cache: Box, + revision: Option, + }, + Missing { + revision: Option, + }, + Corrupt { + revision: Option, + }, + Retryable(Error), +} + +struct DataUsageCacheLoadResult { + cache: DataUsageCache, + main_revision: DataUsageCacheRevision, + backup_revision: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DataUsageCacheLoadState { + Loaded, + Missing, + Corrupt, + Retryable, +} + +impl DataUsageCacheLoadAttempt { + fn revision(&self) -> Option { + match self { + Self::Loaded { revision, .. } | Self::Missing { revision } | Self::Corrupt { revision } => revision.clone(), + Self::Retryable(_) => None, + } + } +} + // Data usage paths (computed at runtime) pub static DATA_USAGE_BUCKET: LazyLock = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{BUCKET_META_PREFIX}")); pub static DATA_USAGE_OBJ_NAME_PATH: LazyLock = - LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJ_NAME}")); + LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJECT_NAME}")); + +pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock = + LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{LEGACY_DATA_USAGE_OBJECT_NAME}")); pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}")); @@ -174,15 +279,16 @@ pub struct SizeSummary { impl SizeSummary { pub fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) { if oi.delete_marker { - self.delete_markers += 1; + self.delete_markers = self.delete_markers.saturating_add(1); return; } if oi.version_id.is_some_and(|v| !v.is_nil()) && size == actual_size { - self.versions += 1; + self.versions = self.versions.saturating_add(1); } - self.total_size += if size > 0 { size as usize } else { 0 }; + let size = usize::try_from(size.max(0)).unwrap_or(usize::MAX); + self.total_size = self.total_size.saturating_add(size); if oi.transitioned_object.free_version { return; @@ -262,6 +368,23 @@ pub struct DataUsageEntryInfo { pub entry: DataUsageEntry, } +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(deny_unknown_fields)] +pub struct DataUsageCacheSource { + pub pool_index: usize, + pub set_index: usize, +} + +impl DataUsageCacheSource { + pub const fn new(pool_index: usize, set_index: usize) -> Self { + Self { pool_index, set_index } + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(transparent)] +pub struct DataUsageScanPlanDigest(pub [u8; 32]); + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum PendingScannerHealKind { @@ -288,7 +411,7 @@ pub struct PendingScannerHeal { } /// Data usage cache info -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] pub struct DataUsageCacheInfo { pub name: String, pub next_cycle: u64, @@ -306,6 +429,41 @@ pub struct DataUsageCacheInfo { pub pending_heals: Vec, #[serde(default)] pub object_lock: Option>, + #[serde(default)] + pub leader_epoch: u64, + #[serde(default)] + pub source: Option, + #[serde(default)] + pub snapshot_complete: bool, + #[serde(default)] + pub scan_plan_digest: Option, +} + +impl Serialize for DataUsageCacheInfo { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // Keep this metadata map-encoded so older readers can ignore fields + // appended by newer scanner versions during rolling upgrades. + let mut state = serializer.serialize_map(Some(15))?; + state.serialize_entry("name", &self.name)?; + state.serialize_entry("next_cycle", &self.next_cycle)?; + state.serialize_entry("leader_epoch", &self.leader_epoch)?; + state.serialize_entry("last_update", &self.last_update)?; + state.serialize_entry("skip_healing", &self.skip_healing)?; + state.serialize_entry("lifecycle", &self.lifecycle)?; + state.serialize_entry("replication", &self.replication)?; + state.serialize_entry("failed_objects", &self.failed_objects)?; + state.serialize_entry("scan_resume_after", &self.scan_resume_after)?; + state.serialize_entry("scan_checkpoint", &self.scan_checkpoint)?; + state.serialize_entry("pending_heals", &self.pending_heals)?; + state.serialize_entry("object_lock", &self.object_lock)?; + state.serialize_entry("source", &self.source)?; + state.serialize_entry("snapshot_complete", &self.snapshot_complete)?; + state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?; + state.end() + } } /// Data usage cache @@ -315,7 +473,54 @@ pub struct DataUsageCache { pub cache: HashMap, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DataUsageCachePrepareOutcome { + Reused, + Reset, + RejectedNewerCycle, + RejectedNewerLeader, +} + impl DataUsageCache { + pub(crate) fn prepare_for_scan( + &mut self, + name: &str, + next_cycle: u64, + leader_epoch: u64, + source: DataUsageCacheSource, + scan_plan_digest: DataUsageScanPlanDigest, + require_source: bool, + ) -> DataUsageCachePrepareOutcome { + if self.info.next_cycle > next_cycle { + return DataUsageCachePrepareOutcome::RejectedNewerCycle; + } + if self.info.leader_epoch > leader_epoch { + return DataUsageCachePrepareOutcome::RejectedNewerLeader; + } + + let source_matches = self.info.source == Some(source); + let plan_matches = self.info.scan_plan_digest == Some(scan_plan_digest); + let reusable = self.info.name == name + && self.info.leader_epoch == leader_epoch + && plan_matches + && (source_matches || (!require_source && self.info.source.is_none())); + if !reusable { + *self = Self::default(); + self.info.name = name.to_string(); + } + + self.info.next_cycle = next_cycle; + self.info.leader_epoch = leader_epoch; + self.info.source = Some(source); + self.info.scan_plan_digest = Some(scan_plan_digest); + self.info.snapshot_complete = false; + if reusable { + DataUsageCachePrepareOutcome::Reused + } else { + DataUsageCachePrepareOutcome::Reset + } + } + fn ensure_cache_save_metrics_registered() { CACHE_SAVE_METRICS_ONCE.call_once(|| { describe_counter!( @@ -370,6 +575,39 @@ impl DataUsageCache { self.flatten_with_guard(root, &mut visited, 0) } + pub(crate) fn checked_flatten(&self, path: &str) -> Option { + let root_key = hash_path(path).key(); + let root = self.cache.get(&root_key)?; + let mut visited = HashSet::from([root_key]); + let mut pending = root.children.iter().map(|child| (child.clone(), 1usize)).collect::>(); + let mut flattened = DataUsageEntry::default(); + let mut root_entry = root.clone(); + root_entry.children.clear(); + if !flattened.checked_merge(&root_entry) { + return None; + } + flattened.compacted = root.compacted; + + while let Some((key, depth)) = pending.pop() { + if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key.clone()) { + return None; + } + let entry = self.cache.get(&key)?; + if depth == MAX_DATA_USAGE_CACHE_DEPTH && !entry.children.is_empty() { + return None; + } + pending.extend(entry.children.iter().map(|child| (child.clone(), depth + 1))); + + let mut child_entry = entry.clone(); + child_entry.children.clear(); + if !flattened.checked_merge(&child_entry) { + return None; + } + } + + Some(flattened) + } + fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet, depth: usize) -> DataUsageEntry { let mut root = root.clone(); if depth >= MAX_DATA_USAGE_CACHE_DEPTH { @@ -703,65 +941,169 @@ impl DataUsageCache { /// The loader is optimistic and has no locking, but tries 5 times before giving up. /// If the object is not found, a nil error with empty data usage cache is returned. pub async fn load(&mut self, store: Arc, name: &str) -> StorageResult<()> { - // By default, empty data usage cache - *self = DataUsageCache::default(); - - // Caches are read+written without locks - let mut retries = 0; - while retries < 5 { - let (should_retry, cache_opt, result) = Self::try_load_inner(store.clone(), name, Duration::from_secs(60)).await; - result?; - if let Some(cache) = cache_opt { - *self = cache; - return Ok(()); - } - if !should_retry { - break; - } - - // Try backup file - let backup_name = format!("{name}.bkp"); - let (backup_retry, backup_cache_opt, backup_result) = - Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await; - if backup_result.is_err() { - // Error loading backup, continue retry - } else if let Some(cache) = backup_cache_opt { - // Only return when we have valid data from the backup - *self = cache; - return Ok(()); - } else if !backup_retry { - // Backup not found and not retryable - break; - } - - retries += 1; - // Random sleep between 0 and 1 second - let sleep_ms: u64 = rand::random::() % 1000; - sleep(Duration::from_millis(sleep_ms)).await; - } - - if retries == 5 { - warn!( - target: "rustfs::scanner::data_usage", - event = EVENT_SCANNER_CACHE_LOAD_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_CACHE, - cache_name = %name, - retries, - state = "max_retries_reached", - "Scanner cache load reached retry limit" - ); - } - + let loaded = Self::load_cache(store, name).await?; + *self = loaded.cache; Ok(()) } - // Inner load function that attempts to load from a specific path - // Returns (should_retry, cache_option, error_option) + + pub(crate) async fn load_with_revisions( + &mut self, + store: Arc, + name: &str, + ) -> StorageResult { + let backup_name = format!("{name}.bkp"); + let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]); + let loaded = Self::load_cache(store.clone(), name).await?; + let backup = match loaded.backup_revision { + Some(revision) => Some(revision), + None => match Self::revision_for_path(store, &backup_path).await { + Ok(revision) => Some(revision), + Err(err) => { + counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1); + debug!( + target: "rustfs::scanner::data_usage", + event = EVENT_SCANNER_CACHE_LOAD_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_CACHE, + cache_name = %name, + backup_path = %backup_path, + state = "backup_revision_unavailable", + error = %err, + "Scanner cache backup revision lookup failed" + ); + None + } + }, + }; + let main = loaded.main_revision; + *self = loaded.cache; + + Ok(DataUsageCacheRevisions { main, backup }) + } + + async fn load_cache(store: Arc, name: &str) -> StorageResult { + let mut last_retryable = None; + + for attempt in 0..5 { + let main_attempt = Self::try_load_inner(store.clone(), name, Duration::from_secs(60)).await?; + let main_revision = main_attempt.revision(); + let main_state = match main_attempt { + DataUsageCacheLoadAttempt::Loaded { + cache, + revision: Some(main_revision), + } => { + return Ok(DataUsageCacheLoadResult { + cache: *cache, + main_revision, + backup_revision: None, + }); + } + DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => { + last_retryable = Some(Error::other(format!("scanner cache object has no revision: {name}"))); + DataUsageCacheLoadState::Retryable + } + DataUsageCacheLoadAttempt::Missing { .. } => DataUsageCacheLoadState::Missing, + DataUsageCacheLoadAttempt::Corrupt { .. } => DataUsageCacheLoadState::Corrupt, + DataUsageCacheLoadAttempt::Retryable(err) => { + last_retryable = Some(err); + DataUsageCacheLoadState::Retryable + } + }; + if main_state == DataUsageCacheLoadState::Retryable { + if attempt < 4 { + let sleep_ms: u64 = rand::random::() % 1000; + sleep(Duration::from_millis(sleep_ms)).await; + } + continue; + } + + let backup_name = format!("{name}.bkp"); + let backup_attempt = Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await?; + let backup_revision = backup_attempt.revision(); + let backup_state = match backup_attempt { + DataUsageCacheLoadAttempt::Loaded { + cache, + revision: Some(backup_revision), + } => { + if matches!(main_state, DataUsageCacheLoadState::Missing | DataUsageCacheLoadState::Corrupt) { + let main_revision = main_revision.ok_or_else(|| { + Error::other(format!("scanner cache main revision is unavailable while loading backup: {name}")) + })?; + return Ok(DataUsageCacheLoadResult { + cache: *cache, + main_revision, + backup_revision: Some(backup_revision), + }); + } + DataUsageCacheLoadState::Loaded + } + DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => { + last_retryable = Some(Error::other(format!("scanner cache backup object has no revision: {backup_name}"))); + DataUsageCacheLoadState::Retryable + } + DataUsageCacheLoadAttempt::Missing { .. } => DataUsageCacheLoadState::Missing, + DataUsageCacheLoadAttempt::Corrupt { .. } => DataUsageCacheLoadState::Corrupt, + DataUsageCacheLoadAttempt::Retryable(err) => { + last_retryable = Some(err); + DataUsageCacheLoadState::Retryable + } + }; + + match (main_state, backup_state) { + (DataUsageCacheLoadState::Missing, DataUsageCacheLoadState::Missing) => { + return Ok(DataUsageCacheLoadResult { + cache: DataUsageCache::default(), + main_revision: main_revision + .ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?, + backup_revision, + }); + } + (DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing) + | (DataUsageCacheLoadState::Missing, DataUsageCacheLoadState::Corrupt) + | (DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Corrupt) => { + warn!( + target: "rustfs::scanner::data_usage", + event = EVENT_SCANNER_CACHE_LOAD_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_CACHE, + cache_name = %name, + state = "corrupt_cache_rebuild", + "Scanner cache is corrupt and will be rebuilt" + ); + return Ok(DataUsageCacheLoadResult { + cache: DataUsageCache::default(), + main_revision: main_revision + .ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?, + backup_revision, + }); + } + _ => {} + } + + if attempt < 4 { + let sleep_ms: u64 = rand::random::() % 1000; + sleep(Duration::from_millis(sleep_ms)).await; + } + } + + warn!( + target: "rustfs::scanner::data_usage", + event = EVENT_SCANNER_CACHE_LOAD_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_CACHE, + cache_name = %name, + retries = 5, + state = "max_retries_reached", + "Scanner cache load reached retry limit" + ); + Err(last_retryable.unwrap_or_else(|| Error::other(format!("scanner cache could not be loaded: {name}")))) + } + async fn try_load_inner( store: Arc, load_name: &str, timeout_duration: Duration, - ) -> (bool, Option, StorageResult<()>) { + ) -> StorageResult { // Abandon if more than time.Minute, so we don't hold up scanner. // drive timeout by default is 2 minutes, we do not need to wait longer. let load_fut = async { @@ -781,16 +1123,18 @@ impl DataUsageCache { .await { Ok(mut reader) => { + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag); match reader.read_all().await { - Ok(data) => { - match DataUsageCache::unmarshal(&data) { - Ok(cache) => Ok(Some(cache)), - Err(_) => { - // Deserialization failed, but we got data - Ok(None) - } - } - } + Ok(data) => match DataUsageCache::unmarshal(&data) { + Ok(cache) => Ok((Some(cache), revision, false)), + Err(_) => Ok((None, revision, true)), + }, Err(e) => { // Read error Err(e) @@ -816,8 +1160,8 @@ impl DataUsageCache { { Ok(mut reader) => match reader.read_all().await { Ok(data) => match DataUsageCache::unmarshal(&data) { - Ok(cache) => Ok(Some(cache)), - Err(_) => Ok(None), + Ok(cache) => Ok((Some(cache), Some(DataUsageCacheRevision::Missing), false)), + Err(_) => Ok((None, Some(DataUsageCacheRevision::Missing), true)), }, Err(e) => Err(e), }, @@ -827,11 +1171,11 @@ impl DataUsageCache { | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_) => { // Object not found in both locations - Ok(None) + Ok((None, Some(DataUsageCacheRevision::Missing), false)) } Error::ErasureReadQuorum => { // InsufficientReadQuorum - retry - Ok(None) + Err(Error::ErasureReadQuorum) } _ => { // Other storage errors - retry @@ -839,7 +1183,7 @@ impl DataUsageCache { inner_err, Error::FaultyDisk | Error::DiskFull | Error::StorageFull | Error::SlowDown ) { - return Ok(None); + return Err(inner_err); } Err(inner_err) } @@ -848,12 +1192,12 @@ impl DataUsageCache { } Error::ErasureReadQuorum => { // InsufficientReadQuorum - retry - Ok(None) + Err(Error::ErasureReadQuorum) } _ => { // Other storage errors - retry if matches!(err, Error::FaultyDisk | Error::DiskFull | Error::StorageFull | Error::SlowDown) { - return Ok(None); + return Err(err); } Err(err) } @@ -863,29 +1207,41 @@ impl DataUsageCache { }; match timeout(timeout_duration, load_fut).await { - Ok(result) => match result { - Ok(Some(cache)) => (false, Some(cache), Ok(())), - Ok(None) => { - // Not found or deserialization failed - check if we should retry - // For now, we don't retry on not found - (false, None, Ok(())) - } - Err(e) => { - // Check if it's a retryable error - if matches!( - e, - Error::ErasureReadQuorum | Error::FaultyDisk | Error::DiskFull | Error::StorageFull | Error::SlowDown - ) { - (true, None, Ok(())) - } else { - (false, None, Err(e)) - } - } - }, - Err(_) => { - // Timeout - retry - (true, None, Ok(())) + Ok(Ok((Some(cache), revision, _))) => Ok(DataUsageCacheLoadAttempt::Loaded { + cache: Box::new(cache), + revision, + }), + Ok(Ok((None, revision, true))) => Ok(DataUsageCacheLoadAttempt::Corrupt { revision }), + Ok(Ok((None, revision, false))) => Ok(DataUsageCacheLoadAttempt::Missing { revision }), + Ok(Err(err)) => Ok(DataUsageCacheLoadAttempt::Retryable(err)), + Err(_) => Ok(DataUsageCacheLoadAttempt::Retryable(Error::other("scanner cache load timed out"))), + } + } + + async fn revision_for_path(store: Arc, path: &str) -> StorageResult { + match store + .get_object_reader( + RUSTFS_META_BUCKET, + path, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader + .object_info + .etag + .filter(|etag| !etag.is_empty()) + .map(DataUsageCacheRevision::Etag) + .ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))), + Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { + Ok(DataUsageCacheRevision::Missing) } + Err(err) => Err(err), } } @@ -893,6 +1249,14 @@ impl DataUsageCache { crate::runtime_config::scanner_cache_save_timeout() } + pub(crate) fn persistence_timeout() -> Duration { + Self::cache_save_timeout() + .saturating_mul(DATA_USAGE_CACHE_SAVE_RETRIES + 1) + .saturating_add(DATA_USAGE_CACHE_SAVE_RETRY_BACKOFF_MAX) + .saturating_add(Self::backup_cache_save_timeout(Self::cache_save_timeout())) + .saturating_add(DATA_USAGE_CACHE_PERSISTENCE_MARGIN) + } + fn backup_cache_save_timeout(timeout_duration: Duration) -> Duration { timeout_duration.min(Duration::from_secs(DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX)) } @@ -913,7 +1277,13 @@ impl DataUsageCache { fn should_retry_save_error(err: &StorageError) -> bool { // Usage-cache files are best-effort scanner checkpoints. Retrying namespace // lock failures immediately only adds more lock traffic to the same hot object. - !matches!(err, StorageError::Lock(_) | StorageError::NamespaceLockQuorumUnavailable { .. }) + !matches!( + err, + StorageError::Lock(_) + | StorageError::NamespaceLockQuorumUnavailable { .. } + | StorageError::PreconditionFailed + | StorageError::ObjectNotFound(_, _) + ) } async fn retry_save_op( @@ -968,37 +1338,113 @@ impl DataUsageCache { buf: &[u8], timeout_duration: Duration, max_retries: u32, + revision: Option, ) -> StorageResult<()> { Self::ensure_cache_save_metrics_registered(); let path_type = Self::cache_path_type(path); let path = path.to_string(); - Self::retry_save_op(path_type, timeout_duration, max_retries, move || { + let save_result = Self::retry_save_op(path_type, timeout_duration, max_retries, || { let store_clone = store.clone(); let path_clone = path.clone(); let buf_clone = buf.to_vec(); + let revision = revision.clone(); async move { - save_config(store_clone, &path_clone, buf_clone).await?; + if let Some(revision) = revision { + save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?; + } else { + save_config(store_clone, &path_clone, buf_clone).await?; + } Ok::<(), StorageError>(()) } }) - .await + .await; + let Err(save_err) = save_result else { + return Ok(()); + }; + + for attempt in 0..=max_retries { + let reconcile = timeout(timeout_duration, async { + let mut reader = store + .get_object_reader( + RUSTFS_META_BUCKET, + &path, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await?; + Ok::(reader.read_all().await? == buf) + }) + .await; + if matches!(reconcile, Ok(Ok(true))) { + Self::record_save_attempt(path_type, "reconciled", Duration::ZERO); + return Ok(()); + } + if matches!(reconcile, Ok(Ok(false))) { + break; + } + if attempt < max_retries { + sleep(Duration::from_millis(50_u64 * (u64::from(attempt) + 1))).await; + } + } + + Err(save_err) } pub async fn save(&self, store: Arc, name: &str) -> StorageResult<()> { + self.save_inner(store, name, None).await + } + + pub(crate) async fn save_with_revisions( + &self, + store: Arc, + name: &str, + revisions: &DataUsageCacheRevisions, + ) -> StorageResult<()> { + self.save_inner(store, name, Some(revisions)).await + } + + async fn save_inner( + &self, + store: Arc, + name: &str, + revisions: Option<&DataUsageCacheRevisions>, + ) -> StorageResult<()> { let mut buf = Vec::new(); self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?; let timeout_duration = Self::cache_save_timeout(); let path = path_join_buf(&[BUCKET_META_PREFIX, name]); - Self::save_path_with_retry(store.clone(), &path, &buf, timeout_duration, DATA_USAGE_CACHE_SAVE_RETRIES).await?; + Self::save_path_with_retry( + store.clone(), + &path, + &buf, + timeout_duration, + DATA_USAGE_CACHE_SAVE_RETRIES, + revisions.map(|revisions| revisions.main.clone()), + ) + .await?; let backup_name = format!("{name}.bkp"); let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]); let backup_timeout_duration = Self::backup_cache_save_timeout(timeout_duration); - if let Err(e) = - Self::save_path_with_retry(store, &backup_path, &buf, backup_timeout_duration, DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES) - .await + let backup_revision = revisions.and_then(|revisions| revisions.backup.clone()); + if revisions.is_some() && backup_revision.is_none() { + return Ok(()); + } + if let Err(e) = Self::save_path_with_retry( + store, + &backup_path, + &buf, + backup_timeout_duration, + DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES, + backup_revision, + ) + .await { warn!( target: "rustfs::scanner::data_usage", @@ -1098,27 +1544,27 @@ impl SizeSummary { /// Add another SizeSummary to this one pub fn add(&mut self, other: &SizeSummary) { - self.total_size += other.total_size; - self.versions += other.versions; - self.delete_markers += other.delete_markers; - self.replicated_size += other.replicated_size; - self.replicated_count += other.replicated_count; - self.pending_size += other.pending_size; - self.failed_size += other.failed_size; - self.replica_size += other.replica_size; - self.replica_count += other.replica_count; - self.pending_count += other.pending_count; - self.failed_count += other.failed_count; + self.total_size = self.total_size.saturating_add(other.total_size); + self.versions = self.versions.saturating_add(other.versions); + self.delete_markers = self.delete_markers.saturating_add(other.delete_markers); + self.replicated_size = self.replicated_size.saturating_add(other.replicated_size); + self.replicated_count = self.replicated_count.saturating_add(other.replicated_count); + self.pending_size = self.pending_size.saturating_add(other.pending_size); + self.failed_size = self.failed_size.saturating_add(other.failed_size); + self.replica_size = self.replica_size.saturating_add(other.replica_size); + self.replica_count = self.replica_count.saturating_add(other.replica_count); + self.pending_count = self.pending_count.saturating_add(other.pending_count); + self.failed_count = self.failed_count.saturating_add(other.failed_count); // Merge replication target stats for (target, stats) in &other.repl_target_stats { let entry = self.repl_target_stats.entry(target.clone()).or_default(); - entry.replicated_size += stats.replicated_size; - entry.replicated_count += stats.replicated_count; - entry.pending_size += stats.pending_size; - entry.failed_size += stats.failed_size; - entry.pending_count += stats.pending_count; - entry.failed_count += stats.failed_count; + entry.replicated_size = entry.replicated_size.saturating_add(stats.replicated_size); + entry.replicated_count = entry.replicated_count.saturating_add(stats.replicated_count); + entry.pending_size = entry.pending_size.saturating_add(stats.pending_size); + entry.failed_size = entry.failed_size.saturating_add(stats.failed_size); + entry.pending_count = entry.pending_count.saturating_add(stats.pending_count); + entry.failed_count = entry.failed_count.saturating_add(stats.failed_count); } } } @@ -1126,10 +1572,489 @@ impl SizeSummary { #[cfg(test)] mod tests { use super::*; + use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO}; + use crate::{ScannerGetObjectReader, ScannerPutObjReader}; use serde_json::Value; + use std::io::Cursor; + use std::pin::Pin; use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::task::{Context, Poll}; use temp_env::{with_var, with_var_unset}; + use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; + use tokio::sync::Mutex; + + const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([3; 32]); + + #[derive(Debug, PartialEq, Eq)] + struct CachePutRecord { + object: String, + if_match: Option, + if_none_match: Option, + } + + #[derive(Debug)] + struct BackupFallbackStore { + backup: Vec, + recovered_main: Vec, + main_reads: AtomicUsize, + recover_main_revision: AtomicBool, + backup_reads: Mutex, + puts: Mutex>, + } + + impl BackupFallbackStore { + fn new(backup: Vec, recover_main_revision: bool) -> Self { + Self { + backup, + recovered_main: Vec::new(), + main_reads: AtomicUsize::new(0), + recover_main_revision: AtomicBool::new(recover_main_revision), + backup_reads: Mutex::new(0), + puts: Mutex::new(Vec::new()), + } + } + + fn with_recovered_main(backup: Vec, recovered_main: Vec) -> Self { + Self { + backup, + recovered_main, + main_reads: AtomicUsize::new(0), + recover_main_revision: AtomicBool::new(true), + backup_reads: Mutex::new(0), + puts: Mutex::new(Vec::new()), + } + } + + fn reader(data: Vec, etag: &str) -> ScannerGetObjectReader { + ScannerGetObjectReader { + stream: Box::new(Cursor::new(data)), + object_info: ObjectInfo { + etag: Some(etag.to_string()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + } + } + } + + #[derive(Clone, Debug)] + enum CacheReadBody { + Bytes(Vec), + PrefixThenError(Vec), + } + + #[derive(Debug)] + struct PrefixThenErrorReader { + prefix: Cursor>, + failed: bool, + } + + impl AsyncRead for PrefixThenErrorReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.prefix.position() < u64::try_from(self.prefix.get_ref().len()).unwrap_or(u64::MAX) { + return Pin::new(&mut self.prefix).poll_read(cx, buf); + } + if !self.failed { + self.failed = true; + return Poll::Ready(Err(std::io::Error::other("injected cache body read failure"))); + } + Poll::Ready(Ok(())) + } + } + + #[derive(Debug)] + struct CacheReadStore { + main: CacheReadBody, + backup: Option>, + puts: AtomicUsize, + } + + #[derive(Debug, Default)] + struct AmbiguousCacheCommitStore { + data: Mutex>>, + puts: AtomicUsize, + } + + impl CacheReadStore { + fn new(main: CacheReadBody, backup: Option>) -> Self { + Self { + main, + backup, + puts: AtomicUsize::new(0), + } + } + + fn reader(body: CacheReadBody, etag: &str) -> ScannerGetObjectReader { + let stream: Box = match body { + CacheReadBody::Bytes(data) => Box::new(Cursor::new(data)), + CacheReadBody::PrefixThenError(prefix) => Box::new(PrefixThenErrorReader { + prefix: Cursor::new(prefix), + failed: false, + }), + }; + ScannerGetObjectReader { + stream, + object_info: ObjectInfo { + etag: Some(etag.to_string()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + } + } + } + + #[async_trait::async_trait] + impl ObjectIO for CacheReadStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = ScannerGetObjectReader; + type PutObjectReader = ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> StorageResult { + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + + let main_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]); + let backup_path = format!("{main_path}.bkp"); + if object == main_path { + return Ok(Self::reader(self.main.clone(), "main-etag")); + } + if object == backup_path { + return self + .backup + .clone() + .map(|data| Self::reader(CacheReadBody::Bytes(data), "backup-etag")) + .ok_or(Error::FileNotFound); + } + Err(Error::FileNotFound) + } + + async fn put_object( + &self, + _bucket: &str, + _object: &str, + _data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> StorageResult { + self.puts.fetch_add(1, Ordering::SeqCst); + Ok(ObjectInfo::default()) + } + } + + #[async_trait::async_trait] + impl ObjectIO for AmbiguousCacheCommitStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = ScannerGetObjectReader; + type PutObjectReader = ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> StorageResult { + let expected_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]); + if bucket != RUSTFS_META_BUCKET || object != expected_path { + return Err(Error::FileNotFound); + } + let data = self.data.lock().await.clone().ok_or(Error::FileNotFound)?; + Ok(CacheReadStore::reader(CacheReadBody::Bytes(data), "committed-etag")) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> StorageResult { + let expected_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]); + if bucket != RUSTFS_META_BUCKET || object != expected_path { + return Err(Error::FileNotFound); + } + let mut bytes = Vec::new(); + data.stream.read_to_end(&mut bytes).await?; + *self.data.lock().await = Some(bytes); + self.puts.fetch_add(1, Ordering::SeqCst); + Err(StorageError::PreconditionFailed) + } + } + + #[async_trait::async_trait] + impl ObjectIO for BackupFallbackStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = ScannerGetObjectReader; + type PutObjectReader = ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> StorageResult { + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + + let main_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]); + let backup_path = format!("{main_path}.bkp"); + if object == main_path { + let read = self.main_reads.fetch_add(1, Ordering::SeqCst); + if read == 0 || !self.recover_main_revision.load(Ordering::SeqCst) { + return Err(Error::ErasureReadQuorum); + } + return Ok(Self::reader(self.recovered_main.clone(), "main-etag")); + } + if object == backup_path { + *self.backup_reads.lock().await += 1; + return Ok(Self::reader(self.backup.clone(), "backup-etag")); + } + + Err(Error::FileNotFound) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + _data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> StorageResult { + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + let if_match = opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_match_value) + .map(str::to_owned); + let if_none_match = opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_none_match_value) + .map(str::to_owned); + self.puts.lock().await.push(CachePutRecord { + object: object.to_string(), + if_match, + if_none_match, + }); + Ok(ObjectInfo { + etag: Some(format!("saved-{object}")), + ..Default::default() + }) + } + } + + #[test] + fn cache_revisions_map_to_compare_and_swap_preconditions() { + let missing = DataUsageCacheRevision::Missing.preconditions(); + let existing = DataUsageCacheRevision::Etag("etag-1".to_string()).preconditions(); + + assert_eq!(missing.if_none_match_value(), Some("*")); + assert!(missing.if_match_value().is_none()); + assert_eq!(existing.if_match_value(), Some("etag-1")); + assert!(existing.if_none_match_value().is_none()); + } + + #[tokio::test] + async fn backup_cache_load_recovers_main_revision_before_cas_save() { + let mut expected = DataUsageCache::default(); + expected.info.name = "bucket".to_string(); + let store = Arc::new(BackupFallbackStore::new(expected.marshal_msg().expect("serialize backup cache"), true)); + let mut loaded = DataUsageCache::default(); + + let revisions = loaded + .load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME) + .await + .expect("backup cache should load after the main revision recovers"); + + assert_eq!(loaded.info.name, "bucket"); + assert_eq!(store.main_reads.load(Ordering::SeqCst), 2); + assert!(matches!(revisions.main, DataUsageCacheRevision::Etag(ref etag) if etag == "main-etag")); + assert!(matches!( + revisions.backup, + Some(DataUsageCacheRevision::Etag(ref etag)) if etag == "backup-etag" + )); + assert_eq!(*store.backup_reads.lock().await, 1); + + loaded + .save_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME, &revisions) + .await + .expect("recovered revisions should protect both cache writes"); + let puts = store.puts.lock().await; + assert_eq!( + *puts, + vec![ + CachePutRecord { + object: path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]), + if_match: Some("main-etag".to_string()), + if_none_match: None, + }, + CachePutRecord { + object: path_join_buf(&[BUCKET_META_PREFIX, &format!("{DATA_USAGE_CACHE_NAME}.bkp")]), + if_match: Some("backup-etag".to_string()), + if_none_match: None, + }, + ] + ); + } + + #[tokio::test] + async fn recovered_main_cache_wins_over_stale_backup() { + let mut main = DataUsageCache::default(); + main.info.name = "current-main".to_string(); + let mut backup = DataUsageCache::default(); + backup.info.name = "stale-backup".to_string(); + let store = Arc::new(BackupFallbackStore::with_recovered_main( + backup.marshal_msg().expect("serialize stale backup cache"), + main.marshal_msg().expect("serialize recovered main cache"), + )); + let mut loaded = DataUsageCache::default(); + + let revisions = loaded + .load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME) + .await + .expect("a recovered valid main cache should supersede the fallback backup"); + + assert_eq!(loaded.info.name, "current-main"); + assert_eq!(store.main_reads.load(Ordering::SeqCst), 2); + assert_eq!(*store.backup_reads.lock().await, 1); + assert!(matches!(revisions.main, DataUsageCacheRevision::Etag(ref etag) if etag == "main-etag")); + assert!(matches!( + revisions.backup, + Some(DataUsageCacheRevision::Etag(ref etag)) if etag == "backup-etag" + )); + } + + #[tokio::test] + async fn cache_save_reconciles_an_ambiguous_committed_write() { + let store = Arc::new(AmbiguousCacheCommitStore::default()); + let mut cache = DataUsageCache::default(); + cache.info.name = "bucket".to_string(); + cache.replace("bucket", "", DataUsageEntry::default()); + let revisions = DataUsageCacheRevisions { + main: DataUsageCacheRevision::Missing, + backup: None, + }; + + cache + .save_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME, &revisions) + .await + .expect("read-after-error reconciliation should recognize the committed cache"); + + assert_eq!(store.puts.load(Ordering::SeqCst), 1); + let persisted = store.data.lock().await.clone().expect("cache should be committed"); + assert_eq!( + DataUsageCache::unmarshal(&persisted) + .expect("committed cache should decode") + .info + .name, + "bucket" + ); + } + + #[tokio::test] + async fn backup_cache_load_fails_closed_without_main_revision_quorum() { + let backup = DataUsageCache::default().marshal_msg().expect("serialize backup cache"); + let store = Arc::new(BackupFallbackStore::new(backup, false)); + let mut loaded = DataUsageCache::default(); + + let error = loaded + .load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME) + .await + .expect_err("missing main revision quorum must prevent a CAS save"); + + assert!(matches!(error, Error::ErasureReadQuorum)); + assert_eq!(store.main_reads.load(Ordering::SeqCst), 5); + assert_eq!(*store.backup_reads.lock().await, 0); + } + + #[tokio::test] + async fn corrupt_primary_cache_loads_valid_backup() { + let mut expected = DataUsageCache::default(); + expected.info.name = "recovered".to_string(); + let store = Arc::new(CacheReadStore::new( + CacheReadBody::Bytes(b"not-msgpack".to_vec()), + Some(expected.marshal_msg().expect("serialize backup cache")), + )); + let mut loaded = DataUsageCache::default(); + + loaded + .load(store.clone(), DATA_USAGE_CACHE_NAME) + .await + .expect("valid backup must recover a corrupt primary cache"); + + assert_eq!(loaded.info.name, "recovered"); + assert_eq!(store.puts.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn corrupt_cache_without_valid_backup_rebuilds_with_cas_revisions() { + for backup in [None, Some(b"also-not-msgpack".to_vec())] { + let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(b"not-msgpack".to_vec()), backup)); + let mut loaded = DataUsageCache::default(); + + let revisions = loaded + .load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME) + .await + .expect("corrupt scanner caches should be discarded for a full rebuild"); + + assert!(loaded.info.name.is_empty()); + assert!(loaded.cache.is_empty()); + assert!(matches!( + revisions.main, + DataUsageCacheRevision::Etag(ref etag) if etag == "main-etag" + )); + assert!(matches!( + revisions.backup, + Some(DataUsageCacheRevision::Missing | DataUsageCacheRevision::Etag(_)) + )); + + loaded + .save_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME, &revisions) + .await + .expect("rebuilt cache should replace corrupt cache objects with CAS protection"); + assert_eq!(store.puts.load(Ordering::SeqCst), 2); + } + } + + #[tokio::test] + async fn partial_cache_body_read_is_retryable_and_does_not_save() { + let store = Arc::new(CacheReadStore::new(CacheReadBody::PrefixThenError(vec![0x81, 0xa4, b'n', b'a']), None)); + + let attempt = DataUsageCache::try_load_inner(store.clone(), DATA_USAGE_CACHE_NAME, Duration::from_secs(1)) + .await + .expect("body read failures should remain recoverable load attempts"); + + assert!(matches!(attempt, DataUsageCacheLoadAttempt::Retryable(_))); + assert_eq!(store.puts.load(Ordering::SeqCst), 0); + } #[test] fn test_data_usage_info_creation() { @@ -1177,6 +2102,87 @@ mod tests { assert_eq!(summary1.versions, 15); } + #[test] + fn size_summary_add_saturates_all_usage_counters() { + let target = "arn:minio:replication::target".to_string(); + let mut summary = SizeSummary { + total_size: usize::MAX, + versions: usize::MAX, + delete_markers: usize::MAX, + replicated_size: i64::MAX, + replicated_count: usize::MAX, + pending_size: i64::MAX, + failed_size: i64::MAX, + replica_size: i64::MAX, + replica_count: usize::MAX, + pending_count: usize::MAX, + failed_count: usize::MAX, + ..Default::default() + }; + summary.repl_target_stats.insert( + target.clone(), + ReplTargetSizeSummary { + replicated_size: i64::MAX, + replicated_count: usize::MAX, + pending_size: i64::MAX, + failed_size: i64::MAX, + pending_count: usize::MAX, + failed_count: usize::MAX, + }, + ); + + let mut increment = SizeSummary { + total_size: 1, + versions: 1, + delete_markers: 1, + replicated_size: 1, + replicated_count: 1, + pending_size: 1, + failed_size: 1, + replica_size: 1, + replica_count: 1, + pending_count: 1, + failed_count: 1, + ..Default::default() + }; + increment.repl_target_stats.insert( + target.clone(), + ReplTargetSizeSummary { + replicated_size: 1, + replicated_count: 1, + pending_size: 1, + failed_size: 1, + pending_count: 1, + failed_count: 1, + }, + ); + + summary.add(&increment); + + assert_eq!(summary.total_size, usize::MAX); + assert_eq!(summary.versions, usize::MAX); + assert_eq!(summary.delete_markers, usize::MAX); + assert_eq!(summary.replicated_size, i64::MAX); + assert_eq!(summary.replicated_count, usize::MAX); + assert_eq!(summary.pending_size, i64::MAX); + assert_eq!(summary.failed_size, i64::MAX); + assert_eq!(summary.replica_size, i64::MAX); + assert_eq!(summary.replica_count, usize::MAX); + assert_eq!(summary.pending_count, usize::MAX); + assert_eq!(summary.failed_count, usize::MAX); + + let target_summary = summary + .repl_target_stats + .get(&target) + .expect("replication target summary should remain present"); + assert_eq!(target_summary.replicated_size, i64::MAX); + assert_eq!(target_summary.replicated_count, usize::MAX); + assert_eq!(target_summary.pending_size, i64::MAX); + assert_eq!(target_summary.failed_size, i64::MAX); + assert_eq!(target_summary.pending_count, usize::MAX); + assert_eq!(target_summary.failed_count, usize::MAX); + } + #[test] fn size_summary_counts_delete_markers_separately_from_versions() { let mut summary = SizeSummary::new(); @@ -1272,10 +2278,14 @@ mod tests { assert_eq!(decoded.name, "bucket"); assert_eq!(decoded.next_cycle, 7); + assert_eq!(decoded.leader_epoch, 0); assert!(decoded.scan_resume_after.is_none()); assert!(decoded.scan_checkpoint.is_none()); assert!(decoded.object_lock.is_none()); assert!(decoded.pending_heals.is_empty()); + assert!(decoded.source.is_none()); + assert!(!decoded.snapshot_complete); + assert!(decoded.scan_plan_digest.is_none()); } #[test] @@ -1314,6 +2324,268 @@ mod tests { assert!(decoded.scan_resume_after.is_none()); assert!(decoded.scan_checkpoint.is_none()); assert!(decoded.pending_heals.is_empty()); + assert!(decoded.source.is_none()); + assert!(!decoded.snapshot_complete); + assert!(decoded.scan_plan_digest.is_none()); + } + + #[test] + fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() { + #[derive(Deserialize)] + struct OldDataUsageCacheInfo { + name: String, + next_cycle: u64, + last_update: Option, + skip_healing: bool, + lifecycle: Option>, + replication: Option>, + failed_objects: HashMap, + scan_resume_after: Option, + scan_checkpoint: Option, + pending_heals: Vec, + object_lock: Option>, + } + + #[derive(Deserialize)] + struct OldDataUsageCache { + info: OldDataUsageCacheInfo, + cache: HashMap, + } + + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 7, + leader_epoch: 9, + skip_healing: true, + failed_objects: HashMap::from([("bad-object".to_string(), 11)]), + source: Some(DataUsageCacheSource::new(1, 2)), + snapshot_complete: true, + scan_plan_digest: Some(TEST_PLAN_DIGEST), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + let buf = cache.marshal_msg().expect("Failed to serialize new cache"); + let current = DataUsageCache::unmarshal(&buf).expect("Current reader failed to deserialize new cache"); + assert_eq!(current.info.leader_epoch, 9); + assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2))); + assert!(current.info.snapshot_complete); + assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3)); + + let decoded: OldDataUsageCache = rmp_serde::from_slice(&buf).expect("Old reader failed to deserialize new cache"); + + assert_eq!(decoded.info.name, "bucket"); + assert_eq!(decoded.info.next_cycle, 7); + assert!(decoded.info.last_update.is_none()); + assert!(decoded.info.skip_healing); + assert!(decoded.info.lifecycle.is_none()); + assert!(decoded.info.replication.is_none()); + assert_eq!(decoded.info.failed_objects.get("bad-object"), Some(&11)); + assert!(decoded.info.scan_resume_after.is_none()); + assert!(decoded.info.scan_checkpoint.is_none()); + assert!(decoded.info.pending_heals.is_empty()); + assert!(decoded.info.object_lock.is_none()); + assert_eq!(decoded.cache.get("bucket").map(|entry| entry.objects), Some(3)); + } + + #[test] + fn data_usage_cache_prepare_for_scan_rejects_unscoped_distributed_cache() { + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 7, + scan_resume_after: Some("bucket/prefix".to_string()), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + + let reused = cache.prepare_for_scan("bucket", 8, 0, DataUsageCacheSource::new(1, 0), TEST_PLAN_DIGEST, true); + + assert_eq!(reused, DataUsageCachePrepareOutcome::Reset); + assert_eq!(cache.info.name, "bucket"); + assert_eq!(cache.info.next_cycle, 8); + assert_eq!(cache.info.source, Some(DataUsageCacheSource::new(1, 0))); + assert_eq!(cache.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert!(!cache.info.snapshot_complete); + assert!(cache.info.scan_resume_after.is_none()); + assert!(cache.cache.is_empty()); + } + + #[test] + fn data_usage_cache_prepare_for_scan_preserves_matching_partial_progress() { + let source = DataUsageCacheSource::new(1, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 7, + scan_resume_after: Some("bucket/prefix".to_string()), + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(TEST_PLAN_DIGEST), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + + let reused = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true); + + assert_eq!(reused, DataUsageCachePrepareOutcome::Reused); + assert_eq!(cache.info.scan_resume_after.as_deref(), Some("bucket/prefix")); + assert_eq!(cache.find("bucket").map(|entry| entry.objects), Some(3)); + assert_eq!(cache.info.next_cycle, 8); + assert_eq!(cache.info.source, Some(source)); + assert_eq!(cache.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert!(!cache.info.snapshot_complete); + } + + #[test] + fn data_usage_cache_prepare_for_scan_rejects_legacy_cache_without_a_bucket_plan() { + let source = DataUsageCacheSource::new(0, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 7, + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + + let reused = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, false); + + assert_eq!(reused, DataUsageCachePrepareOutcome::Reset); + assert!(cache.find("bucket").is_none()); + assert_eq!(cache.info.source, Some(source)); + assert_eq!(cache.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert!(!cache.info.snapshot_complete); + } + + #[test] + fn data_usage_cache_prepare_for_scan_rejects_a_different_bucket_plan() { + let source = DataUsageCacheSource::new(1, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 7, + source: Some(source), + scan_plan_digest: Some(TEST_PLAN_DIGEST), + scan_resume_after: Some("bucket/prefix".to_string()), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + + let next_plan = DataUsageScanPlanDigest([4; 32]); + let reused = cache.prepare_for_scan("bucket", 8, 0, source, next_plan, true); + + assert_eq!(reused, DataUsageCachePrepareOutcome::Reset); + assert_eq!(cache.info.scan_plan_digest, Some(next_plan)); + assert!(cache.info.scan_resume_after.is_none()); + assert!(cache.cache.is_empty()); + } + + #[test] + fn data_usage_cache_prepare_for_scan_rejects_cycle_regression_without_mutation() { + let source = DataUsageCacheSource::new(1, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 8, + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(TEST_PLAN_DIGEST), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + "", + DataUsageEntry { + objects: 3, + ..Default::default() + }, + ); + + let outcome = cache.prepare_for_scan("bucket", 7, 0, source, DataUsageScanPlanDigest([4; 32]), true); + + assert_eq!(outcome, DataUsageCachePrepareOutcome::RejectedNewerCycle); + assert_eq!(cache.info.next_cycle, 8); + assert_eq!(cache.info.source, Some(source)); + assert_eq!(cache.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert!(cache.info.snapshot_complete); + assert_eq!(cache.find("bucket").map(|entry| entry.objects), Some(3)); + } + + #[test] + fn data_usage_cache_prepare_for_scan_fences_leader_epochs() { + let source = DataUsageCacheSource::new(1, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 8, + leader_epoch: 11, + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(TEST_PLAN_DIGEST), + ..Default::default() + }, + ..Default::default() + }; + cache.replace("bucket", "", DataUsageEntry::default()); + + let stale = cache.prepare_for_scan("bucket", 8, 10, source, TEST_PLAN_DIGEST, true); + assert_eq!(stale, DataUsageCachePrepareOutcome::RejectedNewerLeader); + assert_eq!(cache.info.leader_epoch, 11); + assert!(cache.info.snapshot_complete); + + let replacement = cache.prepare_for_scan("bucket", 8, 12, source, TEST_PLAN_DIGEST, true); + assert_eq!(replacement, DataUsageCachePrepareOutcome::Reset); + assert_eq!(cache.info.leader_epoch, 12); + assert!(!cache.info.snapshot_complete); + assert!(cache.cache.is_empty()); } #[test] @@ -1495,6 +2767,82 @@ mod tests { assert!(flat.children.is_empty()); } + #[test] + fn checked_flatten_rejects_dangling_child() { + let root_key = hash_path("bucket").key(); + let mut cache = DataUsageCache::default(); + cache.cache.insert( + root_key, + DataUsageEntry { + objects: 1, + children: HashSet::from(["missing-child".to_string()]), + ..Default::default() + }, + ); + + assert!( + cache.checked_flatten("bucket").is_none(), + "a missing child must invalidate an exact usage snapshot" + ); + } + + #[test] + fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() { + let root_key = hash_path("bucket").key(); + let mut cache = DataUsageCache::default(); + cache.cache.insert( + root_key.clone(), + DataUsageEntry { + objects: 1, + ..Default::default() + }, + ); + + let mut parent = root_key; + for depth in 1..=MAX_DATA_USAGE_CACHE_DEPTH { + let child = format!("depth-{depth}"); + cache + .cache + .get_mut(&parent) + .expect("parent should exist") + .children + .insert(child.clone()); + cache.cache.insert( + child.clone(), + DataUsageEntry { + objects: 1, + ..Default::default() + }, + ); + parent = child; + } + + let flattened = cache + .checked_flatten("bucket") + .expect("a tree ending at the configured depth limit should be valid"); + assert_eq!(flattened.objects, MAX_DATA_USAGE_CACHE_DEPTH + 1); + + let too_deep = "depth-too-deep".to_string(); + cache + .cache + .get_mut(&parent) + .expect("last valid node should exist") + .children + .insert(too_deep.clone()); + cache.cache.insert( + too_deep, + DataUsageEntry { + objects: 1, + ..Default::default() + }, + ); + + assert!( + cache.checked_flatten("bucket").is_none(), + "a tree deeper than the configured limit must be rejected" + ); + } + #[test] fn test_find_children_copy_preserves_missing_entry_behavior() { let mut cache = DataUsageCache::default(); @@ -1564,6 +2912,15 @@ mod tests { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } + #[test] + fn test_cache_persistence_timeout_covers_all_save_attempts() { + with_var(ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + assert_eq!(DataUsageCache::persistence_timeout(), Duration::from_millis(31_350)); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + } + #[tokio::test] async fn test_retry_save_op_retries_on_error_then_succeeds() { let attempts = Arc::new(AtomicUsize::new(0)); diff --git a/crates/scanner/src/error.rs b/crates/scanner/src/error.rs index cbd628c78..26e613ec0 100644 --- a/crates/scanner/src/error.rs +++ b/crates/scanner/src/error.rs @@ -36,7 +36,23 @@ pub enum ScannerError { #[error("Scanner error: {0}")] Other(String), + /// A remote namespace scanner request ID was already accepted. + #[error("Remote namespace scanner request replay detected")] + RemoteRequestReplay, + + /// The bounded remote namespace scanner replay cache cannot accept more IDs yet. + #[error("Remote namespace scanner replay cache capacity exceeded")] + RemoteReplayCapacity, + + /// A remote namespace scanner request is already active for the target disk. + #[error("Remote namespace scanner disk is already active")] + RemoteDiskBusy, + /// Partial data usage cache produced before the scanner stopped. #[error("Scanner stopped with partial data usage cache")] PartialCache(Box), + + /// Partial cache retained because the bucket or scan root disappeared. + #[error("Scanner namespace path disappeared during the scan")] + NamespaceNotFoundCache(Box), } diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index f8bb760d9..44a798ea3 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -20,6 +20,7 @@ rust_2018_idioms )] +use bytes::Bytes; use http::HeaderMap; use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config}; use std::path::PathBuf; @@ -29,11 +30,11 @@ use storage_api::owner::{ ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle, - EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, - EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, - EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, ScannerReplicationHealObject, - ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, - ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, + EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, + EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, + EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, + ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, + ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config, @@ -41,11 +42,16 @@ use storage_api::owner::{ scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] -use storage_api::owner::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk}; +use storage_api::owner::{ + EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext, + EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx, + ecstore_new_disk, +}; use tokio_util::sync::CancellationToken; pub mod data_usage_define; pub mod error; +mod remote_scanner; pub mod runtime_config; pub mod scanner; pub mod scanner_budget; @@ -56,16 +62,23 @@ pub(crate) mod storage_api; pub use data_usage_define::*; pub use error::ScannerError; +pub use remote_scanner::{ + NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request, + claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request, + remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence, +}; pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use rustfs_common::last_minute; -pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status}; +pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest}; pub use scanner_io::{ - clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, + ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, + record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state, scanner_maintenance_generation, }; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; pub use storage_api::ScannerReplicationConfig as ReplicationConfig; +pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION; static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0); static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0); @@ -150,6 +163,7 @@ pub(crate) type BucketTargetSys = EcstoreBucketTargetSys; pub(crate) type BucketVersioningSys = EcstoreBucketVersioningSys; pub(crate) type DiskInfo = EcstoreDiskInfo; pub(crate) type DiskInfoOptions = EcstoreDiskInfoOptions; +pub(crate) type NsScannerOpenRequest = EcstoreNsScannerOpenRequest; pub(crate) type DiskBytes = EcstoreDiskBytes; pub(crate) type Evaluator = EcstoreEvaluator; pub(crate) type Event = EcstoreEvent; @@ -185,6 +199,27 @@ pub(crate) fn init_ecstore_config_for_scanner_tests() { pub(crate) type DiskOption = EcstoreDiskOption; #[cfg(test)] pub(crate) type Endpoint = EcstoreEndpoint; +#[cfg(test)] +pub(crate) type EndpointServerPools = EcstoreEndpointServerPools; +#[cfg(test)] +pub(crate) type Endpoints = EcstoreEndpoints; +#[cfg(test)] +pub(crate) type InstanceContext = EcstoreInstanceContext; +#[cfg(test)] +pub(crate) type PoolEndpoints = EcstorePoolEndpoints; + +#[cfg(test)] +pub(crate) async fn init_local_disks_with_instance_ctx( + ctx: &Arc, + pools: EndpointServerPools, +) -> EcstoreResult<()> { + ecstore_init_local_disks_with_instance_ctx(ctx, pools).await +} + +#[cfg(test)] +pub(crate) async fn init_bucket_metadata_sys_for_scanner_tests(store: Arc) { + ecstore_init_bucket_metadata_sys(store, Vec::new()).await; +} #[cfg(test)] pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> DiskResult { @@ -247,6 +282,8 @@ impl ScannerVersioningConfigExt for s3s::dto::VersioningConfiguration { pub(crate) trait ScannerDiskExt { async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; + fn is_local(&self) -> bool; + fn host_name(&self) -> String; fn path(&self) -> PathBuf; fn get_disk_location(&self) -> DiskLocation; fn start_scan(&self) -> ScanGuard; @@ -264,6 +301,14 @@ where EcstoreDiskAPI::read_metadata(self, volume, path).await } + fn is_local(&self) -> bool { + EcstoreDiskAPI::is_local(self) + } + + fn host_name(&self) -> String { + EcstoreDiskAPI::host_name(self) + } + fn path(&self) -> PathBuf { EcstoreDiskAPI::path(self) } @@ -344,6 +389,10 @@ pub(crate) async fn scanner_is_erasure_sd() -> bool { ecstore_is_erasure_sd().await } +pub(crate) async fn scanner_disk_is_online(disk: &Disk) -> bool { + EcstoreDiskAPI::is_online(disk).await +} + pub(crate) async fn read_config(api: Arc, file: &str) -> EcstoreResult> where S: ScannerObjectIO, @@ -358,6 +407,53 @@ where ecstore_save_config(api, file, data).await } +pub(crate) async fn save_config_with_preconditions( + api: Arc, + file: &str, + data: Vec, + preconditions: HTTPPreconditions, +) -> EcstoreResult +where + S: ScannerObjectIO, +{ + let mut reader = ScannerPutObjReader::from_vec(data); + api.put_object( + RUSTFS_META_BUCKET, + file, + &mut reader, + &ScannerObjectOptions { + max_parity: true, + http_preconditions: Some(preconditions), + ..Default::default() + }, + ) + .await +} + +pub(crate) async fn save_config_shared_with_preconditions( + api: Arc, + file: &str, + data: Bytes, + sha256hex: Option, + preconditions: HTTPPreconditions, +) -> EcstoreResult +where + S: ScannerObjectIO, +{ + let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?; + api.put_object( + RUSTFS_META_BUCKET, + file, + &mut reader, + &ScannerObjectOptions { + max_parity: true, + http_preconditions: Some(preconditions), + ..Default::default() + }, + ) + .await +} + pub(crate) async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> std::result::Result<(), DiskError> { ecstore_list_path_raw(rx, opts).await } diff --git a/crates/scanner/src/remote_scanner.rs b/crates/scanner/src/remote_scanner.rs new file mode 100644 index 000000000..1164587d0 --- /dev/null +++ b/crates/scanner/src/remote_scanner.rs @@ -0,0 +1,3011 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig}; +use crate::scanner_io::{ + ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, cache_snapshot_is_current, scanner_cache_lock_resource, + scanner_cache_lock_timeout, scanner_set_disk_inventory, +}; +use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION; +use crate::storage_api::scan::NamespaceLocking as _; +use crate::{ + DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, + DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, EcstoreError, RUSTFS_META_BUCKET, ScannerDiskExt as _, ScannerError, + ScannerObjectIO, StorageError, is_reserved_or_invalid_bucket, read_config, resolve_scanner_object_store_handle, +}; +use hmac::{Hmac, KeyInit, Mac}; +use rustfs_common::heal_channel::HealScanMode; +use rustfs_common::metrics::{Metric, Metrics}; +use rustfs_credentials::try_get_rpc_token; +use rustfs_utils::path::path_join_buf; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::collections::HashMap; +use std::future::Future; +use std::io::{Error as IoError, ErrorKind}; +use std::pin::Pin; +use std::sync::{ + Arc, LazyLock, Mutex, + atomic::{AtomicU8, Ordering}, +}; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::sync::{Mutex as AsyncMutex, Notify, OwnedSemaphorePermit, Semaphore}; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +type HmacSha256 = Hmac; + +const NS_SCANNER_MAX_FRAME_SIZE: usize = 2 * 1024 * 1024; +const NS_SCANNER_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(1); +const NS_SCANNER_BUDGET_HEARTBEAT_INTERVAL: Duration = Duration::from_millis(250); +const NS_SCANNER_STALL_TIMEOUT: Duration = Duration::from_secs(15); +const NS_SCANNER_SEMANTIC_STALL_TIMEOUT: Duration = Duration::from_secs(5 * 60); +const NS_SCANNER_WRITE_TIMEOUT: Duration = Duration::from_secs(15); +const NS_SCANNER_MAX_RPC_LIFETIME: Duration = Duration::from_secs(24 * 60 * 60); +const NS_SCANNER_DISCONNECT_GRACE_MAX: Duration = Duration::from_secs(2 * 60); +const NS_SCANNER_LOCK_POLL_INTERVAL: Duration = Duration::from_millis(250); +const NS_SCANNER_FENCE_POLL_INTERVAL: Duration = Duration::from_secs(5); +const NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +const NS_SCANNER_DISK_HEALTH_TIMEOUT: Duration = Duration::from_secs(5); +const NS_SCANNER_VALIDATED_CYCLE_TTL: Duration = Duration::from_secs(1); +const NS_SCANNER_MAX_REPLAY_SESSIONS: usize = 65_536; +const NS_SCANNER_MAX_ERROR_CHARS: usize = 4096; +const NS_SCANNER_FRAME_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-frame-v3"; + +pub const NS_SCANNER_MAX_REQUEST_BODY_SIZE: usize = 16 * 1024; + +static REMOTE_SCANNER_REPLAY_CACHE: LazyLock> = + LazyLock::new(|| Mutex::new(RemoteScannerReplayCache::default())); +static REMOTE_SCANNER_ADMISSION: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); +static REMOTE_SCANNER_VALIDATED_CYCLE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); +static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock> = LazyLock::new(|| AsyncMutex::new(())); + +pub struct RemoteScannerAdmission { + _permit: OwnedSemaphorePermit, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerBudget { + max_duration_ms: Option, + max_objects: Option, + max_directories: Option, +} + +impl RemoteScannerBudget { + fn from_config(config: ScannerCycleBudgetConfig) -> Self { + Self { + max_duration_ms: config + .max_duration + .map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX).max(1)), + max_objects: config.max_objects, + max_directories: config.max_directories, + } + } + + fn into_config(self) -> ScannerCycleBudgetConfig { + ScannerCycleBudgetConfig { + max_duration: self.max_duration_ms.map(Duration::from_millis), + max_objects: self.max_objects, + max_directories: self.max_directories, + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerRequestWire { + version: u16, + request_id: Uuid, + server_epoch: Uuid, + session_id: Uuid, + session_sequence: u64, + bucket: String, + next_cycle: u64, + leader_epoch: u64, + scan_plan_digest: DataUsageScanPlanDigest, + skip_healing: bool, + scan_mode: HealScanMode, + budget: RemoteScannerBudget, +} + +#[derive(Debug)] +pub struct RemoteScannerRequest(RemoteScannerRequestWire); + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct RemoteScannerProgress { + objects_scanned: u64, + directories_started: u64, + entries_visited: u64, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RemoteScannerPhase { + #[default] + Scanning, + Persisting, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerComplete { + source: DataUsageCacheSource, + scan_plan_digest: DataUsageScanPlanDigest, + usage: DataUsageEntryInfo, + pending_maintenance_work: bool, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RemoteScannerErrorScope { + Bucket, + Worker, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerErrorFrame { + scope: RemoteScannerErrorScope, + message: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +enum RemoteScannerFrameResult { + Progress, + Complete(Box), + Partial, + NamespaceNotFound, + CycleAhead { required_cycle: u64 }, + Error(RemoteScannerErrorFrame), +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerFrame { + progress: RemoteScannerProgress, + phase: RemoteScannerPhase, + result: RemoteScannerFrameResult, +} + +impl RemoteScannerFrame { + #[cfg(test)] + fn progress(progress: RemoteScannerProgress) -> Self { + Self { + progress, + phase: RemoteScannerPhase::Scanning, + result: RemoteScannerFrameResult::Progress, + } + } + + #[cfg(test)] + fn terminal(progress: RemoteScannerProgress, result: RemoteScannerFrameResult) -> Self { + Self { + progress, + phase: RemoteScannerPhase::Persisting, + result, + } + } + + fn with_phase(progress: RemoteScannerProgress, phase: RemoteScannerPhase, result: RemoteScannerFrameResult) -> Self { + Self { progress, phase, result } + } +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RemoteScannerFrameEnvelope { + version: u16, + sequence: u64, + payload: Vec, + mac: Vec, +} + +#[derive(Debug)] +pub(crate) enum RemoteScannerOutcome { + Complete { + usage: Box, + pending_maintenance_work: bool, + }, + Partial, + NamespaceNotFound, + CycleAhead(u64), +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct RemoteScannerScanSpec<'a> { + pub(crate) bucket: &'a str, + pub(crate) next_cycle: u64, + pub(crate) leader_epoch: u64, + pub(crate) server_epoch: Uuid, + pub(crate) session_id: Uuid, + pub(crate) session_sequence: u64, + pub(crate) scan_plan_digest: DataUsageScanPlanDigest, + pub(crate) skip_healing: bool, + pub(crate) scan_mode: HealScanMode, +} + +#[derive(Clone, Copy)] +struct RemoteScannerResponseExpectation<'a> { + bucket: &'a str, + source: DataUsageCacheSource, + next_cycle: u64, + scan_plan_digest: DataUsageScanPlanDigest, +} + +#[derive(Debug)] +pub(crate) struct RemoteScannerFailure { + error: StorageError, + disposition: RemoteScannerFailureDisposition, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RemoteScannerFailureDisposition { + Bucket, + RetireWorker, + RetryBucket, +} + +impl RemoteScannerFailure { + fn transport(error: StorageError) -> Self { + Self { + error, + disposition: RemoteScannerFailureDisposition::RetireWorker, + } + } + + fn bucket(error: StorageError) -> Self { + Self { + error, + disposition: RemoteScannerFailureDisposition::Bucket, + } + } + + fn retry_bucket(error: StorageError) -> Self { + Self { + error, + disposition: RemoteScannerFailureDisposition::RetryBucket, + } + } + + pub(crate) fn retire_worker(&self) -> bool { + self.disposition == RemoteScannerFailureDisposition::RetireWorker + } + + pub(crate) fn retry_bucket_work(&self) -> bool { + self.disposition == RemoteScannerFailureDisposition::RetryBucket + } +} + +#[derive(Debug)] +struct RemoteScannerServerError { + scope: RemoteScannerErrorScope, + error: ScannerError, +} + +impl RemoteScannerServerError { + fn worker(message: impl Into) -> Self { + Self { + scope: RemoteScannerErrorScope::Worker, + error: ScannerError::Other(message.into()), + } + } + + fn disk_scan(error: ScannerError, disk_online: bool) -> Self { + Self { + scope: if disk_online { + RemoteScannerErrorScope::Bucket + } else { + RemoteScannerErrorScope::Worker + }, + error, + } + } + + fn into_frame(self) -> RemoteScannerErrorFrame { + RemoteScannerErrorFrame { + scope: self.scope, + message: limit_error_message(self.error.to_string()), + } + } +} + +impl std::fmt::Display for RemoteScannerFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +#[derive(Debug)] +struct RemoteScannerStreamError { + error: StorageError, + progress_fully_reported: bool, + retire_worker: bool, +} + +impl RemoteScannerStreamError { + fn uncertain(error: StorageError) -> Self { + Self { + error, + progress_fully_reported: false, + retire_worker: true, + } + } + + fn reconciled(error: StorageError) -> Self { + Self { + error, + progress_fully_reported: true, + retire_worker: true, + } + } + + fn bucket(error: StorageError) -> Self { + Self { + error, + progress_fully_reported: true, + retire_worker: false, + } + } + + fn for_phase(error: StorageError, phase: RemoteScannerPhase) -> Self { + if phase == RemoteScannerPhase::Persisting { + Self::reconciled(error) + } else { + Self::uncertain(error) + } + } +} + +impl std::fmt::Display for RemoteScannerStreamError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +type RemoteScannerStreamResult = std::result::Result; + +#[derive(Default)] +struct RemoteScannerReplayCache { + cycle: Option, + leader_epoch: Option, + sessions: HashMap<(String, Uuid), u64>, +} + +#[derive(Clone, Copy)] +struct RemoteScannerValidatedCycle { + cycle: u64, + leader_epoch: u64, + valid_until: Instant, +} + +impl RemoteScannerValidatedCycle { + fn matches(self, requested_cycle: u64, requested_leader_epoch: u64, now: Instant) -> bool { + self.cycle == requested_cycle && self.leader_epoch == requested_leader_epoch && now < self.valid_until + } +} + +impl RemoteScannerReplayCache { + fn preflight_key(&self, key: &(String, Uuid), cycle: u64, leader_epoch: u64, sequence: u64) -> Result<(), ScannerError> { + match self.cycle.zip(self.leader_epoch) { + Some((current_cycle, current_epoch)) if (leader_epoch, cycle) < (current_epoch, current_cycle) => { + return Err(ScannerError::RemoteRequestReplay); + } + Some((current_cycle, current_epoch)) if (leader_epoch, cycle) > (current_epoch, current_cycle) => { + return (sequence == 0).then_some(()).ok_or(ScannerError::RemoteRequestReplay); + } + Some(_) => {} + None => { + if sequence != 0 { + return Err(ScannerError::RemoteRequestReplay); + } + return (self.sessions.len() < NS_SCANNER_MAX_REPLAY_SESSIONS) + .then_some(()) + .ok_or(ScannerError::RemoteReplayCapacity); + } + } + if let Some(last_sequence) = self.sessions.get(key) { + let expected = last_sequence.checked_add(1).ok_or(ScannerError::RemoteRequestReplay)?; + return (sequence == expected).then_some(()).ok_or(ScannerError::RemoteRequestReplay); + } + if sequence != 0 { + return Err(ScannerError::RemoteRequestReplay); + } + (self.sessions.len() < NS_SCANNER_MAX_REPLAY_SESSIONS) + .then_some(()) + .ok_or(ScannerError::RemoteReplayCapacity) + } + + fn preflight( + &self, + disk_key: &str, + session_id: Uuid, + cycle: u64, + leader_epoch: u64, + sequence: u64, + ) -> Result<(), ScannerError> { + self.preflight_key(&(disk_key.to_string(), session_id), cycle, leader_epoch, sequence) + } + + fn claim( + &mut self, + disk_key: String, + session_id: Uuid, + cycle: u64, + leader_epoch: u64, + sequence: u64, + ) -> Result<(), ScannerError> { + let key = (disk_key, session_id); + self.preflight_key(&key, cycle, leader_epoch, sequence)?; + match self.cycle.zip(self.leader_epoch) { + Some((current_cycle, current_epoch)) if (leader_epoch, cycle) > (current_epoch, current_cycle) => { + self.sessions.clear(); + self.cycle = Some(cycle); + self.leader_epoch = Some(leader_epoch); + } + None => { + self.cycle = Some(cycle); + self.leader_epoch = Some(leader_epoch); + } + Some(_) => {} + } + if let Some(last_sequence) = self.sessions.get_mut(&key) { + *last_sequence = sequence; + return Ok(()); + } + self.sessions.insert(key, sequence); + Ok(()) + } +} + +fn cache_source_for_disk(disk: &Disk) -> Result { + let location = disk.get_disk_location(); + let pool_index = location + .pool_idx + .ok_or_else(|| ScannerError::Other("remote namespace scanner disk has no pool index".to_string()))?; + let set_index = location + .set_idx + .ok_or_else(|| ScannerError::Other("remote namespace scanner disk has no set index".to_string()))?; + Ok(DataUsageCacheSource::new(pool_index, set_index)) +} + +struct FrameAuthenticator { + request_id: Uuid, + secret: String, +} + +impl FrameAuthenticator { + fn from_rpc_secret(request_id: Uuid) -> Result { + let secret = try_get_rpc_token() + .map_err(|err| ScannerError::Other(format!("remote namespace scanner authentication unavailable: {err}")))?; + Ok(Self { request_id, secret }) + } + + #[cfg(test)] + fn for_test(request_id: Uuid) -> Self { + Self { + request_id, + secret: "test-remote-scanner-secret".to_string(), + } + } + + fn sign(&self, sequence: u64, payload: &[u8]) -> Result, ScannerError> { + let mut mac = HmacSha256::new_from_slice(self.secret.as_bytes()) + .map_err(|_| ScannerError::Other("invalid remote namespace scanner authentication key".to_string()))?; + update_frame_mac(&mut mac, self.request_id, sequence, payload); + Ok(mac.finalize().into_bytes().to_vec()) + } + + fn verify(&self, sequence: u64, payload: &[u8], signature: &[u8]) -> std::io::Result<()> { + let mut mac = HmacSha256::new_from_slice(self.secret.as_bytes()) + .map_err(|_| IoError::other("invalid remote namespace scanner authentication key"))?; + update_frame_mac(&mut mac, self.request_id, sequence, payload); + mac.verify_slice(signature) + .map_err(|_| IoError::new(ErrorKind::PermissionDenied, "remote namespace scanner frame authentication failed")) + } +} + +fn update_frame_mac(mac: &mut HmacSha256, request_id: Uuid, sequence: u64, payload: &[u8]) { + mac.update(NS_SCANNER_FRAME_AUTH_DOMAIN); + mac.update(request_id.as_bytes()); + mac.update(&sequence.to_be_bytes()); + mac.update(payload); +} + +pub fn decode_remote_scanner_request(body: &[u8]) -> Result { + if body.is_empty() || body.len() > NS_SCANNER_MAX_REQUEST_BODY_SIZE { + return Err(ScannerError::Other("remote namespace scanner request size is invalid".to_string())); + } + + let request: RemoteScannerRequestWire = + rmp_serde::from_slice(body).map_err(|_| ScannerError::Other("invalid remote namespace scanner request".to_string()))?; + if request.version != NS_SCANNER_PROTOCOL_VERSION { + return Err(ScannerError::Other(format!( + "unsupported remote namespace scanner protocol version: {}", + request.version + ))); + } + if request.request_id.is_nil() { + return Err(ScannerError::Other("remote namespace scanner request ID is invalid".to_string())); + } + if request.server_epoch.is_nil() { + return Err(ScannerError::Other("remote namespace scanner server epoch is invalid".to_string())); + } + if request.session_id.is_nil() { + return Err(ScannerError::Other("remote namespace scanner session ID is invalid".to_string())); + } + if request.leader_epoch == 0 { + return Err(ScannerError::Other("remote namespace scanner leader epoch is invalid".to_string())); + } + if request.bucket.contains(['/', '\\']) || is_reserved_or_invalid_bucket(&request.bucket, false) { + return Err(ScannerError::Other("remote namespace scanner bucket is invalid".to_string())); + } + + Ok(RemoteScannerRequest(request)) +} + +pub fn remote_scanner_request_matches_envelope( + request: &RemoteScannerRequest, + request_id: Uuid, + server_epoch: Uuid, + session_id: Uuid, + session_sequence: u64, + next_cycle: u64, + leader_epoch: u64, +) -> bool { + request.0.request_id == request_id + && request.0.server_epoch == server_epoch + && request.0.session_id == session_id + && request.0.session_sequence == session_sequence + && request.0.next_cycle == next_cycle + && request.0.leader_epoch == leader_epoch +} + +pub async fn validate_remote_scanner_request_fence( + requested_cycle: u64, + requested_leader_epoch: u64, +) -> Result<(), ScannerError> { + let store = resolve_scanner_object_store_handle() + .ok_or_else(|| ScannerError::Other("remote namespace scanner object layer is unavailable".to_string()))?; + validate_remote_scanner_request_fence_cached( + requested_cycle, + requested_leader_epoch, + store, + &REMOTE_SCANNER_VALIDATED_CYCLE, + &REMOTE_SCANNER_CYCLE_REFRESH, + NS_SCANNER_VALIDATED_CYCLE_TTL, + ) + .await +} + +async fn validate_remote_scanner_request_fence_cached( + requested_cycle: u64, + requested_leader_epoch: u64, + store: Arc, + cache: &Mutex>, + refresh: &AsyncMutex<()>, + cache_ttl: Duration, +) -> Result<(), ScannerError> { + if cached_remote_scanner_fence_matches(cache, requested_cycle, requested_leader_epoch, Instant::now())? { + return Ok(()); + } + + let _refresh = refresh.lock().await; + if cached_remote_scanner_fence_matches(cache, requested_cycle, requested_leader_epoch, Instant::now())? { + return Ok(()); + } + + let (persisted_cycle, persisted_leader_epoch) = + validate_remote_scanner_request_fence_with_store(requested_cycle, requested_leader_epoch, store).await?; + let valid_until = Instant::now() + .checked_add(cache_ttl) + .ok_or_else(|| ScannerError::Other("remote namespace scanner cycle cache expiry overflow".to_string()))?; + *cache + .lock() + .map_err(|_| ScannerError::Other("remote namespace scanner cycle cache is unavailable".to_string()))? = + Some(RemoteScannerValidatedCycle { + cycle: persisted_cycle, + leader_epoch: persisted_leader_epoch, + valid_until, + }); + Ok(()) +} + +fn cached_remote_scanner_fence_matches( + cache: &Mutex>, + requested_cycle: u64, + requested_leader_epoch: u64, + now: Instant, +) -> Result { + Ok(cache + .lock() + .map_err(|_| ScannerError::Other("remote namespace scanner cycle cache is unavailable".to_string()))? + .is_some_and(|cached| cached.matches(requested_cycle, requested_leader_epoch, now))) +} + +async fn validate_remote_scanner_request_fence_with_store( + requested_cycle: u64, + requested_leader_epoch: u64, + store: Arc, +) -> Result<(u64, u64), ScannerError> { + let (persisted_cycle, persisted_leader_epoch) = match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await { + Ok(buf) => crate::scanner::decode_persisted_scanner_cycle_fence(&buf)?, + Err(EcstoreError::ConfigNotFound) => (0, 0), + Err(err) => { + return Err(ScannerError::Other(format!("failed to read persisted scanner cycle state: {err}"))); + } + }; + if requested_cycle != persisted_cycle || requested_leader_epoch != persisted_leader_epoch { + return Err(ScannerError::Other(format!( + "remote namespace scanner fence does not match persisted state: requested_cycle={requested_cycle}, \ + persisted_cycle={persisted_cycle}, requested_epoch={requested_leader_epoch}, persisted_epoch={persisted_leader_epoch}" + ))); + } + Ok((persisted_cycle, persisted_leader_epoch)) +} + +async fn watch_remote_scanner_request_fence( + requested_cycle: u64, + requested_leader_epoch: u64, + store: Arc, + poll_interval: Duration, +) -> Result<(), ScannerError> { + watch_remote_scanner_request_fence_with_cache( + requested_cycle, + requested_leader_epoch, + store, + poll_interval, + &REMOTE_SCANNER_VALIDATED_CYCLE, + &REMOTE_SCANNER_CYCLE_REFRESH, + NS_SCANNER_VALIDATED_CYCLE_TTL, + ) + .await +} + +async fn watch_remote_scanner_request_fence_with_cache( + requested_cycle: u64, + requested_leader_epoch: u64, + store: Arc, + poll_interval: Duration, + cache: &Mutex>, + refresh: &AsyncMutex<()>, + cache_ttl: Duration, +) -> Result<(), ScannerError> { + if poll_interval.is_zero() { + return Err(ScannerError::Other( + "remote namespace scanner fence poll interval must be positive".to_string(), + )); + } + let first_poll = Instant::now() + .checked_add(poll_interval) + .ok_or_else(|| ScannerError::Other("remote namespace scanner fence poll deadline overflow".to_string()))?; + let mut poll = tokio::time::interval_at(first_poll, poll_interval); + poll.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + poll.tick().await; + validate_remote_scanner_request_fence_cached( + requested_cycle, + requested_leader_epoch, + store.clone(), + cache, + refresh, + cache_ttl, + ) + .await?; + } +} + +pub fn admit_remote_scanner_request(disk: &Disk) -> Result { + if !disk.is_local() { + return Err(ScannerError::Other( + "remote namespace scanner admission requires a local disk".to_string(), + )); + } + try_admit_remote_scanner_key(disk.path().to_string_lossy().into_owned()) +} + +pub fn preflight_remote_scanner_request( + disk: &Disk, + session_id: Uuid, + next_cycle: u64, + leader_epoch: u64, + session_sequence: u64, +) -> Result<(), ScannerError> { + if !disk.is_local() { + return Err(ScannerError::Other( + "remote namespace scanner replay preflight requires a local disk".to_string(), + )); + } + REMOTE_SCANNER_REPLAY_CACHE + .lock() + .map_err(|_| ScannerError::Other("remote namespace scanner replay cache is unavailable".to_string()))? + .preflight( + disk.path().to_string_lossy().as_ref(), + session_id, + next_cycle, + leader_epoch, + session_sequence, + ) +} + +pub fn claim_remote_scanner_request( + disk: &Disk, + session_id: Uuid, + next_cycle: u64, + leader_epoch: u64, + session_sequence: u64, +) -> Result<(), ScannerError> { + if !disk.is_local() { + return Err(ScannerError::Other( + "remote namespace scanner replay claim requires a local disk".to_string(), + )); + } + REMOTE_SCANNER_REPLAY_CACHE + .lock() + .map_err(|_| ScannerError::Other("remote namespace scanner replay cache is unavailable".to_string()))? + .claim( + disk.path().to_string_lossy().into_owned(), + session_id, + next_cycle, + leader_epoch, + session_sequence, + ) +} + +pub(crate) fn try_admit_remote_scanner(disk: &Disk) -> Result { + if !disk.is_local() { + return Err(ScannerError::Other( + "remote namespace scanner admission requires a local disk".to_string(), + )); + } + try_admit_remote_scanner_key(disk.path().to_string_lossy().into_owned()) +} + +fn try_admit_remote_scanner_key(disk_key: String) -> Result { + let semaphore = REMOTE_SCANNER_ADMISSION + .lock() + .map_err(|_| ScannerError::Other("remote namespace scanner admission is unavailable".to_string()))? + .entry(disk_key) + .or_insert_with(|| Arc::new(Semaphore::new(1))) + .clone(); + let permit = semaphore.try_acquire_owned().map_err(|_| ScannerError::RemoteDiskBusy)?; + Ok(RemoteScannerAdmission { _permit: permit }) +} + +pub async fn serve_remote_scanner_request( + disk: Arc, + request: RemoteScannerRequest, + mut writer: W, + disconnect: CancellationToken, +) -> Result<(), ScannerError> +where + W: AsyncWrite + Unpin + Send, +{ + if !disk.is_local() { + return Err(ScannerError::Other( + "remote namespace scanner request resolved to a non-local disk".to_string(), + )); + } + let request = request.0; + let request_id = request.request_id; + let budget_config = request.budget; + let authenticator = FrameAuthenticator::from_rpc_secret(request_id)?; + let heartbeat_interval = if budget_config.max_objects.is_some() || budget_config.max_directories.is_some() { + NS_SCANNER_BUDGET_HEARTBEAT_INTERVAL + } else { + NS_SCANNER_HEARTBEAT_INTERVAL + }; + let parent = CancellationToken::new(); + let _cancel_on_drop = parent.clone().drop_guard(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, budget_config.into_config()); + let ctx = budget.token(); + let phase = Arc::new(AtomicU8::new(RemoteScannerPhase::Scanning as u8)); + let phase_changed = Arc::new(Notify::new()); + let scan = scan_and_persist_local_bucket(disk, ctx, budget.clone(), phase.clone(), phase_changed.clone(), request); + tokio::pin!(scan); + + let mut heartbeat = tokio::time::interval_at(Instant::now() + heartbeat_interval, heartbeat_interval); + heartbeat.set_missed_tick_behavior(MissedTickBehavior::Delay); + let rpc_lifetime = tokio::time::sleep(NS_SCANNER_MAX_RPC_LIFETIME); + tokio::pin!(rpc_lifetime); + let mut sequence = 0_u64; + let mut persistence_announced = false; + + loop { + tokio::select! { + biased; + result = &mut scan => { + let frame_result = match result { + Ok(result) => result, + Err(err) => RemoteScannerFrameResult::Error(err.into_frame()), + }; + write_frame_bounded( + &mut writer, + &authenticator, + &mut sequence, + &RemoteScannerFrame::with_phase( + budget_progress(&budget), + remote_scanner_phase(&phase), + frame_result, + ), + &disconnect, + NS_SCANNER_WRITE_TIMEOUT, + ) + .await?; + shutdown_writer_bounded(&mut writer, &disconnect, NS_SCANNER_WRITE_TIMEOUT).await?; + return Ok(()); + } + _ = disconnect.cancelled() => { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Ok(()); + } + _ = &mut rpc_lifetime => { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Err(ScannerError::Other("remote namespace scanner RPC lifetime exceeded".to_string())); + } + _ = phase_changed.notified(), if !persistence_announced => { + persistence_announced = true; + if let Err(err) = write_frame_bounded( + &mut writer, + &authenticator, + &mut sequence, + &RemoteScannerFrame::with_phase( + budget_progress(&budget), + RemoteScannerPhase::Persisting, + RemoteScannerFrameResult::Progress, + ), + &disconnect, + NS_SCANNER_WRITE_TIMEOUT, + ) + .await + { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Err(err); + } + if let Err(err) = flush_writer_bounded(&mut writer, &disconnect, NS_SCANNER_WRITE_TIMEOUT).await { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Err(err); + } + } + _ = heartbeat.tick() => { + if let Err(err) = write_frame_bounded( + &mut writer, + &authenticator, + &mut sequence, + &RemoteScannerFrame::with_phase( + budget_progress(&budget), + remote_scanner_phase(&phase), + RemoteScannerFrameResult::Progress, + ), + &disconnect, + NS_SCANNER_WRITE_TIMEOUT, + ) + .await + { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Err(err); + } + if let Err(err) = flush_writer_bounded(&mut writer, &disconnect, NS_SCANNER_WRITE_TIMEOUT).await { + parent.cancel(); + await_remote_scan_shutdown(scan.as_mut()).await; + return Err(err); + } + } + } + } +} + +async fn scan_and_persist_local_bucket( + disk: Arc, + ctx: CancellationToken, + budget: Arc, + phase: Arc, + phase_changed: Arc, + request: RemoteScannerRequestWire, +) -> std::result::Result { + let RemoteScannerRequestWire { + bucket, + next_cycle, + leader_epoch, + scan_plan_digest, + skip_healing, + scan_mode, + .. + } = request; + let store = resolve_scanner_object_store_handle() + .ok_or_else(|| RemoteScannerServerError::worker("remote namespace scanner object layer is unavailable"))?; + validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store.clone()) + .await + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence is stale: {err}")))?; + let source = cache_source_for_disk(disk.as_ref()) + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner source is unavailable: {err}")))?; + let set = store + .pools + .get(source.pool_index) + .and_then(|pool| pool.disk_set.get(source.set_index)) + .cloned() + .ok_or_else(|| { + RemoteScannerServerError::worker(format!( + "remote namespace scanner set is unavailable: pool={}, set={}", + source.pool_index, source.set_index + )) + })?; + let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]); + let lock_resource = scanner_cache_lock_resource(&cache_name); + let ns_lock = set + .new_ns_lock(RUSTFS_META_BUCKET, &lock_resource) + .await + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache lock creation failed: {err}")))?; + let guard = ns_lock + .get_write_lock_quiet(scanner_cache_lock_timeout()) + .await + .map_err(|err| { + RemoteScannerServerError::worker(format!("remote namespace scanner cache lock acquisition failed: {err}")) + })?; + let mut cache = DataUsageCache::default(); + let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| { + RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}")) + })?; + if cache_snapshot_is_current(&cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest) { + if guard.is_lock_lost() { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner cache lock was lost before reusing the current snapshot", + )); + } + return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source, + scan_plan_digest, + usage: cache_root_entry_info(&cache) + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?, + pending_maintenance_work: !cache.info.pending_heals.is_empty(), + }))); + } + match cache.prepare_for_scan(&bucket, next_cycle, leader_epoch, source, scan_plan_digest, true) { + DataUsageCachePrepareOutcome::RejectedNewerCycle => { + return Ok(RemoteScannerFrameResult::CycleAhead { + required_cycle: cache.info.next_cycle, + }); + } + DataUsageCachePrepareOutcome::RejectedNewerLeader => { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner rejected work from an older leader epoch", + )); + } + DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {} + } + cache.info.skip_healing = skip_healing; + + let set_disks = scanner_set_disk_inventory(set.as_ref()).await; + let scan_ctx = ctx.child_token(); + let scan = ScannerIODisk::nsscanner_disk(disk.clone(), scan_ctx.clone(), budget, set_disks, cache, None, scan_mode); + tokio::pin!(scan); + let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL); + tokio::pin!(fence_watch); + let mut lock_watch = tokio::time::interval(NS_SCANNER_LOCK_POLL_INTERVAL); + lock_watch.set_missed_tick_behavior(MissedTickBehavior::Delay); + let outcome = loop { + tokio::select! { + result = &mut scan => { + match result { + Ok(outcome) => break outcome, + Err(err) => { + let disk_online = + tokio::time::timeout(NS_SCANNER_DISK_HEALTH_TIMEOUT, crate::scanner_disk_is_online(disk.as_ref())) + .await + .unwrap_or(false); + return Err(RemoteScannerServerError::disk_scan( + ScannerError::Other(format!("remote namespace scanner disk scan failed: {err}")), + disk_online, + )); + } + } + } + _ = lock_watch.tick() => { + if guard.is_lock_lost() { + scan_ctx.cancel(); + let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await; + return Err(RemoteScannerServerError::worker( + "remote namespace scanner cache lock was lost during bucket scan", + )); + } + } + result = &mut fence_watch => { + scan_ctx.cancel(); + let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await; + let err = match result { + Ok(()) => ScannerError::Other("remote namespace scanner fence watcher stopped unexpectedly".to_string()), + Err(err) => err, + }; + return Err(RemoteScannerServerError::worker(format!( + "remote namespace scanner leader fence changed during bucket scan: {}", + err + ))); + } + } + }; + let (cache, partial_result) = match outcome { + ScannerDiskScanOutcome::Complete(cache) => (cache, None), + ScannerDiskScanOutcome::Partial(cache) => (cache, Some(RemoteScannerFrameResult::Partial)), + ScannerDiskScanOutcome::NamespaceNotFound(cache) => (cache, Some(RemoteScannerFrameResult::NamespaceNotFound)), + }; + + if guard.is_lock_lost() { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner cache lock was lost before persistence", + )); + } + phase.store(RemoteScannerPhase::Persisting as u8, Ordering::Release); + phase_changed.notify_one(); + validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store.clone()) + .await + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; + let done_save = Metrics::time(Metric::SaveUsage); + let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await; + done_save(); + save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?; + validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store) + .await + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; + if guard.is_lock_lost() { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner cache lock was lost during persistence", + )); + } + + if let Some(partial_result) = partial_result { + return Ok(partial_result); + } + if cache.info.source != Some(source) || !cache.info.snapshot_complete { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner completed without a complete source snapshot", + )); + } + if cache.info.scan_plan_digest != Some(scan_plan_digest) { + return Err(RemoteScannerServerError::worker( + "remote namespace scanner completed with a different bucket plan", + )); + } + Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source, + scan_plan_digest, + usage: cache_root_entry_info(&cache) + .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?, + pending_maintenance_work: !cache.info.pending_heals.is_empty(), + }))) +} + +pub(crate) async fn scan_remote_bucket( + disk: &Disk, + ctx: CancellationToken, + budget: Arc, + spec: RemoteScannerScanSpec<'_>, +) -> std::result::Result { + let RemoteScannerScanSpec { + bucket, + next_cycle, + leader_epoch, + server_epoch, + session_id, + session_sequence, + scan_plan_digest, + skip_healing, + scan_mode, + } = spec; + let expected_source = cache_source_for_disk(disk).map_err(|err| { + RemoteScannerFailure::transport(StorageError::other(format!("failed to resolve remote namespace scanner source: {err}"))) + })?; + let request_id = Uuid::new_v4(); + let request = RemoteScannerRequestWire { + version: NS_SCANNER_PROTOCOL_VERSION, + request_id, + server_epoch, + session_id, + session_sequence, + bucket: bucket.to_string(), + next_cycle, + leader_epoch, + scan_plan_digest, + skip_healing, + scan_mode, + budget: RemoteScannerBudget::from_config(budget.remaining_config()), + }; + let body = rmp_serde::to_vec_named(&request).map_err(|err| { + RemoteScannerFailure::transport(StorageError::other(format!("failed to encode remote namespace scanner request: {err}"))) + })?; + if body.is_empty() || body.len() > NS_SCANNER_MAX_REQUEST_BODY_SIZE { + return Err(RemoteScannerFailure::transport(StorageError::other( + "remote namespace scanner request size is invalid", + ))); + } + + let rpc_deadline = Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME; + let open_deadline = (Instant::now() + NS_SCANNER_STALL_TIMEOUT).min(rpc_deadline); + let open_stream = disk.open_ns_scanner_stream(crate::NsScannerOpenRequest { + request_id, + server_epoch, + session_id, + session_sequence, + next_cycle, + leader_epoch, + body, + stall_timeout: Some(NS_SCANNER_STALL_TIMEOUT), + }); + tokio::pin!(open_stream); + let reader = tokio::select! { + _ = ctx.cancelled() => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other( + "remote namespace scanner cancelled while opening stream", + ))); + } + result = tokio::time::timeout_at(open_deadline, &mut open_stream) => { + match result { + Err(_) => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other( + "remote namespace scanner response headers timed out", + ))); + } + Ok(Err(err)) if err.is_internode_http_status(429) => { + return Err(RemoteScannerFailure::retry_bucket(StorageError::other(format!( + "remote namespace scanner worker rejected zero-progress work: {err}" + )))); + } + Ok(Err(err)) => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other(format!( + "failed to open remote namespace scanner stream: {err}" + )))); + } + Ok(Ok(reader)) => reader, + } + } + }; + let authenticator = FrameAuthenticator::from_rpc_secret(request_id).map_err(|err| { + RemoteScannerFailure::transport(StorageError::other(format!( + "failed to authenticate remote namespace scanner stream: {err}" + ))) + })?; + + let stream_result = consume_remote_scanner_stream_until( + reader, + ctx, + budget.clone(), + RemoteScannerResponseExpectation { + bucket, + source: expected_source, + next_cycle, + scan_plan_digest, + }, + authenticator, + rpc_deadline, + ) + .await; + finish_remote_scanner_stream(stream_result, budget.as_ref()) +} + +fn finish_remote_scanner_stream( + result: RemoteScannerStreamResult, + budget: &ScannerCycleBudget, +) -> std::result::Result { + match result { + Ok(outcome) => Ok(outcome), + Err(error) => { + if !error.progress_fully_reported { + budget.cancel_after_unreported_remote_progress(); + } + let failure = if error.retire_worker { + RemoteScannerFailure::transport(error.error) + } else { + RemoteScannerFailure::bucket(error.error) + }; + Err(failure) + } + } +} + +#[cfg(test)] +const TEST_NEXT_CYCLE: u64 = 11; + +#[cfg(test)] +async fn consume_remote_scanner_stream( + reader: R, + ctx: CancellationToken, + budget: Arc, + expected_bucket: &str, + expected_source: DataUsageCacheSource, + expected_scan_plan_digest: DataUsageScanPlanDigest, + authenticator: FrameAuthenticator, +) -> RemoteScannerStreamResult +where + R: AsyncRead + Unpin, +{ + consume_remote_scanner_stream_until( + reader, + ctx, + budget, + RemoteScannerResponseExpectation { + bucket: expected_bucket, + source: expected_source, + next_cycle: TEST_NEXT_CYCLE, + scan_plan_digest: expected_scan_plan_digest, + }, + authenticator, + Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME, + ) + .await +} + +async fn consume_remote_scanner_stream_until( + mut reader: R, + ctx: CancellationToken, + budget: Arc, + expected: RemoteScannerResponseExpectation<'_>, + authenticator: FrameAuthenticator, + rpc_deadline: Instant, +) -> RemoteScannerStreamResult +where + R: AsyncRead + Unpin, +{ + let mut expected_sequence = 0_u64; + let mut last_progress = RemoteScannerProgress::default(); + let mut last_phase = RemoteScannerPhase::Scanning; + let mut semantic_progress_deadline = + bounded_remote_scanner_deadline(Instant::now(), NS_SCANNER_SEMANTIC_STALL_TIMEOUT, rpc_deadline); + + loop { + if ctx.is_cancelled() && budget.reason().is_none() { + return Err(RemoteScannerStreamError::for_phase( + StorageError::other("remote namespace scanner cancelled"), + last_phase, + )); + } + let lifetime_limited = rpc_deadline <= semantic_progress_deadline; + let read_deadline = rpc_deadline.min(semantic_progress_deadline); + let frame = tokio::select! { + biased; + _ = ctx.cancelled(), if budget.reason().is_none() => { + return Err(RemoteScannerStreamError::for_phase( + StorageError::other("remote namespace scanner cancelled"), + last_phase, + )); + } + result = tokio::time::timeout_at( + read_deadline, + read_frame(&mut reader, &authenticator, &mut expected_sequence), + ) => match result { + Ok(Ok(frame)) => frame, + Ok(Err(err)) => { + return Err(RemoteScannerStreamError::for_phase(StorageError::other(err), last_phase)); + } + Err(_) => { + let message = if lifetime_limited { + "remote namespace scanner RPC lifetime exceeded" + } else { + "remote namespace scanner made no semantic progress" + }; + return Err(RemoteScannerStreamError::for_phase(StorageError::other(message), last_phase)); + } + } + }; + + let advanced = apply_remote_progress(&budget, &mut last_progress, frame.progress) + .map_err(|err| RemoteScannerStreamError::for_phase(err, last_phase))?; + if last_phase == RemoteScannerPhase::Persisting && frame.phase == RemoteScannerPhase::Scanning { + return Err(RemoteScannerStreamError::uncertain(StorageError::other( + "remote namespace scanner phase moved backwards", + ))); + } + let phase_advanced = frame.phase != last_phase; + last_phase = frame.phase; + if phase_advanced && frame.phase == RemoteScannerPhase::Persisting { + semantic_progress_deadline = + bounded_remote_scanner_deadline(Instant::now(), DataUsageCache::persistence_timeout(), rpc_deadline); + } else if advanced { + semantic_progress_deadline = + bounded_remote_scanner_deadline(Instant::now(), NS_SCANNER_SEMANTIC_STALL_TIMEOUT, rpc_deadline); + } + + match frame.result { + RemoteScannerFrameResult::Progress => { + if budget.budget_elapsed() && frame.phase == RemoteScannerPhase::Scanning { + return Ok(RemoteScannerOutcome::Partial); + } + } + RemoteScannerFrameResult::Complete(complete) => { + if complete.usage.name != expected.bucket || complete.usage.parent != crate::DATA_USAGE_ROOT { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned usage for the wrong bucket", + ))); + } + if complete.source != expected.source { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned usage for the wrong pool or set", + ))); + } + if complete.scan_plan_digest != expected.scan_plan_digest { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned usage for a different bucket plan", + ))); + } + if !complete.usage.entry.children.is_empty() { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned non-flattened bucket usage", + ))); + } + if budget.budget_elapsed() { + return Ok(RemoteScannerOutcome::Partial); + } + return Ok(RemoteScannerOutcome::Complete { + usage: Box::new(complete.usage), + pending_maintenance_work: complete.pending_maintenance_work, + }); + } + RemoteScannerFrameResult::Partial => return Ok(RemoteScannerOutcome::Partial), + RemoteScannerFrameResult::NamespaceNotFound => return Ok(RemoteScannerOutcome::NamespaceNotFound), + RemoteScannerFrameResult::CycleAhead { required_cycle } => { + if required_cycle <= expected.next_cycle || required_cycle == u64::MAX { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned an invalid required cycle", + ))); + } + return Ok(RemoteScannerOutcome::CycleAhead(required_cycle)); + } + RemoteScannerFrameResult::Error(error_frame) => { + let error = + StorageError::other(format!("remote namespace scanner failed: {}", limit_error_message(error_frame.message))); + return Err(match error_frame.scope { + RemoteScannerErrorScope::Bucket => RemoteScannerStreamError::bucket(error), + RemoteScannerErrorScope::Worker => RemoteScannerStreamError::reconciled(error), + }); + } + } + } +} + +fn bounded_remote_scanner_deadline(now: Instant, timeout_duration: Duration, rpc_deadline: Instant) -> Instant { + now.checked_add(timeout_duration).unwrap_or(rpc_deadline).min(rpc_deadline) +} + +fn apply_remote_progress( + budget: &ScannerCycleBudget, + last: &mut RemoteScannerProgress, + current: RemoteScannerProgress, +) -> std::result::Result { + if current.objects_scanned < last.objects_scanned + || current.directories_started < last.directories_started + || current.entries_visited < last.entries_visited + { + return Err(StorageError::other("remote namespace scanner progress moved backwards")); + } + + let advanced = current != *last; + budget.record_remote_progress( + current.objects_scanned - last.objects_scanned, + current.directories_started - last.directories_started, + ); + *last = current; + Ok(advanced) +} + +fn budget_progress(budget: &ScannerCycleBudget) -> RemoteScannerProgress { + let (objects_scanned, directories_started) = budget.progress(); + RemoteScannerProgress { + objects_scanned, + directories_started, + entries_visited: budget.entries_visited(), + } +} + +fn remote_scanner_phase(phase: &AtomicU8) -> RemoteScannerPhase { + if phase.load(Ordering::Acquire) == RemoteScannerPhase::Persisting as u8 { + RemoteScannerPhase::Persisting + } else { + RemoteScannerPhase::Scanning + } +} + +async fn await_remote_scan_shutdown(scan: Pin<&mut F>) +where + F: Future, +{ + let grace = DataUsageCache::persistence_timeout().min(NS_SCANNER_DISCONNECT_GRACE_MAX); + let _ = tokio::time::timeout(grace, scan).await; +} + +fn disconnected_writer_error() -> ScannerError { + IoError::new(ErrorKind::ConnectionAborted, "remote namespace scanner response disconnected").into() +} + +fn writer_timeout_error(operation: &str, timeout_duration: Duration) -> ScannerError { + ScannerError::Other(format!( + "remote namespace scanner response {operation} timed out after {timeout_duration:?}" + )) +} + +async fn write_frame_bounded( + writer: &mut W, + authenticator: &FrameAuthenticator, + sequence: &mut u64, + frame: &RemoteScannerFrame, + disconnect: &CancellationToken, + timeout_duration: Duration, +) -> Result<(), ScannerError> +where + W: AsyncWrite + Unpin, +{ + tokio::select! { + biased; + _ = disconnect.cancelled() => Err(disconnected_writer_error()), + result = tokio::time::timeout(timeout_duration, write_frame(writer, authenticator, sequence, frame)) => { + result.map_err(|_| writer_timeout_error("write", timeout_duration))? + } + } +} + +async fn flush_writer_bounded( + writer: &mut W, + disconnect: &CancellationToken, + timeout_duration: Duration, +) -> Result<(), ScannerError> +where + W: AsyncWrite + Unpin, +{ + tokio::select! { + biased; + _ = disconnect.cancelled() => Err(disconnected_writer_error()), + result = tokio::time::timeout(timeout_duration, writer.flush()) => { + result + .map_err(|_| writer_timeout_error("flush", timeout_duration))? + .map_err(ScannerError::from) + } + } +} + +async fn shutdown_writer_bounded( + writer: &mut W, + disconnect: &CancellationToken, + timeout_duration: Duration, +) -> Result<(), ScannerError> +where + W: AsyncWrite + Unpin, +{ + tokio::select! { + biased; + _ = disconnect.cancelled() => Err(disconnected_writer_error()), + result = tokio::time::timeout(timeout_duration, writer.shutdown()) => { + result + .map_err(|_| writer_timeout_error("shutdown", timeout_duration))? + .map_err(ScannerError::from) + } + } +} + +async fn write_frame( + writer: &mut W, + authenticator: &FrameAuthenticator, + sequence: &mut u64, + frame: &RemoteScannerFrame, +) -> Result<(), ScannerError> +where + W: AsyncWrite + Unpin, +{ + let payload = rmp_serde::to_vec_named(frame) + .map_err(|err| ScannerError::Other(format!("failed to encode remote namespace scanner frame: {err}")))?; + let envelope = RemoteScannerFrameEnvelope { + version: NS_SCANNER_PROTOCOL_VERSION, + sequence: *sequence, + mac: authenticator.sign(*sequence, &payload)?, + payload, + }; + let encoded = rmp_serde::to_vec_named(&envelope) + .map_err(|err| ScannerError::Other(format!("failed to encode remote namespace scanner envelope: {err}")))?; + if encoded.is_empty() || encoded.len() > NS_SCANNER_MAX_FRAME_SIZE { + return Err(ScannerError::Other(format!( + "remote namespace scanner frame size {} is invalid", + encoded.len() + ))); + } + let len = u32::try_from(encoded.len()) + .map_err(|_| ScannerError::Other("remote namespace scanner frame is too large".to_string()))?; + + writer.write_all(&len.to_be_bytes()).await?; + writer.write_all(&encoded).await?; + *sequence = sequence + .checked_add(1) + .ok_or_else(|| ScannerError::Other("remote namespace scanner frame sequence overflow".to_string()))?; + Ok(()) +} + +async fn read_frame( + reader: &mut R, + authenticator: &FrameAuthenticator, + expected_sequence: &mut u64, +) -> std::io::Result +where + R: AsyncRead + Unpin, +{ + let mut len = [0_u8; 4]; + reader.read_exact(&mut len).await.map_err(|err| { + if err.kind() == ErrorKind::UnexpectedEof { + IoError::new(ErrorKind::UnexpectedEof, "remote namespace scanner stream ended before a terminal frame") + } else { + err + } + })?; + let len = usize::try_from(u32::from_be_bytes(len)) + .map_err(|_| IoError::new(ErrorKind::InvalidData, "remote namespace scanner frame length is invalid"))?; + if len == 0 || len > NS_SCANNER_MAX_FRAME_SIZE { + return Err(IoError::new( + ErrorKind::InvalidData, + format!("remote namespace scanner frame size {len} is invalid"), + )); + } + + let mut encoded = vec![0_u8; len]; + reader.read_exact(&mut encoded).await?; + let envelope: RemoteScannerFrameEnvelope = rmp_serde::from_slice(&encoded) + .map_err(|_| IoError::new(ErrorKind::InvalidData, "invalid remote namespace scanner frame envelope"))?; + if envelope.version != NS_SCANNER_PROTOCOL_VERSION { + return Err(IoError::new( + ErrorKind::InvalidData, + format!("unsupported remote namespace scanner frame version: {}", envelope.version), + )); + } + if envelope.sequence != *expected_sequence { + return Err(IoError::new(ErrorKind::InvalidData, "remote namespace scanner frame sequence is invalid")); + } + authenticator.verify(envelope.sequence, &envelope.payload, &envelope.mac)?; + let frame: RemoteScannerFrame = rmp_serde::from_slice(&envelope.payload) + .map_err(|_| IoError::new(ErrorKind::InvalidData, "invalid remote namespace scanner frame payload"))?; + *expected_sequence = expected_sequence + .checked_add(1) + .ok_or_else(|| IoError::new(ErrorKind::InvalidData, "remote namespace scanner frame sequence overflow"))?; + Ok(frame) +} + +fn limit_error_message(message: String) -> String { + message.chars().take(NS_SCANNER_MAX_ERROR_CHARS).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use std::sync::RwLock; + use std::sync::atomic::AtomicUsize; + + const TEST_SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(1, 2); + const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([5; 32]); + + #[test] + fn semantic_deadline_is_capped_by_rpc_lifetime_without_overflow() { + let now = Instant::now(); + let rpc_deadline = now + Duration::from_secs(10); + + assert_eq!(bounded_remote_scanner_deadline(now, Duration::MAX, rpc_deadline), rpc_deadline); + assert_eq!( + bounded_remote_scanner_deadline(now, Duration::from_secs(2), rpc_deadline), + now + Duration::from_secs(2) + ); + } + + #[derive(Debug)] + struct CycleStateStore { + state: Option>, + } + + #[derive(Debug)] + struct CountingCycleStateStore { + state: Vec, + reads: AtomicUsize, + } + + #[derive(Debug)] + struct MutableCycleStateStore { + state: RwLock>, + } + + #[async_trait::async_trait] + impl crate::storage_api::owner::ObjectIO for CycleStateStore { + type Error = EcstoreError; + type RangeSpec = crate::storage_api::owner::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> Result { + if bucket != RUSTFS_META_BUCKET || object != DATA_USAGE_BLOOM_NAME_PATH.as_str() { + return Err(EcstoreError::FileNotFound); + } + let state = self.state.clone().ok_or(EcstoreError::FileNotFound)?; + Ok(crate::ScannerGetObjectReader { + stream: Box::new(Cursor::new(state)), + object_info: crate::ScannerObjectInfo { + etag: Some("cycle-state".to_string()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + _bucket: &str, + _object: &str, + _data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> Result { + Err(EcstoreError::other("cycle state test store is read-only")) + } + } + + #[async_trait::async_trait] + impl crate::storage_api::owner::ObjectIO for CountingCycleStateStore { + type Error = EcstoreError; + type RangeSpec = crate::storage_api::owner::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> Result { + if bucket != RUSTFS_META_BUCKET || object != DATA_USAGE_BLOOM_NAME_PATH.as_str() { + return Err(EcstoreError::FileNotFound); + } + self.reads.fetch_add(1, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(25)).await; + Ok(crate::ScannerGetObjectReader { + stream: Box::new(Cursor::new(self.state.clone())), + object_info: crate::ScannerObjectInfo { + etag: Some("cycle-state".to_string()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + _bucket: &str, + _object: &str, + _data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> Result { + Err(EcstoreError::other("cycle state test store is read-only")) + } + } + + #[async_trait::async_trait] + impl crate::storage_api::owner::ObjectIO for MutableCycleStateStore { + type Error = EcstoreError; + type RangeSpec = crate::storage_api::owner::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> Result { + if bucket != RUSTFS_META_BUCKET || object != DATA_USAGE_BLOOM_NAME_PATH.as_str() { + return Err(EcstoreError::FileNotFound); + } + let state = self + .state + .read() + .map_err(|_| EcstoreError::other("cycle state test lock is poisoned"))? + .clone(); + Ok(crate::ScannerGetObjectReader { + stream: Box::new(Cursor::new(state)), + object_info: crate::ScannerObjectInfo { + etag: Some("cycle-state".to_string()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + _bucket: &str, + _object: &str, + _data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> Result { + Err(EcstoreError::other("cycle state test store is read-only")) + } + } + + fn test_usage(bucket: &str, objects: usize) -> DataUsageEntryInfo { + let entry = crate::DataUsageEntry { + objects, + ..Default::default() + }; + DataUsageEntryInfo { + name: bucket.to_string(), + parent: crate::DATA_USAGE_ROOT.to_string(), + entry, + } + } + + fn test_request(request_id: Uuid) -> RemoteScannerRequestWire { + RemoteScannerRequestWire { + version: NS_SCANNER_PROTOCOL_VERSION, + request_id, + server_epoch: Uuid::new_v4(), + session_id: Uuid::new_v4(), + session_sequence: 0, + bucket: "bucket".to_string(), + next_cycle: 7, + leader_epoch: 9, + scan_plan_digest: TEST_PLAN_DIGEST, + skip_healing: true, + scan_mode: HealScanMode::Deep, + budget: RemoteScannerBudget { + max_duration_ms: Some(1234), + max_objects: Some(10), + max_directories: Some(20), + }, + } + } + + #[test] + fn request_round_trip_preserves_scan_inputs_without_cache_data() { + let request_id = Uuid::new_v4(); + let body = rmp_serde::to_vec_named(&test_request(request_id)).expect("request should encode"); + let decoded = decode_remote_scanner_request(&body).expect("request should decode"); + + assert_eq!(decoded.0.request_id, request_id); + assert!(!decoded.0.server_epoch.is_nil()); + assert!(!decoded.0.session_id.is_nil()); + assert_eq!(decoded.0.session_sequence, 0); + assert_eq!(decoded.0.bucket, "bucket"); + assert_eq!(decoded.0.next_cycle, 7); + assert_eq!(decoded.0.leader_epoch, 9); + assert_eq!(decoded.0.scan_plan_digest, TEST_PLAN_DIGEST); + assert!(decoded.0.skip_healing); + assert_eq!(decoded.0.scan_mode, HealScanMode::Deep); + assert_eq!(decoded.0.budget.max_objects, Some(10)); + assert_eq!(decoded.0.budget.max_directories, Some(20)); + assert!(body.len() < 512); + } + + #[test] + fn request_envelope_must_match_every_pre_body_replay_field() { + let request = RemoteScannerRequest(test_request(Uuid::new_v4())); + assert!(remote_scanner_request_matches_envelope( + &request, + request.0.request_id, + request.0.server_epoch, + request.0.session_id, + request.0.session_sequence, + request.0.next_cycle, + request.0.leader_epoch, + )); + assert!(!remote_scanner_request_matches_envelope( + &request, + request.0.request_id, + Uuid::new_v4(), + request.0.session_id, + request.0.session_sequence, + request.0.next_cycle, + request.0.leader_epoch, + )); + assert!(!remote_scanner_request_matches_envelope( + &request, + request.0.request_id, + request.0.server_epoch, + request.0.session_id, + request.0.session_sequence + 1, + request.0.next_cycle, + request.0.leader_epoch, + )); + assert!(!remote_scanner_request_matches_envelope( + &request, + request.0.request_id, + request.0.server_epoch, + request.0.session_id, + request.0.session_sequence, + request.0.next_cycle, + request.0.leader_epoch + 1, + )); + } + + #[tokio::test] + async fn bounded_frame_write_releases_a_stalled_response() { + let request_id = Uuid::new_v4(); + let authenticator = FrameAuthenticator::for_test(request_id); + let (_reader, mut writer) = tokio::io::duplex(1); + let disconnect = CancellationToken::new(); + let mut sequence = 0; + + let error = write_frame_bounded( + &mut writer, + &authenticator, + &mut sequence, + &RemoteScannerFrame::progress(RemoteScannerProgress::default()), + &disconnect, + Duration::from_millis(10), + ) + .await + .expect_err("a response with no reader must time out"); + + assert!(error.to_string().contains("timed out")); + assert_eq!(sequence, 0); + } + + #[tokio::test] + async fn external_cancellation_interrupts_a_stalled_frame_read() { + let request_id = Uuid::new_v4(); + let authenticator = FrameAuthenticator::for_test(request_id); + let (_writer, reader) = tokio::io::duplex(1); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let cancel = parent.clone(); + let task = tokio::spawn(async move { + consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, authenticator).await + }); + + tokio::task::yield_now().await; + cancel.cancel(); + let error = tokio::time::timeout(Duration::from_millis(100), task) + .await + .expect("cancelled frame read should finish promptly") + .expect("cancelled frame read task should not panic") + .expect_err("cancelled frame read must fail"); + + assert!(error.to_string().contains("cancelled")); + } + + #[test] + fn request_decode_rejects_unknown_fields_without_a_protocol_bump() { + #[derive(Serialize)] + struct AdditiveRequestWire { + version: u16, + request_id: Uuid, + server_epoch: Uuid, + session_id: Uuid, + session_sequence: u64, + bucket: String, + next_cycle: u64, + leader_epoch: u64, + scan_plan_digest: DataUsageScanPlanDigest, + skip_healing: bool, + scan_mode: HealScanMode, + budget: RemoteScannerBudget, + future_optional_hint: bool, + } + + let request_id = Uuid::new_v4(); + let body = rmp_serde::to_vec_named(&AdditiveRequestWire { + version: NS_SCANNER_PROTOCOL_VERSION, + request_id, + server_epoch: Uuid::new_v4(), + session_id: Uuid::new_v4(), + session_sequence: 0, + bucket: "bucket".to_string(), + next_cycle: 7, + leader_epoch: 9, + scan_plan_digest: TEST_PLAN_DIGEST, + skip_healing: true, + scan_mode: HealScanMode::Deep, + budget: RemoteScannerBudget::default(), + future_optional_hint: true, + }) + .expect("additive request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + } + + #[test] + fn request_rejects_invalid_bucket_and_nil_ids() { + let mut invalid_bucket = test_request(Uuid::new_v4()); + invalid_bucket.bucket = "../bucket".to_string(); + let body = rmp_serde::to_vec_named(&invalid_bucket).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + + let body = rmp_serde::to_vec_named(&test_request(Uuid::nil())).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + + let mut nil_epoch = test_request(Uuid::new_v4()); + nil_epoch.server_epoch = Uuid::nil(); + let body = rmp_serde::to_vec_named(&nil_epoch).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + + let mut nil_session = test_request(Uuid::new_v4()); + nil_session.session_id = Uuid::nil(); + let body = rmp_serde::to_vec_named(&nil_session).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + + let mut zero_leader_epoch = test_request(Uuid::new_v4()); + zero_leader_epoch.leader_epoch = 0; + let body = rmp_serde::to_vec_named(&zero_leader_epoch).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + } + + #[test] + fn request_rejects_empty_truncated_oversized_and_wrong_version_payloads() { + assert!(decode_remote_scanner_request(&[]).is_err()); + + let mut body = rmp_serde::to_vec_named(&test_request(Uuid::new_v4())).expect("request should encode"); + body.truncate(body.len() / 2); + assert!(decode_remote_scanner_request(&body).is_err()); + + let oversized = vec![0_u8; NS_SCANNER_MAX_REQUEST_BODY_SIZE + 1]; + assert!(decode_remote_scanner_request(&oversized).is_err()); + + for version in [NS_SCANNER_PROTOCOL_VERSION - 1, NS_SCANNER_PROTOCOL_VERSION + 1] { + let mut wrong_version = test_request(Uuid::new_v4()); + wrong_version.version = version; + let body = rmp_serde::to_vec_named(&wrong_version).expect("request should encode"); + assert!(decode_remote_scanner_request(&body).is_err()); + } + } + + #[test] + fn disk_scan_error_scope_distinguishes_bucket_failures_from_offline_workers() { + let bucket_error = RemoteScannerServerError::disk_scan(ScannerError::Other("metadata corrupt".to_string()), true); + let worker_error = RemoteScannerServerError::disk_scan(ScannerError::Other("disk offline".to_string()), false); + + assert_eq!(bucket_error.scope, RemoteScannerErrorScope::Bucket); + assert_eq!(worker_error.scope, RemoteScannerErrorScope::Worker); + } + + #[test] + fn persisted_cycle_decoder_requires_a_valid_leader_fence() { + let extended = crate::scanner::encode_scanner_cycle_fence_for_test(42, 9); + + assert_eq!( + crate::scanner::decode_persisted_scanner_cycle_fence(&extended).expect("fenced cycle"), + (42, 9) + ); + assert_eq!( + crate::scanner::decode_persisted_scanner_cycle_fence(&42_u64.to_le_bytes()).expect("legacy cycle"), + (42, 0) + ); + assert!(crate::scanner::decode_persisted_scanner_cycle_fence(&[0_u8; 7]).is_err()); + } + + #[tokio::test] + async fn request_cycle_validation_reads_persisted_store_state() { + let request = RemoteScannerRequest(test_request(Uuid::new_v4())); + let store = Arc::new(CycleStateStore { + state: Some(crate::scanner::encode_scanner_cycle_fence_for_test( + request.0.next_cycle, + request.0.leader_epoch, + )), + }); + assert_eq!( + validate_remote_scanner_request_fence_with_store(request.0.next_cycle, request.0.leader_epoch, store) + .await + .expect("matching persisted fence should validate"), + (7, 9) + ); + + let stale_store = Arc::new(CycleStateStore { + state: Some(crate::scanner::encode_scanner_cycle_fence_for_test(8, request.0.leader_epoch)), + }); + let err = validate_remote_scanner_request_fence_with_store(request.0.next_cycle, request.0.leader_epoch, stale_store) + .await + .expect_err("mismatched persisted cycle must be rejected"); + assert!(err.to_string().contains("requested_cycle=7")); + + let mut initial_request = test_request(Uuid::new_v4()); + initial_request.next_cycle = 0; + initial_request.leader_epoch = 0; + assert_eq!( + validate_remote_scanner_request_fence_with_store( + initial_request.next_cycle, + initial_request.leader_epoch, + Arc::new(CycleStateStore { state: None }), + ) + .await + .expect("missing state should validate only the initial fence"), + (0, 0) + ); + } + + #[tokio::test] + async fn fence_watcher_stops_work_after_persisted_leader_changes() { + let store = Arc::new(MutableCycleStateStore { + state: RwLock::new(crate::scanner::encode_scanner_cycle_fence_for_test(7, 9)), + }); + let cache = Arc::new(Mutex::new(None)); + let refresh = Arc::new(AsyncMutex::new(())); + let watcher = tokio::spawn({ + let cache = cache.clone(); + let refresh = refresh.clone(); + let store = store.clone(); + async move { + watch_remote_scanner_request_fence_with_cache( + 7, + 9, + store, + Duration::from_millis(5), + cache.as_ref(), + refresh.as_ref(), + Duration::from_millis(5), + ) + .await + } + }); + + tokio::time::sleep(Duration::from_millis(20)).await; + *store.state.write().expect("cycle state test lock should remain available") = + crate::scanner::encode_scanner_cycle_fence_for_test(7, 10); + + let err = tokio::time::timeout(Duration::from_millis(100), watcher) + .await + .expect("fence watcher should observe the leader change") + .expect("fence watcher task should not panic") + .expect_err("a replacement leader must cancel old scanner work"); + assert!(err.to_string().contains("requested_epoch=9")); + } + + #[tokio::test] + async fn concurrent_cycle_cache_misses_share_one_backend_read() { + let request = RemoteScannerRequest(test_request(Uuid::new_v4())); + let store = Arc::new(CountingCycleStateStore { + state: crate::scanner::encode_scanner_cycle_fence_for_test(request.0.next_cycle, request.0.leader_epoch), + reads: AtomicUsize::new(0), + }); + let cache = Mutex::new(None); + let refresh = AsyncMutex::new(()); + + let results = futures::future::join_all((0..16).map(|_| { + validate_remote_scanner_request_fence_cached( + request.0.next_cycle, + request.0.leader_epoch, + store.clone(), + &cache, + &refresh, + NS_SCANNER_VALIDATED_CYCLE_TTL, + ) + })) + .await; + + assert!(results.into_iter().all(|result| result.is_ok())); + assert_eq!(store.reads.load(Ordering::Relaxed), 1); + } + + #[test] + fn remote_scanner_admission_allows_one_request_per_disk() { + let key = format!("test-disk-{}", Uuid::new_v4()); + let first = try_admit_remote_scanner_key(key.clone()).expect("first request should be admitted"); + assert!(try_admit_remote_scanner_key(key.clone()).is_err()); + drop(first); + assert!(try_admit_remote_scanner_key(key).is_ok()); + } + + #[test] + fn admission_rejects_busy_disk_before_replay_state_changes() { + let key = format!("test-disk-{}", Uuid::new_v4()); + let active = try_admit_remote_scanner_key(key.clone()).expect("first request should be admitted"); + + assert!(matches!(try_admit_remote_scanner_key(key.clone()), Err(ScannerError::RemoteDiskBusy))); + drop(active); + + assert!(try_admit_remote_scanner_key(key).is_ok()); + } + + #[test] + fn replay_cache_requires_contiguous_sequence_per_disk_session() { + let session_id = Uuid::new_v4(); + let mut cache = RemoteScannerReplayCache::default(); + + cache + .claim("disk-a".to_string(), session_id, 7, 9, 0) + .expect("first session request should be accepted"); + assert!(matches!( + cache.claim("disk-a".to_string(), session_id, 7, 9, 0), + Err(ScannerError::RemoteRequestReplay) + )); + assert!(matches!( + cache.claim("disk-a".to_string(), session_id, 7, 9, 2), + Err(ScannerError::RemoteRequestReplay) + )); + cache + .claim("disk-a".to_string(), session_id, 7, 9, 1) + .expect("next contiguous session request should be accepted"); + } + + #[test] + fn replay_preflight_rejects_a_claimed_request_without_mutating_state() { + let session_id = Uuid::new_v4(); + let mut cache = RemoteScannerReplayCache::default(); + cache + .claim("disk-a".to_string(), session_id, 7, 9, 0) + .expect("first request should be accepted"); + let state_before = (cache.cycle, cache.leader_epoch, cache.sessions.clone()); + + assert!(matches!( + cache.preflight("disk-a", session_id, 7, 9, 0), + Err(ScannerError::RemoteRequestReplay) + )); + assert_eq!((cache.cycle, cache.leader_epoch, cache.sessions), state_before); + } + + #[test] + fn replay_cache_discards_sessions_from_prior_cycle() { + let session_id = Uuid::new_v4(); + let mut cache = RemoteScannerReplayCache::default(); + + cache + .claim("disk-a".to_string(), session_id, 7, 9, 0) + .expect("first cycle session should be accepted"); + cache + .claim("disk-a".to_string(), session_id, 8, 9, 0) + .expect("new cycle should start a fresh session sequence"); + assert_eq!(cache.sessions.len(), 1); + assert!(matches!( + cache.claim("disk-a".to_string(), Uuid::new_v4(), 7, 9, 0), + Err(ScannerError::RemoteRequestReplay) + )); + assert_eq!(cache.cycle, Some(8)); + assert_eq!(cache.sessions.len(), 1); + } + + #[test] + fn replay_cache_fences_sessions_by_leader_epoch() { + let mut cache = RemoteScannerReplayCache::default(); + cache + .claim("disk-a".to_string(), Uuid::new_v4(), 7, 9, 0) + .expect("first leader session should be accepted"); + cache + .claim("disk-a".to_string(), Uuid::new_v4(), 7, 10, 0) + .expect("replacement leader should reset replay state"); + + assert_eq!(cache.cycle, Some(7)); + assert_eq!(cache.leader_epoch, Some(10)); + assert_eq!(cache.sessions.len(), 1); + assert!(matches!( + cache.claim("disk-a".to_string(), Uuid::new_v4(), 8, 9, 0), + Err(ScannerError::RemoteRequestReplay) + )); + } + + #[test] + fn replay_cache_scales_with_sessions_instead_of_bucket_requests() { + let session_id = Uuid::new_v4(); + let mut cache = RemoteScannerReplayCache::default(); + let request_count = u64::try_from(NS_SCANNER_MAX_REPLAY_SESSIONS) + .expect("session limit should fit in u64") + .saturating_add(1024); + + for sequence in 0..request_count { + cache + .claim("disk-a".to_string(), session_id, 7, 9, sequence) + .expect("contiguous requests in one worker session should not consume session capacity"); + } + + assert_eq!(cache.sessions.len(), 1); + assert_eq!(cache.sessions.get(&("disk-a".to_string(), session_id)), Some(&(request_count - 1))); + } + + #[test] + fn validated_cycle_cache_expires_and_rejects_other_cycles() { + let now = Instant::now(); + let cached = RemoteScannerValidatedCycle { + cycle: 7, + leader_epoch: 9, + valid_until: now + NS_SCANNER_VALIDATED_CYCLE_TTL, + }; + + assert!(cached.matches(7, 9, now)); + assert!(!cached.matches(8, 9, now)); + assert!(!cached.matches(7, 10, now)); + assert!(!cached.matches(7, 9, now + NS_SCANNER_VALIDATED_CYCLE_TTL)); + } + + #[test] + fn replay_cache_fails_closed_at_capacity() { + let mut cache = RemoteScannerReplayCache::default(); + for _ in 0..NS_SCANNER_MAX_REPLAY_SESSIONS { + cache + .claim("disk-a".to_string(), Uuid::new_v4(), 7, 9, 0) + .expect("session should fit in replay cache"); + } + + let error = cache + .claim("disk-a".to_string(), Uuid::new_v4(), 7, 9, 0) + .expect_err("session beyond replay cache capacity must fail"); + assert!(matches!(error, ScannerError::RemoteReplayCapacity)); + } + + #[tokio::test] + async fn complete_terminal_frame_reconciles_progress_and_usage() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::progress(RemoteScannerProgress { + objects_scanned: 2, + directories_started: 1, + ..Default::default() + }), + ) + .await + .expect("progress should write"); + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: 3, + directories_started: 2, + ..Default::default() + }, + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage: test_usage("bucket", 3), + pending_maintenance_work: true, + })), + ), + ) + .await + .expect("terminal frame should write"); + writer.shutdown().await.expect("writer should shut down"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + max_directories: Some(10), + ..Default::default() + }, + ); + let outcome = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("stream should complete"); + + let RemoteScannerOutcome::Complete { + usage, + pending_maintenance_work, + } = outcome + else { + panic!("expected complete result"); + }; + assert_eq!(usage.entry.objects, 3); + assert!(pending_maintenance_work); + assert_eq!(budget.progress(), (3, 2)); + } + + #[tokio::test] + async fn complete_terminal_frame_after_budget_expiry_is_partial() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: 3, + directories_started: 1, + ..Default::default() + }, + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage: test_usage("bucket", 3), + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(3), + ..Default::default() + }, + ); + let outcome = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("stream should finish as partial"); + + assert!(matches!(outcome, RemoteScannerOutcome::Partial)); + assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); + } + + #[tokio::test] + async fn partial_terminal_frame_is_reported_without_usage() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: 4, + directories_started: 2, + ..Default::default() + }, + RemoteScannerFrameResult::Partial, + ), + ) + .await + .expect("partial terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default()); + let outcome = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("partial terminal frame should be accepted"); + + assert!(matches!(outcome, RemoteScannerOutcome::Partial)); + assert_eq!(budget.progress(), (4, 2)); + } + + #[tokio::test] + async fn namespace_not_found_terminal_frame_is_reported_without_usage() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: 4, + directories_started: 2, + ..Default::default() + }, + RemoteScannerFrameResult::NamespaceNotFound, + ), + ) + .await + .expect("namespace-not-found terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default()); + let outcome = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("namespace-not-found terminal frame should be accepted"); + + assert!(matches!(outcome, RemoteScannerOutcome::NamespaceNotFound)); + assert_eq!(budget.progress(), (4, 2)); + } + + #[tokio::test] + async fn error_terminal_frame_is_reported_as_failure() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Error(RemoteScannerErrorFrame { + scope: RemoteScannerErrorScope::Bucket, + message: "cache save failed".to_string(), + }), + ), + ) + .await + .expect("error terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + ..Default::default() + }, + ); + let stream_result = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await; + let error = finish_remote_scanner_stream(stream_result, budget.as_ref()).expect_err("error terminal frame must fail"); + + assert!(error.to_string().contains("cache save failed")); + assert!(!error.retire_worker()); + assert!(!budget.budget_elapsed()); + } + + #[tokio::test] + async fn worker_error_terminal_frame_retires_worker_without_cancelling_reported_budget() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Error(RemoteScannerErrorFrame { + scope: RemoteScannerErrorScope::Worker, + message: "object layer unavailable".to_string(), + }), + ), + ) + .await + .expect("error terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + ..Default::default() + }, + ); + let stream_result = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await; + let error = finish_remote_scanner_stream(stream_result, budget.as_ref()).expect_err("worker error must fail"); + + assert!(error.to_string().contains("object layer unavailable")); + assert!(error.retire_worker()); + assert!(!error.retry_bucket_work()); + assert!(!budget.budget_elapsed()); + } + + #[test] + fn uncertain_stream_failure_cancels_count_budget() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + ..Default::default() + }, + ); + let result = Err(RemoteScannerStreamError::uncertain(StorageError::other("connection lost"))); + + let error = finish_remote_scanner_stream(result, budget.as_ref()).expect_err("transport failure must fail"); + + assert!(error.to_string().contains("connection lost")); + assert!(error.retire_worker()); + assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); + } + + #[test] + fn directory_entry_activity_counts_as_semantic_progress() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let mut last = RemoteScannerProgress::default(); + + let advanced = apply_remote_progress( + budget.as_ref(), + &mut last, + RemoteScannerProgress { + entries_visited: 32, + ..Default::default() + }, + ) + .expect("entry activity should be valid progress"); + + assert!(advanced); + assert_eq!(last.entries_visited, 32); + assert_eq!(budget.progress(), (0, 0)); + } + + #[tokio::test] + async fn disconnect_after_persisting_keeps_reported_count_budget() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::with_phase( + RemoteScannerProgress { + objects_scanned: 3, + directories_started: 2, + ..Default::default() + }, + RemoteScannerPhase::Persisting, + RemoteScannerFrameResult::Progress, + ), + ) + .await + .expect("persisting progress should write"); + writer.shutdown().await.expect("writer should shut down"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + max_directories: Some(10), + ..Default::default() + }, + ); + let stream_result = + consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await; + + finish_remote_scanner_stream(stream_result, budget.as_ref()) + .expect_err("disconnect before terminal persistence result must fail"); + assert_eq!(budget.progress(), (3, 2)); + assert!(!budget.budget_elapsed()); + } + + #[tokio::test] + async fn scanner_phase_cannot_move_back_to_scanning() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + for phase in [RemoteScannerPhase::Persisting, RemoteScannerPhase::Scanning] { + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::with_phase(RemoteScannerProgress::default(), phase, RemoteScannerFrameResult::Progress), + ) + .await + .expect("phase frame should write"); + } + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("backwards phase must fail"); + + assert!(error.to_string().contains("phase moved backwards")); + } + + #[tokio::test] + async fn terminal_usage_for_another_bucket_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage: test_usage("other-bucket", 1), + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("wrong bucket must fail"); + assert!(error.to_string().contains("wrong bucket")); + } + + #[tokio::test] + async fn terminal_usage_for_another_source_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: DataUsageCacheSource::new(TEST_SOURCE.pool_index + 1, TEST_SOURCE.set_index), + scan_plan_digest: TEST_PLAN_DIGEST, + usage: test_usage("bucket", 1), + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("wrong source must fail"); + assert!(error.to_string().contains("wrong pool or set")); + } + + #[tokio::test] + async fn terminal_usage_must_be_flattened() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut usage = test_usage("bucket", 1); + usage.entry.children.insert("child-hash".to_string()); + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage, + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("non-flattened usage must fail"); + assert!(error.to_string().contains("non-flattened")); + } + + #[tokio::test] + async fn terminal_cycle_ahead_reports_required_cycle() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::CycleAhead { + required_cycle: TEST_NEXT_CYCLE + 1, + }, + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let outcome = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("newer remote cache cycle should be reported"); + assert!(matches!( + outcome, + RemoteScannerOutcome::CycleAhead(required_cycle) if required_cycle == TEST_NEXT_CYCLE + 1 + )); + } + + #[tokio::test] + async fn terminal_cycle_ahead_rejects_nonincreasing_cycle() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::CycleAhead { + required_cycle: TEST_NEXT_CYCLE, + }, + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("nonincreasing required cycle must fail"); + assert!(error.to_string().contains("invalid required cycle")); + } + + #[tokio::test] + async fn terminal_usage_accepts_large_replication_target_payload_within_frame_limit() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(NS_SCANNER_MAX_FRAME_SIZE); + tokio::spawn(async move { + let mut usage = test_usage("bucket", 1); + let stats = usage.entry.replication_stats.get_or_insert_default(); + for index in 0..=1024 { + stats.targets.insert(format!("target-{index}"), Default::default()); + } + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage, + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let outcome = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect("large historical replication target payload should remain readable"); + assert!(matches!(outcome, RemoteScannerOutcome::Complete { .. })); + } + + #[tokio::test] + async fn tampered_frame_authentication_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator { + request_id, + secret: "different-secret".to_string(), + }; + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal(RemoteScannerProgress::default(), RemoteScannerFrameResult::Partial), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("tampered authentication must fail"); + assert!(error.to_string().contains("authentication failed")); + } + + #[tokio::test] + async fn wrong_frame_version_and_sequence_are_rejected() { + let request_id = Uuid::new_v4(); + let auth = FrameAuthenticator::for_test(request_id); + let frame = RemoteScannerFrame::progress(RemoteScannerProgress::default()); + let payload = rmp_serde::to_vec_named(&frame).expect("frame should encode"); + + for (version, sequence, expected_error) in [ + (NS_SCANNER_PROTOCOL_VERSION - 1, 0, "unsupported remote namespace scanner frame version"), + (NS_SCANNER_PROTOCOL_VERSION + 1, 0, "unsupported remote namespace scanner frame version"), + (NS_SCANNER_PROTOCOL_VERSION, 1, "frame sequence is invalid"), + ] { + let envelope = RemoteScannerFrameEnvelope { + version, + sequence, + mac: auth.sign(sequence, &payload).expect("frame should sign"), + payload: payload.clone(), + }; + let encoded = rmp_serde::to_vec_named(&envelope).expect("envelope should encode"); + let mut input = std::io::Cursor::new(Vec::with_capacity(encoded.len() + 4)); + input.get_mut().extend_from_slice( + &u32::try_from(encoded.len()) + .expect("encoded frame length should fit u32") + .to_be_bytes(), + ); + input.get_mut().extend_from_slice(&encoded); + input.set_position(0); + + let mut expected_sequence = 0; + let error = read_frame(&mut input, &auth, &mut expected_sequence) + .await + .expect_err("invalid frame must fail"); + assert!(error.to_string().contains(expected_error)); + } + } + + #[tokio::test] + async fn backwards_progress_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::progress(RemoteScannerProgress { + objects_scanned: 2, + directories_started: 1, + ..Default::default() + }), + ) + .await + .expect("progress should write"); + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: 1, + directories_started: 1, + ..Default::default() + }, + RemoteScannerFrameResult::Partial, + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("backwards progress must fail"); + assert!(error.to_string().contains("moved backwards")); + } + + #[tokio::test] + async fn eof_without_terminal_frame_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::progress(RemoteScannerProgress::default()), + ) + .await + .expect("progress should write"); + writer.shutdown().await.expect("writer should shut down"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("missing terminal frame must fail"); + assert!(error.to_string().contains("before a terminal frame")); + } + + #[test] + fn oversized_frame_is_rejected_before_allocation() { + let request_id = Uuid::new_v4(); + let auth = FrameAuthenticator::for_test(request_id); + let mut input = std::io::Cursor::new( + u32::try_from(NS_SCANNER_MAX_FRAME_SIZE + 1) + .expect("test frame size should fit u32") + .to_be_bytes() + .to_vec(), + ); + let runtime = tokio::runtime::Runtime::new().expect("runtime should start"); + let mut sequence = 0; + let error = runtime + .block_on(read_frame(&mut input, &auth, &mut sequence)) + .expect_err("oversized frame must fail"); + assert_eq!(error.kind(), ErrorKind::InvalidData); + } +} diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 5c4caa2d9..b4bba4494 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -17,7 +17,10 @@ use std::future::Future; use std::sync::{Arc, LazyLock, RwLock}; use crate::ScannerObjectIO; -use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH}; +use crate::data_usage_define::{ + BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DataUsageCache, DataUsageCacheRevision, + LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision, +}; use crate::runtime_config::{ ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle, scanner_cycle_interval, scanner_runtime_config_changed, scanner_runtime_config_generation, scanner_start_delay, @@ -27,15 +30,16 @@ use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, Scanne use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob}; use crate::scanner_io::{ ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, dirty_usage_generation, - scanner_maintenance_changed, scanner_maintenance_generation, + scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation, }; use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed}; use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError}; +use bytes::Bytes; use chrono::{DateTime, Utc}; use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{ CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete, - emit_scan_cycle_partial_with_source, global_metrics, + emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics, }; use rustfs_config::ScannerSpeed; #[cfg(test)] @@ -45,16 +49,20 @@ use rustfs_config::{ }; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; use tokio::sync::mpsc; use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; -use crate::storage_api::scan::{BucketOperations, BucketOptions, NamespaceLocking as _}; +use crate::storage_api::scan::{ + BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, +}; use crate::{ ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config, - scanner_is_erasure_sd, + save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd, }; const LOG_COMPONENT_SCANNER: &str = "scanner"; @@ -67,15 +75,40 @@ const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config"; const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state"; const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total"; const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); +const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60); const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1); +#[cfg(not(test))] +const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] +const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(50); const MAINTENANCE_FEATURE_INSPECTION_TIMEOUT: Duration = Duration::from_secs(30); const MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5 * 60); const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(60 * 60); const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2; +const SCANNER_PERSIST_CAS_RETRIES: usize = 2; +const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10; +const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001"; +const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24; #[cfg(test)] const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS"; +#[derive(Debug, thiserror::Error)] +enum ScannerCycleStateError { + #[error("failed to encode scanner cycle state: {0}")] + Encode(#[from] rmp_serde::encode::Error), + #[error("failed to decode scanner cycle state: {0}")] + Decode(#[from] rmp_serde::decode::Error), + #[error("{0}")] + InvalidData(&'static str), +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct PersistedUsageFloor { + next_cycle: u64, + leader_epoch: u64, +} + #[derive(Clone, Copy, Debug, Serialize)] #[non_exhaustive] pub struct ScannerCycleScheduleStatus { @@ -191,11 +224,11 @@ fn randomized_cycle_delay() -> Duration { } fn randomized_cycle_delay_for(interval: Duration) -> Duration { - let interval = interval.max(Duration::from_secs(1)); + let interval = interval.max(Duration::from_secs(1)).min(MAX_SCANNER_SCHEDULE_DELAY); // Uniform in [-0.1, 0.1), keeping actual delay within 10% of interval. let jitter_factor = (rand::random::() * 0.2) - 0.1; let delay = interval.mul_f64(1.0 + jitter_factor); - delay.max(Duration::from_secs(1)) + delay.max(Duration::from_secs(1)).min(MAX_SCANNER_SCHEDULE_DELAY) } fn cap_clean_idle_cycle_delay(delay: Duration, max_interval: Duration, enabled: bool) -> Duration { @@ -233,6 +266,7 @@ pub(crate) enum ScannerCycleOutcome { Completed, CompletedWithPendingMaintenance, Partial, + Superseded, Failed, } @@ -247,6 +281,30 @@ pub(crate) fn scanner_cycle_outcome_with_pending_maintenance( } } +async fn remote_dirty_usage_acknowledgement_pending(cycle: u64, acknowledgement_count: usize, acknowledgement: F) -> bool +where + F: Future>, + E: std::fmt::Display, +{ + match acknowledgement.await { + Ok(dirty_usage_pending) => dirty_usage_pending, + Err(err) => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle, + acknowledgement_count, + error = %err, + state = "remote_dirty_usage_acknowledgement_pending", + "Scanner cycle left remote dirty usage acknowledgements pending" + ); + true + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct ScannerCleanIdleBackoff { interval_multiplier: u32, @@ -372,13 +430,25 @@ struct ScannerCycleObservedGenerations { const LOCAL_SCANNER_ACTIVITY_NODE: &str = ""; #[derive(Clone, Debug, PartialEq, Eq)] -struct ScannerNodeActivity { +pub(crate) struct ScannerNodeActivity { instance_id: String, namespace_generation: u64, maintenance_generation: u64, + protocol_version: u32, + topology_digest: [u8; 32], + data_movement_active: bool, + dirty_usage_generation: u64, + dirty_usage_pending: bool, } -type ScannerActivitySnapshot = BTreeMap; +pub(crate) type ScannerActivitySnapshot = BTreeMap; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ScannerDirtyUsageAcknowledgement { + pub(crate) host: String, + pub(crate) instance_id: String, + pub(crate) generation: u64, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ScannerActivityObservation { @@ -709,13 +779,108 @@ async fn observe_scanner_activity( observation } -async fn probe_scanner_activity(storeapi: &Arc, distributed: bool) -> Result { +pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (host, activity) in snapshot { + let host = host.as_bytes(); + let instance_id = activity.instance_id.as_bytes(); + hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(host); + hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(instance_id); + hasher.update(activity.namespace_generation.to_be_bytes()); + hasher.update(activity.maintenance_generation.to_be_bytes()); + hasher.update(activity.protocol_version.to_be_bytes()); + hasher.update(activity.topology_digest); + hasher.update([u8::from(activity.data_movement_active)]); + hasher.update(activity.dirty_usage_generation.to_be_bytes()); + hasher.update([u8::from(activity.dirty_usage_pending)]); + } + hasher.finalize().into() +} + +pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { + snapshot.values().all(|activity| !activity.data_movement_active) +} + +pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec { + snapshot + .iter() + .filter(|(host, activity)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE && activity.dirty_usage_pending) + .map(|(host, activity)| ScannerDirtyUsageAcknowledgement { + host: host.clone(), + instance_id: activity.instance_id.clone(), + generation: activity.dirty_usage_generation, + }) + .collect() +} + +pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { + let endpoint_pools = storeapi.endpoints(); + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(endpoint_pools.0.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (pool_index, pool) in endpoint_pools.0.iter().enumerate() { + hasher.update(u64::try_from(pool_index).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(u64::try_from(pool.set_count).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(u64::try_from(pool.drives_per_set).unwrap_or(u64::MAX).to_be_bytes()); + let mut endpoints = pool.endpoints.as_ref().iter().collect::>(); + endpoints.sort_unstable_by(|left, right| { + (left.pool_idx, left.set_idx, left.disk_idx, left.url.as_str()).cmp(&( + right.pool_idx, + right.set_idx, + right.disk_idx, + right.url.as_str(), + )) + }); + hasher.update(u64::try_from(endpoints.len()).unwrap_or(u64::MAX).to_be_bytes()); + for endpoint in endpoints { + hasher.update(endpoint.pool_idx.to_be_bytes()); + hasher.update(endpoint.set_idx.to_be_bytes()); + hasher.update(endpoint.disk_idx.to_be_bytes()); + let url = endpoint.url.as_str().as_bytes(); + hasher.update(u64::try_from(url.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(url); + } + } + hasher.finalize().into() +} + +fn record_scanner_activity_instance( + instance_hosts: &mut BTreeMap, + host: &str, + instance_id: &str, +) -> Result<(), String> { + if let Some(existing_host) = instance_hosts.insert(instance_id.to_string(), host.to_string()) { + return Err(format!( + "scanner activity peers {existing_host} and {host} report the same process instance" + )); + } + Ok(()) +} + +pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result { + let topology_digest = scanner_topology_digest(storeapi); + let data_movement_active = storeapi.scanner_data_movement_active().await; + let namespace_generation = storeapi.scanner_namespace_mutation_generation(); + let maintenance_generation = scanner_maintenance_generation(); + let dirty_usage = scanner_dirty_usage_state(); + if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX { + return Err("local scanner activity generation is exhausted".to_string()); + } + let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string(); + let mut instance_hosts = BTreeMap::from([(local_instance_id.clone(), LOCAL_SCANNER_ACTIVITY_NODE.to_string())]); let mut snapshot = ScannerActivitySnapshot::from([( LOCAL_SCANNER_ACTIVITY_NODE.to_string(), ScannerNodeActivity { - instance_id: crate::scanner_io::scanner_activity_epoch().to_string(), - namespace_generation: storeapi.scanner_namespace_mutation_generation(), - maintenance_generation: scanner_maintenance_generation(), + instance_id: local_instance_id, + namespace_generation, + maintenance_generation, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest, + data_movement_active, + dirty_usage_generation: dirty_usage.generation, + dirty_usage_pending: dirty_usage.pending, }, )]); if !distributed { @@ -730,6 +895,48 @@ async fn probe_scanner_activity(storeapi: &Arc, distributed: bool) -> R .await .map_err(|err| err.to_string())?; for (host, activity) in peers { + if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX { + return Err(format!("scanner activity peer {host} exhausted its activity generation")); + } + let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) = + match activity.protocol_version { + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => { + return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing")); + } + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + return Err(format!( + "scanner activity peer {host} cannot acknowledge distributed dirty usage with protocol {}", + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION + )); + } + SCANNER_ACTIVITY_PROTOCOL_VERSION => ( + activity + .topology_digest + .ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?, + activity + .data_movement_active + .ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?, + activity + .dirty_usage_generation + .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?, + activity + .dirty_usage_pending + .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?, + ), + version => { + return Err(format!( + "scanner activity peer {host} uses protocol {version}, expected {}", + SCANNER_ACTIVITY_PROTOCOL_VERSION + )); + } + }; + if peer_dirty_usage_generation == u64::MAX { + return Err(format!("scanner activity peer {host} exhausted its dirty usage generation")); + } + if peer_topology_digest != topology_digest { + return Err(format!("scanner activity peer {host} has a different storage topology")); + } + record_scanner_activity_instance(&mut instance_hosts, &host, &activity.instance_id)?; if snapshot .insert( host.clone(), @@ -737,6 +944,11 @@ async fn probe_scanner_activity(storeapi: &Arc, distributed: bool) -> R instance_id: activity.instance_id, namespace_generation: activity.namespace_generation, maintenance_generation: activity.maintenance_generation, + protocol_version: activity.protocol_version, + topology_digest: peer_topology_digest, + data_movement_active: peer_data_movement_active, + dirty_usage_generation: peer_dirty_usage_generation, + dirty_usage_pending: peer_dirty_usage_pending, }, ) .is_some() @@ -776,19 +988,91 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool { info.last_update.is_none() || (info.buckets_usage.is_empty() && info.bucket_sizes.is_empty()) } -async fn read_data_usage_config_for_startup(storeapi: &Arc) -> Result>, EcstoreError> { - match read_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { - Ok(data) => Ok(Some(data)), - Err(EcstoreError::ConfigNotFound) => { - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - match read_config(storeapi.clone(), backup_path.as_str()).await { - Ok(data) => Ok(Some(data)), - Err(EcstoreError::ConfigNotFound) => Ok(None), - Err(err) => Err(err), +async fn read_data_usage_config_for_startup(storeapi: &Arc) -> Result>, EcstoreError> { + async fn read_pair(storeapi: &Arc, primary_path: &str) -> Result>, EcstoreError> { + match read_config(storeapi.clone(), primary_path).await { + Ok(data) => Ok(Some(data)), + Err(EcstoreError::ConfigNotFound) => { + let backup_path = format!("{primary_path}.bkp"); + match read_config(storeapi.clone(), backup_path.as_str()).await { + Ok(data) => Ok(Some(data)), + Err(EcstoreError::ConfigNotFound) => Ok(None), + Err(err) => Err(err), + } + } + Err(err) => Err(err), + } + } + + match read_pair(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()).await? { + Some(data) => Ok(Some(data)), + None => read_pair(storeapi, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()).await, + } +} + +fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool { + data_usage_info + .scanner_cycle + .is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0) +} + +async fn sync_data_usage_backup_from_primary( + ctx: &CancellationToken, + storeapi: Arc, +) -> Result<(), EcstoreError> { + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + return Ok(()); + } + + let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; + let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?; + serde_json::from_slice::(&primary) + .map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?; + let primary = Bytes::from(primary); + + let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?; + if backup.as_deref() == Some(primary.as_ref()) { + return Ok(()); + } + + let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower)); + let save_result = save_config_shared_with_preconditions( + storeapi.clone(), + &backup_path, + primary.clone(), + sha256hex, + revision.preconditions(), + ) + .await; + + match save_result { + Ok(_) => {} + Err(err) => { + let (observed, _) = read_config_with_revision(storeapi.clone(), &backup_path).await?; + if observed.as_deref() == Some(primary.as_ref()) { + // The write committed even though the response was lost. + } else if err == EcstoreError::PreconditionFailed && retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } else { + return Err(err); + } } } - Err(err) => Err(err), + + let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; + if current_primary.as_deref() == Some(primary.as_ref()) { + return Ok(()); + } + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } } + + Err(EcstoreError::other( + "authoritative data usage snapshot changed while synchronizing its backup", + )) } async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc) -> bool { @@ -1400,14 +1684,506 @@ fn get_lock_acquire_timeout() -> Duration { Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5)) } +fn data_usage_persist_timeout() -> Duration { + DataUsageCache::persistence_timeout() +} + async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) { cycle_info.current = 0; global_metrics().clear_current_scan_mode(); global_metrics().set_cycle(Some(cycle_info.clone())).await; } -async fn persist_scanner_cycle_state(storeapi: Arc, cycle_info: &CurrentCycle) -> bool { - let cycle_info_buf = match cycle_info.marshal() { +fn encode_scanner_cycle_state(cycle_info: &CurrentCycle, leader_epoch: u64) -> Result, ScannerCycleStateError> { + if cycle_info.next == u64::MAX { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); + } + let cycle_info_buf = rmp_serde::to_vec(cycle_info)?; + let mut buf = Vec::with_capacity(cycle_info_buf.len() + SCANNER_CYCLE_STATE_HEADER_LEN); + buf.extend_from_slice(&cycle_info.next.to_le_bytes()); + buf.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); + buf.extend_from_slice(&leader_epoch.to_le_bytes()); + buf.extend_from_slice(&cycle_info_buf); + Ok(buf) +} + +fn decode_scanner_cycle_state(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { + if buf.len() < 8 { + return Err(ScannerCycleStateError::InvalidData("scanner cycle state is truncated")); + } + + let persisted_next = u64::from_le_bytes( + buf[0..8] + .try_into() + .map_err(|_| ScannerCycleStateError::InvalidData("scanner cycle counter is truncated"))?, + ); + if persisted_next == u64::MAX { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); + } + if buf.len() == 8 { + return Ok(( + CurrentCycle { + next: persisted_next, + ..Default::default() + }, + 0, + )); + } + + let (leader_epoch, payload) = if buf.len() >= 16 && &buf[8..16] == SCANNER_CYCLE_STATE_MAGIC { + if buf.len() < SCANNER_CYCLE_STATE_HEADER_LEN { + return Err(ScannerCycleStateError::InvalidData("scanner cycle fencing header is truncated")); + } + let epoch = u64::from_le_bytes( + buf[16..24] + .try_into() + .map_err(|_| ScannerCycleStateError::InvalidData("scanner leader epoch is truncated"))?, + ); + if epoch == 0 { + return Err(ScannerCycleStateError::InvalidData("scanner leader epoch is zero")); + } + (epoch, &buf[SCANNER_CYCLE_STATE_HEADER_LEN..]) + } else { + (0, &buf[8..]) + }; + + let cycle_info = rmp_serde::from_slice::(payload)?; + if cycle_info.next != persisted_next { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter disagrees with encoded state")); + } + Ok((cycle_info, leader_epoch)) +} + +pub(crate) fn decode_persisted_scanner_cycle_fence(buf: &[u8]) -> Result<(u64, u64), ScannerError> { + decode_scanner_cycle_state(buf) + .map(|(cycle, leader_epoch)| (cycle.next, leader_epoch)) + .map_err(|err| ScannerError::Other(format!("persisted scanner cycle state is invalid: {err}"))) +} + +#[cfg(test)] +pub(crate) fn encode_scanner_cycle_fence_for_test(next_cycle: u64, leader_epoch: u64) -> Vec { + encode_scanner_cycle_state( + &CurrentCycle { + next: next_cycle, + ..Default::default() + }, + leader_epoch, + ) + .expect("test scanner cycle fence should encode") +} + +pub(crate) async fn current_scanner_leader_epoch() -> Result { + let store = crate::resolve_scanner_object_store_handle() + .ok_or_else(|| ScannerError::Other("scanner object layer is unavailable".to_string()))?; + match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await { + Ok(buf) => { + let (_, leader_epoch) = decode_persisted_scanner_cycle_fence(&buf)?; + if leader_epoch == 0 { + return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string())); + } + Ok(leader_epoch) + } + Err(err) => Err(ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}"))), + } +} + +fn decode_scanner_cycle_state_for_startup(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { + if buf.is_empty() { + Ok((CurrentCycle::default(), 0)) + } else { + decode_scanner_cycle_state(buf) + } +} + +fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), ScannerCycleStateError> { + let next = cycle_info + .next + .checked_add(1) + .filter(|next| *next < u64::MAX) + .ok_or(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted"))?; + cycle_info.next = next; + Ok(()) +} + +async fn persisted_usage_floor(storeapi: Arc) -> Result { + let mut floor = PersistedUsageFloor::default(); + for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] { + let backup_path = format!("{primary_path}.bkp"); + let mut pair_found = false; + for path in [primary_path, backup_path.as_str()] { + let data = match read_config(storeapi.clone(), path).await { + Ok(data) => { + pair_found = true; + data + } + Err(EcstoreError::ConfigNotFound) => continue, + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read scanner usage epoch floor from {path}: {err}" + ))); + } + }; + let usage = serde_json::from_slice::(&data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage floor from {path}: {err}")))?; + floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); + if let Some(completed_cycle) = usage.scanner_cycle { + let next_cycle = completed_cycle + .checked_add(1) + .filter(|next| *next < u64::MAX) + .ok_or_else(|| ScannerError::Other(format!("persisted scanner usage cycle is exhausted in {path}")))?; + floor.next_cycle = floor.next_cycle.max(next_cycle); + } + } + if pair_found { + break; + } + } + Ok(floor) +} + +fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_epoch: &mut u64, floor: PersistedUsageFloor) { + cycle_info.next = cycle_info.next.max(floor.next_cycle); + *leader_epoch = (*leader_epoch).max(floor.leader_epoch); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ScannerLeadershipClaimReconcile { + Durable, + Changed, + Unchanged, +} + +async fn reconcile_scanner_leadership_claim( + storeapi: Arc, + attempted: &[u8], + previous_revision: &DataUsageCacheRevision, + claimed_epoch: u64, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + persisted_epoch: &mut u64, +) -> Result { + let (persisted, persisted_revision) = read_config_with_revision(storeapi, DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner leadership claim: {err}")))?; + let revision_changed = &persisted_revision != previous_revision; + *revision = persisted_revision; + + let Some(persisted) = persisted else { + *cycle_info = CurrentCycle::default(); + return Ok(if revision_changed { + ScannerLeadershipClaimReconcile::Changed + } else { + ScannerLeadershipClaimReconcile::Unchanged + }); + }; + if persisted == attempted { + *persisted_epoch = claimed_epoch; + return Ok(ScannerLeadershipClaimReconcile::Durable); + } + + let (current, epoch) = decode_scanner_cycle_state(&persisted) + .map_err(|err| ScannerError::Other(format!("scanner leadership conflict winner is invalid: {err}")))?; + *cycle_info = current; + *persisted_epoch = (*persisted_epoch).max(epoch); + Ok(if revision_changed { + ScannerLeadershipClaimReconcile::Changed + } else { + ScannerLeadershipClaimReconcile::Unchanged + }) +} + +fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result { + serde_json::from_slice(data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) +} + +async fn usage_snapshot_for_epoch_fence( + storeapi: Arc, + primary: Option<&[u8]>, +) -> Result { + if let Some(primary) = primary { + return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); + } + + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let (backup, _) = read_config_with_revision(storeapi.clone(), &backup_path) + .await + .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; + if let Some(backup) = backup.as_deref() { + return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); + } + + for path in [ + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), + format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + ] { + let (legacy, _) = read_config_with_revision(storeapi.clone(), &path) + .await + .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; + if let Some(legacy) = legacy.as_deref() { + return decode_usage_snapshot_for_epoch_fence(legacy, &path); + } + } + Ok(DataUsageInfo::default()) +} + +async fn fence_scanner_usage_epoch( + ctx: &CancellationToken, + storeapi: Arc, + claimed_epoch: u64, +) -> Result<(), ScannerError> { + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); + } + + let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; + let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; + match usage.scanner_epoch { + Some(epoch) if epoch > claimed_epoch => { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" + ))); + } + Some(epoch) if epoch == claimed_epoch => return Ok(()), + Some(_) | None => {} + } + usage.scanner_epoch = Some(claimed_epoch); + let data = serde_json::to_vec(&usage) + .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; + + let save_result = + save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) + .await; + if save_result + .as_ref() + .ok() + .and_then(|object_info| object_info.etag.as_deref()) + .is_some_and(|etag| !etag.is_empty()) + { + return Ok(()); + } + + let (persisted, persisted_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage epoch fence: {err}")))?; + if let Some(persisted) = persisted { + let persisted = decode_usage_snapshot_for_epoch_fence(&persisted, DATA_USAGE_OBJ_NAME_PATH.as_str())?; + match persisted.scanner_epoch { + Some(epoch) if epoch == claimed_epoch => return Ok(()), + Some(epoch) if epoch > claimed_epoch => { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" + ))); + } + Some(_) | None => {} + } + } + + let precondition_failed = matches!(save_result, Err(EcstoreError::PreconditionFailed)); + if retry < SCANNER_PERSIST_CAS_RETRIES && (precondition_failed || persisted_revision != revision) { + continue; + } + return Err(ScannerError::Other(match save_result { + Ok(_) => "scanner usage epoch fence returned no ETag and could not be confirmed".to_string(), + Err(err) => format!("scanner usage epoch fence save failed: {err}"), + })); + } + + Err(ScannerError::Other("scanner usage epoch fence retries exhausted".to_string())) +} + +async fn complete_scanner_leadership_claim( + ctx: &CancellationToken, + storeapi: Arc, + claimed_epoch: u64, +) -> bool { + if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "usage_epoch_fence_failed", + claimed_epoch, + error = %err, + "Scanner leadership usage epoch fencing failed" + ); + return false; + } + !ctx.is_cancelled() +} + +async fn claim_scanner_leadership( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + persisted_epoch: &mut u64, +) -> bool { + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + return false; + } + let Some(claimed_epoch) = persisted_epoch.checked_add(1) else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_epoch_exhausted", + "Scanner leadership epoch is exhausted" + ); + return false; + }; + let data = match encode_scanner_cycle_state(cycle_info, claimed_epoch) { + Ok(data) => data, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_encode_failed", + error = %err, + "Scanner leadership claim encoding failed" + ); + return false; + } + }; + let previous_revision = revision.clone(); + + let save_result = + save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) + .await; + match save_result { + Ok(object_info) => { + if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { + *revision = DataUsageCacheRevision::Etag(etag); + *persisted_epoch = claimed_epoch; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + + match reconcile_scanner_leadership_claim( + storeapi.clone(), + &data, + &previous_revision, + claimed_epoch, + cycle_info, + revision, + persisted_epoch, + ) + .await + { + Ok(ScannerLeadershipClaimReconcile::Durable) => { + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, + Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_missing_revision", + "Scanner leadership claim returned no ETag and could not be confirmed" + ); + return false; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_reconcile_failed", + error = %err, + "Scanner leadership claim read-back failed" + ); + return false; + } + } + } + Err(err) => { + let precondition_failed = matches!(err, EcstoreError::PreconditionFailed); + match reconcile_scanner_leadership_claim( + storeapi.clone(), + &data, + &previous_revision, + claimed_epoch, + cycle_info, + revision, + persisted_epoch, + ) + .await + { + Ok(ScannerLeadershipClaimReconcile::Durable) => { + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + Ok(ScannerLeadershipClaimReconcile::Changed) + if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => + { + continue; + } + Ok(ScannerLeadershipClaimReconcile::Unchanged) + if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => + { + continue; + } + Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = if precondition_failed { + "leader_claim_conflicts_exhausted" + } else { + "leader_claim_failed" + }, + error = %err, + "Scanner leadership claim failed" + ); + return false; + } + Err(reconcile_err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_reload_failed", + error = %reconcile_err, + save_error = %err, + "Scanner leadership claim reconciliation failed" + ); + return false; + } + } + } + } + } + + false +} + +async fn persist_scanner_cycle_state( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, +) -> bool { + let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { Ok(buf) => buf, Err(e) => { error!( @@ -1424,45 +2200,279 @@ async fn persist_scanner_cycle_state(storeapi: Arc, cycle_ } }; - let mut buf = Vec::with_capacity(cycle_info_buf.len() + 8); - buf.extend_from_slice(&cycle_info.next.to_le_bytes()); - buf.extend_from_slice(&cycle_info_buf); + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_before_save", + retry, + "Scanner state persistence cancelled by the leader fence" + ); + return false; + } - if let Err(e) = save_config(storeapi, &DATA_USAGE_BLOOM_NAME_PATH, buf).await { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "failed", - error = %e, - "Scanner state persistence failed" - ); - false - } else { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "saved", - "Scanner state saved" - ); - true + match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) + .await + { + Ok(object_info) => { + let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "missing_revision", + "Scanner state save returned no ETag" + ); + return false; + }; + *revision = DataUsageCacheRevision::Etag(etag); + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_after_save", + retry, + "Scanner state save completed after the leader fence was cancelled" + ); + return false; + } + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "saved", + "Scanner state saved" + ); + return true; + } + Err(EcstoreError::PreconditionFailed) => { + let (persisted, persisted_revision) = + match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await { + Ok(result) => result, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_reload_failed", + error = %e, + "Scanner state conflict reconciliation failed" + ); + return false; + } + }; + *revision = persisted_revision; + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_after_conflict", + retry, + "Scanner state conflict reconciliation cancelled by the leader fence" + ); + return false; + } + + if let Some(persisted) = persisted { + if persisted.len() < 8 { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_state_invalid", + length = persisted.len(), + "Scanner state conflict winner is truncated" + ); + return false; + } + + let (persisted_cycle, persisted_epoch) = match decode_scanner_cycle_state(&persisted) { + Ok(state) => state, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_state_decode_failed", + error = %e, + "Scanner state conflict winner could not be decoded" + ); + return false; + } + }; + if persisted_epoch != leader_epoch { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_epoch_fenced", + expected_epoch = leader_epoch, + persisted_epoch, + "Scanner state save rejected by a newer leadership epoch" + ); + return false; + } + + if persisted_cycle.next >= cycle_info.next { + *cycle_info = persisted_cycle; + global_metrics().set_cycle(Some(cycle_info.clone())).await; + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_reconciled", + retry, + "Scanner state adopted the current persisted cycle" + ); + return true; + } + } + + if retry < SCANNER_PERSIST_CAS_RETRIES { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_retry", + retry = retry + 1, + "Scanner state CAS conflict will be retried" + ); + continue; + } + + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_retries_exhausted", + retries = SCANNER_PERSIST_CAS_RETRIES, + "Scanner state CAS conflict retries exhausted" + ); + return false; + } + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "failed", + error = %e, + "Scanner state persistence failed" + ); + return false; + } + } } + + false } -async fn finalize_partial_scan_cycle(storeapi: Arc, cycle_info: &mut CurrentCycle) -> bool { +async fn finalize_partial_scan_cycle( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, +) -> bool { // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter // must still advance (and persist) because per-bucket next_cycle is stamped from // it and compacted folders are only rescanned when their hash matches // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle // expiry and usage refresh on every folder outside the stuck window. - cycle_info.next += 1; + if let Err(err) = advance_scanner_cycle(cycle_info) { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cycle_counter_exhausted", + error = %err, + "Scanner partial cycle could not advance" + ); + mark_scan_cycle_idle(cycle_info).await; + return false; + } mark_scan_cycle_idle(cycle_info).await; - persist_scanner_cycle_state(storeapi, cycle_info).await + persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await +} + +async fn persist_required_scanner_cycle_floor( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + required_cycle: u64, +) -> bool { + if required_cycle <= cycle_info.current || required_cycle == u64::MAX { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + current_cycle = cycle_info.current, + required_cycle, + state = "invalid_cache_cycle_floor", + "Scanner cache cycle floor is invalid" + ); + mark_scan_cycle_idle(cycle_info).await; + return false; + } + + cycle_info.next = cycle_info.next.max(required_cycle); + mark_scan_cycle_idle(cycle_info).await; + persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await +} + +async fn await_scanner_cycle_with_lock_fence( + cycle_ctx: &CancellationToken, + cycle: Cycle, + lock_lost: LockLost, +) -> Option +where + Cycle: Future, + LockLost: Future, +{ + tokio::pin!(cycle); + tokio::pin!(lock_lost); + tokio::select! { + biased; + _ = &mut lock_lost => { + cycle_ctx.cancel(); + tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await.ok() + } + output = &mut cycle => Some(output), + } } #[instrument(skip_all)] @@ -1470,6 +2480,8 @@ async fn run_data_scanner_cycle( ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle, + cycle_revision: &mut DataUsageCacheRevision, + leader_epoch: u64, ) -> ScannerCycleOutcome { let _activity_guard = ScannerActivityGuard::new(); if let Err(err) = refresh_scanner_runtime_config_from_global() { @@ -1486,7 +2498,7 @@ async fn run_data_scanner_cycle( let configured_cycle_interval = scanner_cycle_interval(); let configured_bitrot_cycle = scanner_bitrot_cycle(); let cycle_budget_config = scanner_cycle_budget_config(); - let usage_persist_timeout = resolve_scanner_runtime_config().cache_save_timeout; + let usage_persist_timeout = data_usage_persist_timeout(); global_metrics().record_scanner_cycle_config( configured_cycle_interval, configured_bitrot_cycle, @@ -1530,19 +2542,56 @@ async fn run_data_scanner_cycle( save_background_heal_info(storeapi.clone(), new_heal_info).await; } + let cycle_start = std::time::Instant::now(); + let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + Ok((data, revision)) => DataUsagePersistBaseline { + data: data.map(Bytes::from), + revision, + }, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "usage_baseline_load_failed", + error = %err, + "Scanner cycle could not capture the data usage persistence baseline" + ); + emit_scan_cycle_complete(false, cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info).await; + return ScannerCycleOutcome::Failed; + } + }; let (sender, receiver) = mpsc::channel::(1); let storeapi_clone = storeapi.clone(); let ctx_clone = ctx.clone(); - let mut usage_persist_task = - tokio::spawn(async move { store_data_usage_in_backend_with_outcome(ctx_clone, storeapi_clone, receiver).await }); + let mut usage_persist_task = tokio::spawn(async move { + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( + ctx_clone, + storeapi_clone, + receiver, + Some(leader_epoch), + Some(usage_persist_baseline), + ) + .await + }); let done_cycle = Metrics::time(Metric::ScanCycle); - let cycle_start = std::time::Instant::now(); let cycle_work_start = global_metrics().start_scan_cycle_work(); let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config); let scan_result = storeapi .clone() - .nsscanner_with_status(cycle_budget.token(), cycle_budget.clone(), sender, cycle_info.current, scan_mode) + .nsscanner_with_status( + cycle_budget.token(), + cycle_budget.clone(), + sender, + cycle_info.current, + leader_epoch, + scan_mode, + ) .await; let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await @@ -1629,6 +2678,33 @@ async fn run_data_scanner_cycle( mark_scan_cycle_idle(cycle_info).await; return ScannerCycleOutcome::Failed; } + if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + required_cycle, + state = "cache_cycle_ahead", + "Scanner cycle is recovering to a newer durable cache generation" + ); + emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); + return if persist_required_scanner_cycle_floor( + ctx, + storeapi.clone(), + cycle_info, + cycle_revision, + leader_epoch, + required_cycle, + ) + .await + { + ScannerCycleOutcome::Partial + } else { + ScannerCycleOutcome::Failed + }; + } if usage_persist_outcome == DataUsagePersistOutcome::Failed { error!( target: "rustfs::scanner", @@ -1664,16 +2740,42 @@ async fn run_data_scanner_cycle( scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - return if finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await { + return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed }; } - let (completion_outcome, scanner_pending_maintenance_work) = + let (completion_outcome, scanner_pending_maintenance_work, remote_dirty_usage_acknowledgements) = finalize_scanner_cycle_result(scan_cycle_result, usage_persist_outcome); - let pending_maintenance_work = scanner_pending_maintenance_work || unresolved_heal_work; + let remote_dirty_usage_pending = if remote_dirty_usage_acknowledgements.is_empty() { + false + } else if let Some(notification_system) = storeapi.notification_system() { + let acknowledgement_count = remote_dirty_usage_acknowledgements.len(); + let acknowledgements = remote_dirty_usage_acknowledgements + .into_iter() + .map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation)) + .collect(); + remote_dirty_usage_acknowledgement_pending( + cycle_info.current, + acknowledgement_count, + notification_system.acknowledge_scanner_dirty_usage(acknowledgements), + ) + .await + } else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "remote_dirty_usage_acknowledgement_unavailable", + "Scanner cycle cannot acknowledge remote dirty usage without a notification system" + ); + true + }; + let pending_maintenance_work = scanner_pending_maintenance_work || unresolved_heal_work || remote_dirty_usage_pending; match completion_outcome { ScannerCycleOutcome::Failed => { error!( @@ -1713,22 +2815,52 @@ async fn run_data_scanner_cycle( ); } emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - return if finalize_partial_scan_cycle(storeapi.clone(), cycle_info).await { + return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed }; } + ScannerCycleOutcome::Superseded => { + info!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle = cycle_info.current, + state = "superseded", + "Scanner cycle usage snapshot was superseded by concurrent namespace activity" + ); + if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { + emit_scan_cycle_superseded(cycle_start.elapsed()); + return ScannerCycleOutcome::Superseded; + } + emit_scan_cycle_complete(false, cycle_start.elapsed()); + return ScannerCycleOutcome::Failed; + } ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {} } - cycle_info.next += 1; + if let Err(err) = advance_scanner_cycle(cycle_info) { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cycle_counter_exhausted", + error = %err, + "Scanner completed cycle could not advance" + ); + mark_scan_cycle_idle(cycle_info).await; + emit_scan_cycle_complete(false, cycle_start.elapsed()); + return ScannerCycleOutcome::Failed; + } cycle_info.current = 0; cycle_info.cycle_completed.push(Utc::now()); global_metrics().clear_current_scan_mode(); retain_recent_cycle_completions(&mut cycle_info.cycle_completed); global_metrics().set_cycle(Some(cycle_info.clone())).await; - if !persist_scanner_cycle_state(storeapi.clone(), cycle_info).await { + if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { mark_scan_cycle_idle(cycle_info).await; emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; @@ -1870,26 +3002,91 @@ async fn run_data_scanner_with_maintenance_state( observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await; } - let mut cycle_info = CurrentCycle::default(); - let buf = read_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .unwrap_or_default(); - if buf.len() == 8 { - cycle_info.next = u64::from_le_bytes(buf.try_into().unwrap_or_default()); - } else if buf.len() > 8 { - cycle_info.next = u64::from_le_bytes(buf[0..8].try_into().unwrap_or_default()); - if let Err(e) = cycle_info.unmarshal(&buf[8..]) { - warn!( + let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await { + Ok((buf, revision)) => (buf.unwrap_or_default(), revision), + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "revision_load_failed", + error = %err, + "Scanner cycle state revision load failed" + ); + global_metrics().set_cycle(None).await; + return Ok(()); + } + }; + let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) { + Ok(state) => state, + Err(err) => { + error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, path = %&*DATA_USAGE_BLOOM_NAME_PATH, state = "cycle_decode_failed", - error = %e, - "Scanner cycle state decode failed" + error = %err, + "Scanner stopped because persisted cycle state is invalid" ); + global_metrics().set_cycle(None).await; + return Ok(()); } + }; + let usage_floor = match persisted_usage_floor(storeapi.clone()).await { + Ok(floor) => floor, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "usage_floor_load_failed", + error = %err, + "Scanner stopped because the persisted usage floor could not be loaded" + ); + global_metrics().set_cycle(None).await; + return Ok(()); + } + }; + apply_persisted_usage_floor(&mut cycle_info, &mut leader_epoch, usage_floor); + + if ctx.is_cancelled() || guard.is_lock_lost() { + global_metrics().set_cycle(None).await; + return Ok(()); + } + let claim_ctx = ctx.child_token(); + let leadership_claimed = await_scanner_cycle_with_lock_fence( + &claim_ctx, + claim_scanner_leadership(&claim_ctx, storeapi.clone(), &mut cycle_info, &mut cycle_revision, &mut leader_epoch), + guard.lock_lost_notified(), + ) + .await + .unwrap_or(false); + if guard.is_lock_lost() { + record_scanner_leader_lock_lost("Scanner leader lock lost while claiming the leadership epoch").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + if !leadership_claimed { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_LOCK_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + lock_name = "leader.lock", + state = "epoch_claim_failed", + "Scanner stopped because the leadership epoch could not be claimed" + ); + global_metrics() + .record_scanner_leader_liveness("epoch_claim_failed", false, "leadership epoch claim failed") + .await; + global_metrics().set_cycle(None).await; + return Ok(()); } if !ctx.is_cancelled() { @@ -1902,7 +3099,14 @@ async fn run_data_scanner_with_maintenance_state( global_metrics().set_cycle(None).await; return Ok(()); } - let initial_outcome = run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + let cycle_ctx = ctx.child_token(); + let initial_outcome = await_scanner_cycle_with_lock_fence( + &cycle_ctx, + run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + guard.lock_lost_notified(), + ) + .await + .unwrap_or(ScannerCycleOutcome::Failed); dirty_usage_generation_seen = dirty_generation_before_cycle; if guard.is_lock_lost() { record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; @@ -2023,7 +3227,7 @@ async fn run_data_scanner_with_maintenance_state( maintenance: maintenance_generation_before_wait, }, || guard.is_lock_lost(), - || probe_scanner_activity(&storeapi, distributed), + || probe_scanner_activity(storeapi.as_ref(), distributed), ) .await; scanner_activity_backoff_blocked = @@ -2089,7 +3293,14 @@ async fn run_data_scanner_with_maintenance_state( break; } let dirty_generation_before_cycle = dirty_usage_generation(); - let outcome = run_data_scanner_cycle(&ctx, &storeapi, &mut cycle_info).await; + let cycle_ctx = ctx.child_token(); + let outcome = await_scanner_cycle_with_lock_fence( + &cycle_ctx, + run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + guard.lock_lost_notified(), + ) + .await + .unwrap_or(ScannerCycleOutcome::Failed); dirty_usage_generation_seen = dirty_generation_before_cycle; if guard.is_lock_lost() { record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await; @@ -2207,10 +3418,18 @@ enum DataUsagePersistOutcome { #[default] NoUpdate, Current, + AlreadyDurable, + PriorCycleDurable, Saved, Failed, } +#[derive(Clone, Debug)] +struct DataUsagePersistBaseline { + data: Option, + revision: DataUsageCacheRevision, +} + #[derive(Debug)] enum DataUsagePersistTaskResult { Completed(DataUsagePersistOutcome), @@ -2251,11 +3470,17 @@ fn scanner_cycle_completion_outcome( ) -> ScannerCycleOutcome { match (scan_status, usage_persist_outcome) { (_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed, - (ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved) if !has_failed_dirty_usage => { - ScannerCycleOutcome::Partial - } + (ScannerCycleStatus::Superseded, _) if !has_failed_dirty_usage => ScannerCycleOutcome::Superseded, + (ScannerCycleStatus::Superseded, _) => ScannerCycleOutcome::Failed, + ( + ScannerCycleStatus::Incomplete, + DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable | DataUsagePersistOutcome::PriorCycleDurable, + ) if !has_failed_dirty_usage => ScannerCycleOutcome::Partial, (ScannerCycleStatus::Incomplete, _) => ScannerCycleOutcome::Failed, - (ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved) => ScannerCycleOutcome::Completed, + ( + ScannerCycleStatus::Complete, + DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable | DataUsagePersistOutcome::PriorCycleDurable, + ) => ScannerCycleOutcome::Completed, (ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current) if !has_dirty_usage => ScannerCycleOutcome::Completed, (ScannerCycleStatus::Complete, _) => ScannerCycleOutcome::Failed, } @@ -2264,7 +3489,7 @@ fn scanner_cycle_completion_outcome( fn finalize_scanner_cycle_result( scan_cycle_result: crate::scanner_io::ScannerCycleResult, usage_persist_outcome: DataUsagePersistOutcome, -) -> (ScannerCycleOutcome, bool) { +) -> (ScannerCycleOutcome, bool, Vec) { let completion_outcome = scanner_cycle_completion_outcome( scan_cycle_result.status, usage_persist_outcome, @@ -2272,10 +3497,15 @@ fn finalize_scanner_cycle_result( scan_cycle_result.has_failed_dirty_usage(), ); let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work(); - if usage_persist_outcome == DataUsagePersistOutcome::Saved { - scan_cycle_result.acknowledge_durable_usage(); - } - (completion_outcome, pending_maintenance_work) + let remote_dirty_usage_acknowledgements = if matches!( + usage_persist_outcome, + DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable + ) { + scan_cycle_result.acknowledge_durable_usage() + } else { + Vec::new() + }; + (completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements) } /// Decide whether an incoming usage snapshot must be skipped as stale, given the local @@ -2291,6 +3521,28 @@ fn stale_data_usage_update_reason( existing: &DataUsageInfo, now: std::time::SystemTime, ) -> Option<&'static str> { + match (incoming.scanner_epoch, existing.scanner_epoch) { + (Some(incoming_epoch), Some(existing_epoch)) if incoming_epoch < existing_epoch => { + return Some("older_scanner_epoch"); + } + (Some(incoming_epoch), Some(existing_epoch)) if incoming_epoch > existing_epoch => return None, + (Some(_), None) => return None, + (None, Some(_)) => return Some("missing_incoming_scanner_epoch"), + (Some(_), Some(_)) | (None, None) => {} + } + + match (incoming.scanner_cycle, existing.scanner_cycle) { + (Some(incoming_cycle), Some(existing_cycle)) if incoming_cycle < existing_cycle => { + return Some("older_scanner_cycle"); + } + (Some(incoming_cycle), Some(existing_cycle)) if incoming_cycle == existing_cycle => { + return Some("conflicting_same_scanner_cycle"); + } + (Some(_), Some(_)) | (Some(_), None) => return None, + (None, Some(_)) => return Some("missing_incoming_scanner_cycle"), + (None, None) => {} + } + match (incoming.last_update, existing.last_update) { (Some(new_ts), Some(existing_ts)) if new_ts <= existing_ts && !rustfs_data_usage::usage_last_update_is_untrusted_future(existing_ts, now) => @@ -2302,6 +3554,17 @@ fn stale_data_usage_update_reason( } } +fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Option<&DataUsageInfo>) -> bool { + let Some(existing) = existing else { + return !incoming.buckets_usage.is_empty() || !incoming.bucket_sizes.is_empty(); + }; + incoming + .buckets_usage + .keys() + .chain(incoming.bucket_sizes.keys()) + .any(|bucket| !existing.buckets_usage.contains_key(bucket) && !existing.bucket_sizes.contains_key(bucket)) +} + /// Store data usage info in backend. Will store all objects sent on the receiver until closed. #[instrument(skip(ctx, storeapi))] pub async fn store_data_usage_in_backend( @@ -2315,36 +3578,37 @@ pub async fn store_data_usage_in_backend( async fn store_data_usage_in_backend_with_outcome( ctx: CancellationToken, storeapi: Arc, - mut receiver: mpsc::Receiver, + receiver: mpsc::Receiver, ) -> DataUsagePersistOutcome { - let mut attempts = 1u32; - let mut outcome = DataUsagePersistOutcome::NoUpdate; + store_data_usage_in_backend_with_outcome_for_epoch(ctx, storeapi, receiver, None).await +} - while let Some(data_usage_info) = receiver.recv().await { +async fn store_data_usage_in_backend_with_outcome_for_epoch( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, + leader_epoch: Option, +) -> DataUsagePersistOutcome { + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(ctx, storeapi, receiver, leader_epoch, None).await +} + +async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( + ctx: CancellationToken, + storeapi: Arc, + mut receiver: mpsc::Receiver, + leader_epoch: Option, + initial_baseline: Option, +) -> DataUsagePersistOutcome { + let mut outcome = DataUsagePersistOutcome::NoUpdate; + let mut next_baseline = initial_baseline; + + 'updates: while let Some(mut data_usage_info) = receiver.recv().await { let _activity_guard = ScannerActivityGuard::new(); if ctx.is_cancelled() { break; } - - if let Ok(buf) = read_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await - && let Ok(existing) = serde_json::from_slice::(&buf) - && let Some(reason) = stale_data_usage_update_reason(&data_usage_info, &existing, std::time::SystemTime::now()) - { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - incoming_last_update = ?data_usage_info.last_update, - existing_last_update = ?existing.last_update, - reason = reason, - state = "skip_stale_update", - "Scanner stale data usage update skipped" - ); - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale); - outcome = DataUsagePersistOutcome::Current; - continue; + if let Some(leader_epoch) = leader_epoch { + data_usage_info.scanner_epoch = Some(leader_epoch); } let data = match serde_json::to_vec(&data_usage_info) { @@ -2365,51 +3629,185 @@ async fn store_data_usage_in_backend_with_outcome( continue; } }; - let backup_data = (attempts > 10).then(|| data.clone()); + let sha256hex = (!data.is_empty()).then(|| hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower)); + let data = Bytes::from(data); + let backup_due = data_usage_backup_due(&data_usage_info); + let mut cas_retry = 0usize; + let save_outcome = loop { + if ctx.is_cancelled() { + break 'updates; + } - let done_save = Metrics::time(Metric::SaveUsage); - let save_result = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await; - done_save(); - - if let Err(e) = save_result { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "save_failed", - error = %e, - "Scanner data usage save failed" - ); - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); - outcome = DataUsagePersistOutcome::Failed; - } else { - replace_bucket_usage_memory_from_info(&data_usage_info).await; - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); - outcome = DataUsagePersistOutcome::Saved; - - if let Some(data) = backup_data { - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = save_config(storeapi.clone(), &backup_path, data).await { - warn!( + let baseline = if cas_retry == 0 { next_baseline.take() } else { None }; + let (existing_data, revision) = match baseline { + Some(baseline) => (baseline.data, baseline.revision), + None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + Ok((data, revision)) => (data.map(Bytes::from), revision), + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "revision_load_failed", + error = %e, + "Scanner data usage revision load failed" + ); + break DataUsagePersistOutcome::Failed; + } + }, + }; + let existing = existing_data + .as_deref() + .and_then(|buf| serde_json::from_slice::(buf).ok()); + if cas_retry > 0 && data_usage_reintroduces_missing_bucket(&data_usage_info, existing.as_ref()) { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + incoming_scanner_epoch = ?data_usage_info.scanner_epoch, + incoming_scanner_cycle = ?data_usage_info.scanner_cycle, + state = "skip_deleted_bucket_reintroduction", + "Scanner usage update skipped after a concurrent bucket removal" + ); + break DataUsagePersistOutcome::Current; + } + if let Some(existing) = existing.as_ref() { + if existing == &data_usage_info { + break DataUsagePersistOutcome::AlreadyDurable; + } + if existing.scanner_epoch.is_some() + && existing.scanner_epoch == data_usage_info.scanner_epoch + && existing.scanner_cycle.is_some() + && existing.scanner_cycle == data_usage_info.scanner_cycle + { + break DataUsagePersistOutcome::PriorCycleDurable; + } + if let Some(reason) = stale_data_usage_update_reason(&data_usage_info, existing, std::time::SystemTime::now()) { + debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %backup_path, - state = "backup_save_failed", - error = %e, - "Scanner data usage backup save failed" + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + incoming_scanner_epoch = ?data_usage_info.scanner_epoch, + existing_scanner_epoch = ?existing.scanner_epoch, + incoming_scanner_cycle = ?data_usage_info.scanner_cycle, + existing_scanner_cycle = ?existing.scanner_cycle, + incoming_last_update = ?data_usage_info.last_update, + existing_last_update = ?existing.last_update, + reason = reason, + state = "skip_stale_update", + "Scanner stale data usage update skipped" + ); + break DataUsagePersistOutcome::Current; + } + } + if ctx.is_cancelled() { + break 'updates; + } + + let done_save = Metrics::time(Metric::SaveUsage); + let save_result = save_config_shared_with_preconditions( + storeapi.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + data.clone(), + sha256hex.clone(), + revision.preconditions(), + ) + .await; + done_save(); + + match save_result { + Ok(object_info) => { + next_baseline = object_info + .etag + .filter(|etag| !etag.is_empty()) + .map(|etag| DataUsagePersistBaseline { + data: Some(data.clone()), + revision: DataUsageCacheRevision::Etag(etag), + }); + break DataUsagePersistOutcome::Saved; + } + Err(EcstoreError::PreconditionFailed) if cas_retry < SCANNER_PERSIST_CAS_RETRIES => { + cas_retry += 1; + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "conflict_retry", + retry = cas_retry, + "Scanner data usage CAS conflict will be reconciled" ); } - done_save(); - attempts = 1; + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = if matches!(e, EcstoreError::PreconditionFailed) { + "conflict_retries_exhausted" + } else { + "save_failed" + }, + error = %e, + "Scanner data usage save failed" + ); + break DataUsagePersistOutcome::Failed; + } + } + }; + + match save_outcome { + DataUsagePersistOutcome::Current => { + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale); + outcome = DataUsagePersistOutcome::Current; + continue; + } + DataUsagePersistOutcome::AlreadyDurable => { + replace_bucket_usage_memory_from_info(&data_usage_info).await; + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::AlreadyDurable; + } + DataUsagePersistOutcome::PriorCycleDurable => { + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::PriorCycleDurable; + } + DataUsagePersistOutcome::Failed | DataUsagePersistOutcome::NoUpdate => { + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + DataUsagePersistOutcome::Saved => { + replace_bucket_usage_memory_from_info(&data_usage_info).await; + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::Saved; } } - attempts += 1; + if backup_due { + let done_save = Metrics::time(Metric::SaveUsage); + if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), + state = "backup_save_failed", + error = %e, + "Scanner data usage backup save failed" + ); + } + done_save(); + } } outcome @@ -2445,6 +3843,42 @@ mod tests { assert_run_data_scanner_signature(run_data_scanner); } + #[tokio::test] + async fn scanner_cycle_lock_fence_cancels_cycle_context() { + let cycle_ctx = CancellationToken::new(); + let observed_ctx = cycle_ctx.clone(); + let output = await_scanner_cycle_with_lock_fence( + &cycle_ctx, + async move { + observed_ctx.cancelled().await; + observed_ctx.is_cancelled() + }, + std::future::ready(()), + ) + .await; + + assert_eq!(output, Some(true)); + assert!(cycle_ctx.is_cancelled()); + } + + #[tokio::test] + async fn scanner_cycle_lock_fence_preserves_completed_cycle() { + let cycle_ctx = CancellationToken::new(); + let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::ready(7_u8), std::future::pending()).await; + + assert_eq!(output, Some(7)); + assert!(!cycle_ctx.is_cancelled()); + } + + #[tokio::test] + async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { + let cycle_ctx = CancellationToken::new(); + let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::pending::<()>(), std::future::ready(())).await; + + assert_eq!(output, None); + assert!(cycle_ctx.is_cancelled()); + } + struct ScannerDefaultSpeedGuard; impl ScannerDefaultSpeedGuard { @@ -2478,7 +3912,13 @@ mod tests { #[derive(Debug, Default)] struct MemoryConfigStore { objects: Mutex>>, + revisions: Mutex>, fail_put_number: Mutex>, + error_after_commit_put_number: Mutex>, + interleaving_puts: Mutex)>>, + cancel_after_interleaving_puts: Mutex>, + cancel_after_successful_puts: Mutex>, + replace_after_successful_puts: Mutex)>>, put_counts: Mutex>, } @@ -2504,15 +3944,22 @@ mod tests { _h: http::HeaderMap, _opts: &ObjectOptions, ) -> EcstoreResult { - let objects = self.objects.lock().await; - let data = objects - .get(&memory_config_key(bucket, object)) + let key = memory_config_key(bucket, object); + let data = self + .objects + .lock() + .await + .get(&key) .cloned() .ok_or(EcstoreError::FileNotFound)?; + let revision = *self.revisions.lock().await.entry(key).or_insert(1); Ok(GetObjectReader { stream: Box::new(Cursor::new(data)), - object_info: ObjectInfo::default(), + object_info: ObjectInfo { + etag: Some(format!("memory-{revision}")), + ..Default::default() + }, buffered_body: None, body_source: Default::default(), }) @@ -2523,7 +3970,7 @@ mod tests { bucket: &str, object: &str, data: &mut PutObjReader, - _opts: &ObjectOptions, + opts: &ObjectOptions, ) -> EcstoreResult { let mut buf = Vec::new(); data.stream.read_to_end(&mut buf).await?; @@ -2539,8 +3986,96 @@ mod tests { return Err(EcstoreError::other("injected put failure")); } - self.objects.lock().await.insert(key, buf); - Ok(ObjectInfo::default()) + let interleaving_data = { + let mut interleaving_puts = self.interleaving_puts.lock().await; + if interleaving_puts + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + interleaving_puts.remove(&key).map(|(_, data)| data) + } else { + None + } + }; + let cancel_after_interleaving = if interleaving_data.is_some() { + self.cancel_after_interleaving_puts.lock().await.remove(&key) + } else { + None + }; + let replacement = { + let mut replacements = self.replace_after_successful_puts.lock().await; + if replacements + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + replacements.remove(&key).map(|(_, replacement)| replacement) + } else { + None + } + }; + let mut objects = self.objects.lock().await; + let mut revisions = self.revisions.lock().await; + if let Some(interleaving_data) = interleaving_data { + let revision = revisions.get(&key).copied().unwrap_or(0) + 1; + objects.insert(key.clone(), interleaving_data); + revisions.insert(key.clone(), revision); + if let Some(cancel) = cancel_after_interleaving { + cancel.cancel(); + } + } + let current_revision = objects.contains_key(&key).then(|| revisions.get(&key).copied().unwrap_or(1)); + if let Some(preconditions) = &opts.http_preconditions { + if preconditions + .if_none_match + .as_deref() + .is_some_and(|condition| !condition.trim().is_empty()) + && current_revision.is_some() + { + return Err(EcstoreError::PreconditionFailed); + } + if let Some(expected) = preconditions + .if_match + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let actual = current_revision.map(|revision| format!("memory-{revision}")); + if actual.as_deref() != Some(expected.trim_matches('"')) { + return Err(EcstoreError::PreconditionFailed); + } + } + } + + let revision = current_revision.unwrap_or(0) + 1; + objects.insert(key.clone(), buf); + revisions.insert(key.clone(), revision); + if let Some(replacement) = replacement { + objects.insert(key.clone(), replacement); + revisions.insert(key.clone(), revision + 1); + } + drop(revisions); + drop(objects); + let cancel_after_success = { + let mut cancellations = self.cancel_after_successful_puts.lock().await; + if cancellations + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + cancellations.remove(&key).map(|(_, cancel)| cancel) + } else { + None + } + }; + if let Some(cancel) = cancel_after_success { + cancel.cancel(); + } + if self.error_after_commit_put_number.lock().await.get(&key) == Some(&put_count) { + return Err(EcstoreError::other("injected post-commit put failure")); + } + Ok(ObjectInfo { + etag: Some(format!("memory-{revision}")), + ..Default::default() + }) } } @@ -2569,6 +4104,14 @@ mod tests { assert!(delay <= Duration::from_secs(132)); } + #[test] + fn test_randomized_cycle_delay_bounds_extreme_interval() { + let delay = randomized_cycle_delay_for(Duration::MAX); + + assert!(delay >= MAX_SCANNER_SCHEDULE_DELAY.mul_f64(0.9)); + assert!(delay <= MAX_SCANNER_SCHEDULE_DELAY); + } + #[test] #[serial] fn test_initial_scanner_delay_uses_configured_start_delay() { @@ -2769,6 +4312,8 @@ mod tests { #[serial] async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; let mut cycle_info = CurrentCycle { current: 12, next: 12, @@ -2776,11 +4321,12 @@ mod tests { started: Utc::now(), }; - assert!(finalize_partial_scan_cycle(store.clone(), &mut cycle_info).await); + assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1).await); assert_eq!(cycle_info.next, 13); assert_eq!(cycle_info.current, 0); assert!(cycle_info.cycle_completed.is_empty()); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) .await @@ -2789,20 +4335,20 @@ mod tests { u64::from_le_bytes(buf[0..8].try_into().expect("persisted state should start with the counter")), 13 ); - let mut decoded = CurrentCycle::default(); - decoded.unmarshal(&buf[8..]).expect("persisted cycle info should decode"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle info should decode"); assert_eq!(decoded.next, 13); assert_eq!(decoded.current, 0); + assert_eq!(epoch, 1); global_metrics().set_cycle(None).await; } #[tokio::test] #[serial] - async fn test_finalize_partial_scan_cycle_reports_persist_failure() { + async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { let store = Arc::new(MemoryConfigStore::default()); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - store.fail_put_number.lock().await.insert(key, 1); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; let mut cycle_info = CurrentCycle { current: 12, next: 12, @@ -2810,13 +4356,603 @@ mod tests { started: Utc::now(), }; - assert!(!finalize_partial_scan_cycle(store, &mut cycle_info).await); - assert_eq!(cycle_info.next, 13); + assert!(persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 19).await); assert_eq!(cycle_info.current, 0); + assert_eq!(cycle_info.next, 19); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("recovered cycle floor should be persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("recovered cycle state should decode"); + assert_eq!(decoded.current, 0); + assert_eq!(decoded.next, 19); + assert_eq!(epoch, 7); global_metrics().set_cycle(None).await; } + #[tokio::test] + #[serial] + async fn scanner_cycle_rejects_invalid_cache_floor() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + + assert!(!persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 12).await); + assert_eq!(cycle_info.next, 12); + assert_eq!(revision, DataUsageCacheRevision::Missing); + assert!( + !persist_required_scanner_cycle_floor( + &ctx, + store.clone(), + &mut CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }, + &mut revision, + 7, + u64::MAX, + ) + .await + ); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); + + global_metrics().set_cycle(None).await; + } + + #[test] + fn scanner_cycle_state_decodes_legacy_and_fenced_formats() { + let cycle = CurrentCycle { + current: 12, + next: 13, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut legacy = cycle.next.to_le_bytes().to_vec(); + legacy.extend(cycle.marshal().expect("legacy cycle state should encode")); + + let (legacy_cycle, legacy_epoch) = + decode_scanner_cycle_state(&legacy).expect("legacy cycle state should remain readable"); + assert_eq!(legacy_cycle.next, 13); + assert_eq!(legacy_epoch, 0); + + let fenced = encode_scanner_cycle_state(&cycle, 7).expect("fenced cycle state should encode"); + let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode"); + assert_eq!(fenced_cycle.next, 13); + assert_eq!(fenced_epoch, 7); + } + + #[test] + fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() { + assert_eq!( + decode_scanner_cycle_state_for_startup(&[]) + .expect("missing cycle state should use defaults") + .1, + 0 + ); + assert!(decode_scanner_cycle_state_for_startup(&[1]).is_err()); + + let mut corrupt_fenced = 13_u64.to_le_bytes().to_vec(); + corrupt_fenced.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); + corrupt_fenced.extend_from_slice(&7_u64.to_le_bytes()); + corrupt_fenced.extend_from_slice(b"not-msgpack"); + assert!(decode_scanner_cycle_state_for_startup(&corrupt_fenced).is_err()); + assert!(decode_scanner_cycle_state_for_startup(&u64::MAX.to_le_bytes()).is_err()); + + let exhausted = CurrentCycle { + next: u64::MAX, + ..Default::default() + }; + assert!(encode_scanner_cycle_state(&exhausted, 7).is_err()); + } + + #[tokio::test] + async fn scanner_startup_uses_primary_and_backup_usage_floor() { + let store = Arc::new(MemoryConfigStore::default()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, path), + serde_json::to_vec(&DataUsageInfo { + scanner_epoch: Some(epoch), + scanner_cycle: Some(cycle), + ..Default::default() + }) + .expect("usage snapshot should encode"), + ); + } + + let floor = persisted_usage_floor(store).await.expect("usage floor should load"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 104, + leader_epoch: 11, + } + ); + + let mut cycle = CurrentCycle::default(); + let mut epoch = 0; + apply_persisted_usage_floor(&mut cycle, &mut epoch, floor); + assert_eq!(cycle.next, 104); + assert_eq!(epoch, 11); + } + + #[tokio::test] + async fn scanner_startup_migrates_legacy_usage_only_until_v2_exists() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy = DataUsageInfo { + scanner_epoch: Some(19), + scanner_cycle: Some(41), + last_update: Some(std::time::SystemTime::now()), + ..Default::default() + }; + let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + legacy_data.clone(), + ); + + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("legacy startup usage should load"), + Some(legacy_data) + ); + assert_eq!( + persisted_usage_floor(store.clone()) + .await + .expect("legacy usage floor should seed the upgrade"), + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 19, + } + ); + + let authoritative = DataUsageInfo { + scanner_epoch: Some(7), + scanner_cycle: Some(11), + last_update: Some(std::time::SystemTime::now()), + ..Default::default() + }; + let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + authoritative_data.clone(), + ); + + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("v2 startup usage should load"), + Some(authoritative_data) + ); + assert_eq!( + persisted_usage_floor(store.clone()) + .await + .expect("v2 usage floor should be authoritative"), + PersistedUsageFloor { + next_cycle: 12, + leader_epoch: 7, + } + ); + + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + b"corrupt-v2".to_vec(), + ); + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("startup inspection should preserve authoritative bytes"), + Some(b"corrupt-v2".to_vec()) + ); + assert!( + persisted_usage_floor(store).await.is_err(), + "corrupt v2 state must not fall back to a legacy writer" + ); + } + + #[tokio::test] + async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { + let store = Arc::new(MemoryConfigStore::default()); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + b"not-json".to_vec(), + ); + + assert!(persisted_usage_floor(store.clone()).await.is_err()); + + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&DataUsageInfo { + scanner_cycle: Some(u64::MAX - 1), + ..Default::default() + }) + .expect("usage snapshot should encode"), + ); + assert!(persisted_usage_floor(store).await.is_err()); + } + + #[tokio::test] + #[serial] + async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + + for cycle in [9, 10] { + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + scanner_epoch: Some(1), + scanner_cycle: Some(cycle), + last_update: Some(std::time::SystemTime::now()), + ..Default::default() + }) + .await + .expect("usage update should queue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(ctx.clone(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup = read_config(store.clone(), &backup_path).await; + if cycle == 9 { + assert!(matches!(backup, Err(EcstoreError::ConfigNotFound))); + } else { + let saved = serde_json::from_slice::( + &backup.expect("the tenth durable scanner cycle should create a backup"), + ) + .expect("backup usage snapshot should decode"); + assert_eq!(saved.scanner_cycle, Some(10)); + assert_eq!(saved.scanner_epoch, Some(1)); + } + } + } + + #[test] + fn scanner_cycle_advance_fails_before_reserved_exhausted_value() { + let mut cycle = CurrentCycle { + next: u64::MAX - 2, + ..Default::default() + }; + advance_scanner_cycle(&mut cycle).expect("last persistable scanner cycle should remain valid"); + assert_eq!(cycle.next, u64::MAX - 1); + assert!(advance_scanner_cycle(&mut cycle).is_err()); + assert_eq!(cycle.next, u64::MAX - 1); + } + + #[tokio::test] + #[serial] + async fn test_finalize_partial_scan_cycle_reports_persist_failure() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.fail_put_number.lock().await.insert(key, 1); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + + assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1).await); + assert_eq!(cycle_info.next, 13); + assert_eq!(cycle_info.current, 0); + assert_eq!(revision, DataUsageCacheRevision::Missing); + + global_metrics().set_cycle(None).await; + } + + #[tokio::test] + #[serial] + async fn test_persist_scanner_cycle_state_reconciles_newer_winner() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut initial_revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); + + let mut current_revision = initial_revision.clone(); + let mut stale_revision = initial_revision; + let mut current = CurrentCycle { + next: 14, + ..initial.clone() + }; + let mut stale = CurrentCycle { next: 13, ..initial }; + + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut current_revision, 1).await); + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut stale, &mut stale_revision, 1).await); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("new leader cycle state should remain persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 14); + assert_eq!(epoch, 1); + assert_eq!(stale.next, 14); + assert!(matches!(current_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + assert!(matches!(stale_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + + global_metrics().set_cycle(None).await; + } + + #[tokio::test] + async fn test_persist_scanner_cycle_state_retries_after_stale_winner() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut initial_revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let stale = CurrentCycle { + next: 13, + ..initial.clone() + }; + let stale_buf = encode_scanner_cycle_state(&stale, 1).expect("stale cycle state should encode"); + store.interleaving_puts.lock().await.insert(key, (2, stale_buf)); + + let mut current = CurrentCycle { next: 14, ..initial }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut initial_revision, 1).await); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("newer cycle state should replace the stale conflict winner"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 14); + assert_eq!(epoch, 1); + assert_eq!(current.next, 14); + assert!(matches!(initial_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-3")); + } + + #[tokio::test] + async fn test_persist_scanner_cycle_state_stops_retry_after_leader_fence() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let replacement = CurrentCycle { + next: 13, + ..initial.clone() + }; + let replacement_buf = encode_scanner_cycle_state(&replacement, 2).expect("replacement cycle state should encode"); + store.interleaving_puts.lock().await.insert(key.clone(), (2, replacement_buf)); + store + .cancel_after_interleaving_puts + .lock() + .await + .insert(key.clone(), ctx.clone()); + + let mut stale_leader = CurrentCycle { next: 14, ..initial }; + assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut stale_leader, &mut revision, 1).await); + + let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("replacement leader cycle state should remain persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 13); + assert_eq!(epoch, 2); + assert_eq!(stale_leader.next, 14); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); + } + + #[tokio::test] + async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conflict() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let old_epoch_commit = CurrentCycle { + next: 14, + ..cycle.clone() + }; + store.interleaving_puts.lock().await.insert( + key.clone(), + ( + 2, + encode_scanner_cycle_state(&old_epoch_commit, 1).expect("old-epoch cycle state should encode"), + ), + ); + + let mut persisted_epoch = 8; + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("new leadership claim should remain persisted"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); + assert_eq!(claimed_cycle.next, 14); + assert_eq!(claimed_epoch, 9); + assert_eq!(persisted_epoch, 9); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&3)); + } + + #[tokio::test] + async fn test_leadership_claim_confirms_commit_after_returned_error() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store.error_after_commit_put_number.lock().await.insert(key.clone(), 1); + store.error_after_commit_put_number.lock().await.insert(usage_key.clone(), 1); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut persisted_epoch = 0; + + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); + + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("ambiguous leadership claim should be durable"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); + assert_eq!(claimed_cycle.next, 12); + assert_eq!(claimed_epoch, 1); + assert_eq!(persisted_epoch, 1); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); + let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("ambiguous usage epoch fence should be durable"); + assert_eq!( + serde_json::from_slice::(&usage) + .expect("usage epoch fence should decode") + .scanner_epoch, + Some(1) + ); + assert_eq!(store.put_counts.lock().await.get(&usage_key), Some(&1)); + } + + #[tokio::test] + async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut old_usage = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(4), + scanner_cycle: Some(11), + ..Default::default() + }; + old_usage.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + old_usage.buckets_count = 1; + old_usage.calculate_totals(); + let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); + store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + started: Utc::now(), + ..Default::default() + }; + let mut persisted_epoch = 4; + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); + + let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage snapshot should load"); + let fenced = serde_json::from_slice::(fenced_data.as_deref().expect("fenced usage snapshot should exist")) + .expect("fenced usage snapshot should decode"); + assert_eq!(fenced.scanner_epoch, Some(5)); + assert_eq!(fenced.objects_total_count, 2); + assert_eq!(fenced.buckets_usage.get("bucket-a").map(|usage| usage.size), Some(84)); + assert!(matches!(fenced_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + + let stale_save = save_config_with_preconditions( + store, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + old_data, + DataUsageCacheRevision::Etag("memory-1".to_string()).preconditions(), + ) + .await; + assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed))); + } + + #[tokio::test] + async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store + .cancel_after_successful_puts + .lock() + .await + .insert(key.clone(), (2, ctx.clone())); + cycle.next = 14; + assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let (persisted, persisted_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("committed old-epoch state should load"); + let mut replacement_cycle = decode_scanner_cycle_state( + persisted + .as_deref() + .expect("old-epoch state should have committed before cancellation"), + ) + .expect("old-epoch state should decode") + .0; + let mut replacement_revision = persisted_revision; + let mut replacement_epoch = 1; + let replacement_ctx = CancellationToken::new(); + assert!( + claim_scanner_leadership( + &replacement_ctx, + store.clone(), + &mut replacement_cycle, + &mut replacement_revision, + &mut replacement_epoch, + ) + .await + ); + + let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("replacement leadership claim should persist"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode"); + assert_eq!(claimed_cycle.next, 14); + assert_eq!(claimed_epoch, 2); + } + #[tokio::test] async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { let store = Arc::new(MemoryConfigStore::default()); @@ -2851,6 +4987,316 @@ mod tests { assert_eq!(outcome, DataUsagePersistOutcome::Current); } + #[tokio::test] + async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let newer = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + buckets_count: 2, + ..Default::default() + }; + let stale = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + buckets_count: 1, + ..Default::default() + }; + store + .interleaving_puts + .lock() + .await + .insert(key.clone(), (1, serde_json::to_vec(&newer).expect("newer usage snapshot should encode"))); + + sender.send(stale).await.expect("stale usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&key) + .expect("interleaving newer usage snapshot should remain saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.buckets_count, 2); + assert_eq!(saved.last_update, newer.last_update); + assert_eq!(outcome, DataUsagePersistOutcome::Current); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_after_conflict() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut initial = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + ..Default::default() + }; + initial.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + initial.bucket_sizes.insert("bucket-a".to_string(), 84); + initial.buckets_count = 1; + initial.calculate_totals(); + let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); + store.objects.lock().await.insert(key.clone(), initial_data.clone()); + store.revisions.lock().await.insert(key.clone(), 1); + + let mut deleted = initial.clone(); + deleted.buckets_usage.clear(); + deleted.bucket_sizes.clear(); + deleted.buckets_count = 0; + deleted.calculate_totals(); + store + .interleaving_puts + .lock() + .await + .insert(key.clone(), (1, serde_json::to_vec(&deleted).expect("deleted snapshot should encode"))); + + let mut incoming = initial; + incoming.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)); + incoming.scanner_cycle = Some(13); + let (sender, receiver) = mpsc::channel(1); + sender.send(incoming).await.expect("stale scanner snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( + CancellationToken::new(), + store.clone(), + receiver, + Some(8), + Some(DataUsagePersistBaseline { + data: Some(Bytes::from(initial_data)), + revision: DataUsageCacheRevision::Etag("memory-1".to_string()), + }), + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Current); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("deleted usage snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("deleted usage snapshot should decode"); + assert!(!saved.buckets_usage.contains_key("bucket-a")); + assert!(!saved.bucket_sizes.contains_key("bucket-a")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() { + let store = Arc::new(MemoryConfigStore::default()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + let deleted = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(1), + ..Default::default() + }; + store.objects.lock().await.insert( + backup_key.clone(), + serde_json::to_vec(&deleted).expect("deleted backup snapshot should encode"), + ); + store.revisions.lock().await.insert(backup_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(11); + for cycle in 2_u64..=12 { + let mut incoming = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20 + cycle)), + scanner_epoch: Some(8), + scanner_cycle: Some(cycle), + ..Default::default() + }; + incoming.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + incoming.bucket_sizes.insert("bucket-a".to_string(), 84); + incoming.buckets_count = 1; + incoming.calculate_totals(); + sender.send(incoming).await.expect("usage snapshot should enqueue"); + } + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&backup_key) + .cloned() + .expect("deleted backup snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("backup snapshot should decode"); + assert!(saved.buckets_usage.contains_key("bucket-a")); + assert!(saved.bucket_sizes.contains_key("bucket-a")); + assert_eq!(saved.scanner_cycle, Some(10)); + assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_commit() { + let store = Arc::new(MemoryConfigStore::default()); + let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + let durable = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(10), + ..Default::default() + }; + let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode"); + store.objects.lock().await.insert(main_key.clone(), encoded.clone()); + store.revisions.lock().await.insert(main_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender.send(durable).await.expect("usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::AlreadyDurable + ); + assert_eq!(store.objects.lock().await.get(&backup_key), Some(&encoded)); + assert_eq!(store.put_counts.lock().await.get(&main_key), None); + assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_backup() { + let store = Arc::new(MemoryConfigStore::default()); + let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + + let mut incoming = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(10), + ..Default::default() + }; + incoming.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + incoming.bucket_sizes.insert("bucket-a".to_string(), 84); + incoming.buckets_count = 1; + incoming.calculate_totals(); + + let mut deleted = incoming.clone(); + deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31)); + deleted.buckets_usage.clear(); + deleted.bucket_sizes.clear(); + deleted.buckets_count = 0; + deleted.calculate_totals(); + store.replace_after_successful_puts.lock().await.insert( + main_key.clone(), + (1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")), + ); + store.objects.lock().await.insert( + backup_key.clone(), + serde_json::to_vec(&incoming).expect("existing backup snapshot should encode"), + ); + store.revisions.lock().await.insert(backup_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender.send(incoming).await.expect("usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + + for key in [main_key, backup_key] { + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("usage snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("usage snapshot should decode"); + assert!(!saved.buckets_usage.contains_key("bucket-a")); + assert!(!saved.bucket_sizes.contains_key("bucket-a")); + } + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let initial = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + buckets_count: 1, + ..Default::default() + }; + let stale_winner = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + buckets_count: 2, + ..Default::default() + }; + let current = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + buckets_count: 3, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&initial).expect("initial usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + store.interleaving_puts.lock().await.insert( + key.clone(), + (1, serde_json::to_vec(&stale_winner).expect("stale usage snapshot should encode")), + ); + + sender + .send(current.clone()) + .await + .expect("current usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&key) + .expect("current usage snapshot should replace the stale conflict winner"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.buckets_count, 3); + assert_eq!(saved.last_update, current.last_update); + assert_eq!(outcome, DataUsagePersistOutcome::Saved); + drop(objects); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); + } + #[tokio::test] async fn test_store_data_usage_in_backend_rejects_untimestamped_stale_snapshot() { let store = Arc::new(MemoryConfigStore::default()); @@ -2891,6 +5337,264 @@ mod tests { assert_eq!(outcome, DataUsagePersistOutcome::Current); } + #[tokio::test] + async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let snapshot = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_cycle: Some(12), + buckets_count: 2, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&snapshot).expect("durable usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + sender + .send(snapshot) + .await + .expect("ambiguous committed snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + assert_eq!(outcome, DataUsagePersistOutcome::AlreadyDurable); + assert_eq!(store.put_counts.lock().await.get(&key), None); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let durable = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + buckets_count: 2, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&durable).expect("durable usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + buckets_count: 3, + ..Default::default() + }) + .await + .expect("changed retry snapshot should enqueue"); + drop(sender); + + let outcome = + store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), receiver, Some(8)).await; + + assert_eq!(outcome, DataUsagePersistOutcome::PriorCycleDurable); + assert_eq!(store.put_counts.lock().await.get(&key), None); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("first snapshot should remain durable"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("durable usage snapshot should decode") + .buckets_count, + 2 + ); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_clock() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), + scanner_cycle: Some(12), + buckets_count: 2, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (older_sender, older_receiver) = mpsc::channel(1); + let older = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_cycle: Some(11), + buckets_count: 1, + ..Default::default() + }; + older_sender.send(older).await.expect("older-cycle snapshot should enqueue"); + drop(older_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, + DataUsagePersistOutcome::Current + ); + + let (newer_sender, newer_receiver) = mpsc::channel(1); + let newer = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_cycle: Some(13), + buckets_count: 3, + ..Default::default() + }; + newer_sender + .send(newer.clone()) + .await + .expect("newer-cycle snapshot should enqueue"); + drop(newer_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), newer_receiver).await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("newer scanner cycle should be persisted"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("persisted usage snapshot should decode") + .scanner_cycle, + Some(13) + ); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + buckets_count: 2, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (older_sender, older_receiver) = mpsc::channel(1); + older_sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_epoch: Some(7), + scanner_cycle: Some(99), + buckets_count: 1, + ..Default::default() + }) + .await + .expect("old-epoch snapshot should enqueue"); + drop(older_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, + DataUsagePersistOutcome::Current + ); + + let (newer_sender, newer_receiver) = mpsc::channel(1); + newer_sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_epoch: None, + scanner_cycle: Some(1), + buckets_count: 3, + ..Default::default() + }) + .await + .expect("replacement-epoch snapshot should enqueue"); + drop(newer_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), newer_receiver, Some(9),) + .await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("replacement leader snapshot should persist"); + let saved = serde_json::from_slice::(&saved).expect("persisted usage snapshot should decode"); + assert_eq!(saved.scanner_epoch, Some(9)); + assert_eq!(saved.scanner_cycle, Some(1)); + } + + #[tokio::test] + async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_cycle: Some(12), + buckets_count: 2, + ..Default::default() + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_cycle: Some(12), + buckets_count: 3, + ..Default::default() + }) + .await + .expect("conflicting same-cycle snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Current + ); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("first same-cycle snapshot should remain persisted"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("persisted usage snapshot should decode") + .buckets_count, + 2 + ); + } + fn usage_with_last_update(last_update: Option) -> DataUsageInfo { DataUsageInfo { last_update, @@ -3038,6 +5742,19 @@ mod tests { scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved, true, false), ScannerCycleOutcome::Completed ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::AlreadyDurable, true, false,), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome( + ScannerCycleStatus::Complete, + DataUsagePersistOutcome::PriorCycleDurable, + true, + false, + ), + ScannerCycleOutcome::Completed + ); assert_eq!( scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, false, false), ScannerCycleOutcome::Completed @@ -3050,6 +5767,20 @@ mod tests { scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::NoUpdate, false, false), ScannerCycleOutcome::Failed ); + for persist_outcome in [ + DataUsagePersistOutcome::NoUpdate, + DataUsagePersistOutcome::Current, + DataUsagePersistOutcome::Saved, + ] { + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, persist_outcome, true, false), + ScannerCycleOutcome::Superseded + ); + } + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, DataUsagePersistOutcome::Saved, true, true), + ScannerCycleOutcome::Failed + ); } #[test] @@ -3059,17 +5790,111 @@ mod tests { crate::scanner_io::record_dirty_usage_bucket("photos"); let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); - let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone())); - let (outcome, _) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate); + let remote_acknowledgement = ScannerDirtyUsageAcknowledgement { + host: "node-2".to_string(), + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + generation: 11, + }; + let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone())) + .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate); assert_eq!(outcome, ScannerCycleOutcome::Failed); + assert!(acknowledgements.is_empty()); assert!(crate::scanner_io::dirty_usage_buckets_pending()); - let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); - let (outcome, _) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved); + let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)) + .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved); assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert_eq!(acknowledgements, vec![remote_acknowledgement]); assert!(!crate::scanner_io::dirty_usage_buckets_pending()); } + #[tokio::test] + async fn scanner_cycle_keeps_remote_pending_acknowledgement() { + let pending = + remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(true))).await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending), + ScannerCycleOutcome::CompletedWithPendingMaintenance + ); + + let cleared = + remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(false))).await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared), + ScannerCycleOutcome::Completed + ); + + let failed = remote_dirty_usage_acknowledgement_pending( + 7, + 1, + std::future::ready(Err::(std::io::Error::other("injected acknowledgement failure"))), + ) + .await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, failed), + ScannerCycleOutcome::CompletedWithPendingMaintenance + ); + } + + #[test] + #[serial] + fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable); + + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(acknowledgements.is_empty()); + assert!(!crate::scanner_io::dirty_usage_buckets_pending()); + } + + #[test] + #[serial] + fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable); + + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(acknowledgements.is_empty()); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + crate::scanner_io::clear_dirty_usage_bucket("photos"); + } + + #[test] + #[serial] + fn finalizing_a_superseded_cycle_keeps_dirty_work_pending() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::NoUpdate); + + assert_eq!(outcome, ScannerCycleOutcome::Superseded); + assert!(acknowledgements.is_empty()); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + crate::scanner_io::clear_dirty_usage_bucket("photos"); + } + + #[test] + #[serial] + fn data_usage_persist_wait_covers_cache_retries_and_backup() { + with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + assert_eq!(data_usage_persist_timeout(), Duration::from_millis(31_350)); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + } + #[tokio::test] async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { let ctx = CancellationToken::new(); @@ -3922,9 +6747,106 @@ mod tests { instance_id: epoch.to_string(), namespace_generation, maintenance_generation, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: [3; 32], + data_movement_active: false, + dirty_usage_generation: 5, + dirty_usage_pending: false, } } + #[test] + fn scanner_activity_snapshot_digest_fences_storage_topology() { + let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut changed = first.clone(); + changed.get_mut("node-2").expect("node should exist").topology_digest = [4; 32]; + + assert_ne!(scanner_activity_snapshot_digest(&first), scanner_activity_snapshot_digest(&changed)); + } + + #[test] + fn scanner_activity_snapshot_digest_fences_peer_protocol_upgrades() { + let legacy = BTreeMap::from([( + "node-2".to_string(), + ScannerNodeActivity { + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + ..scanner_node_activity("epoch-a", 7, 3) + }, + )]); + let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + + assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(¤t)); + } + + #[test] + fn scanner_activity_snapshot_fences_data_movement() { + let idle = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut moving = idle.clone(); + moving.get_mut("node-2").expect("node should exist").data_movement_active = true; + + assert!(scanner_activity_allows_usage_publication(&idle)); + assert!(!scanner_activity_allows_usage_publication(&moving)); + assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving)); + } + + #[test] + fn scanner_activity_snapshot_digest_fences_dirty_usage_state() { + let clean = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let pending = BTreeMap::from([( + "node-2".to_string(), + ScannerNodeActivity { + dirty_usage_generation: 6, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-a", 7, 3) + }, + )]); + + assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); + } + + #[test] + fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { + let snapshot = BTreeMap::from([ + ( + LOCAL_SCANNER_ACTIVITY_NODE.to_string(), + ScannerNodeActivity { + dirty_usage_generation: 7, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-local", 7, 3) + }, + ), + ("node-2".to_string(), scanner_node_activity("epoch-clean", 7, 3)), + ( + "node-3".to_string(), + ScannerNodeActivity { + dirty_usage_generation: 11, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-dirty", 7, 3) + }, + ), + ]); + + assert_eq!( + scanner_dirty_usage_acknowledgements(&snapshot), + vec![ScannerDirtyUsageAcknowledgement { + host: "node-3".to_string(), + instance_id: "epoch-dirty".to_string(), + generation: 11, + }] + ); + } + + #[test] + fn scanner_activity_rejects_one_process_claimed_by_multiple_hosts() { + let mut instances = BTreeMap::new(); + record_scanner_activity_instance(&mut instances, "node-1", "0123456789abcdef0123456789abcdef") + .expect("first host should establish the instance identity"); + let err = record_scanner_activity_instance(&mut instances, "node-2", "0123456789abcdef0123456789abcdef") + .expect_err("a process identity must not represent two cluster nodes"); + + assert!(err.contains("node-1 and node-2")); + } + #[test] fn scanner_activity_observation_requires_a_complete_baseline() { let mut seen = None; diff --git a/crates/scanner/src/scanner_budget.rs b/crates/scanner/src/scanner_budget.rs index b7e080c82..43ce732d5 100644 --- a/crates/scanner/src/scanner_budget.rs +++ b/crates/scanner/src/scanner_budget.rs @@ -16,6 +16,7 @@ use std::sync::{ Arc, atomic::{AtomicU8, AtomicU64, Ordering}, }; +use std::time::Instant; use tokio::time::Duration; use tokio_util::sync::CancellationToken; @@ -61,15 +62,26 @@ impl ScannerCycleBudgetReason { pub struct ScannerCycleBudget { token: CancellationToken, reason: Arc, + started_at: Instant, max_duration: Option, max_objects: Option, max_directories: Option, + track_progress: bool, objects_scanned: AtomicU64, directories_started: AtomicU64, + entries_visited: AtomicU64, } impl ScannerCycleBudget { pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { + Self::new_inner(parent, config, false) + } + + pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { + Self::new_inner(parent, config, true) + } + + fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc { let token = parent.child_token(); let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE)); @@ -92,11 +104,14 @@ impl ScannerCycleBudget { Arc::new(Self { token, reason, + started_at: Instant::now(), max_duration: config.max_duration, max_objects: config.max_objects, max_directories: config.max_directories, + track_progress, objects_scanned: AtomicU64::new(0), directories_started: AtomicU64::new(0), + entries_visited: AtomicU64::new(0), }) } @@ -124,27 +139,97 @@ impl ScannerCycleBudget { self.max_directories } - pub(crate) fn try_start_directory(&self) -> bool { - let Some(max_directories) = self.max_directories else { - return true; - }; + pub(crate) fn requires_serial_progress_accounting(&self) -> bool { + self.max_objects.is_some() || self.max_directories.is_some() + } - let directories = self.directories_started.fetch_add(1, Ordering::Relaxed) + 1; - if directories <= max_directories { + pub(crate) fn remaining_config(&self) -> ScannerCycleBudgetConfig { + let max_duration = self + .max_duration + .map(|duration| duration.saturating_sub(self.started_at.elapsed())); + if max_duration.is_some_and(|duration| duration.is_zero()) { + self.cancel_for(ScannerCycleBudgetReason::Runtime); + } + + ScannerCycleBudgetConfig { + max_duration, + max_objects: self + .max_objects + .map(|max| max.saturating_sub(self.objects_scanned.load(Ordering::Relaxed))), + max_directories: self + .max_directories + .map(|max| max.saturating_sub(self.directories_started.load(Ordering::Relaxed))), + } + } + + pub(crate) fn progress(&self) -> (u64, u64) { + ( + self.objects_scanned.load(Ordering::Relaxed), + self.directories_started.load(Ordering::Relaxed), + ) + } + + pub(crate) fn entries_visited(&self) -> u64 { + self.entries_visited.load(Ordering::Relaxed) + } + + pub(crate) fn record_entries_visited(&self, entries_visited: u64) { + if self.track_progress { + saturating_fetch_add(&self.entries_visited, entries_visited); + } + } + + pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) { + if self.track_progress || self.max_objects.is_some() { + let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned); + if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { + self.cancel_for(ScannerCycleBudgetReason::Objects); + } + } + + if self.track_progress || self.max_directories.is_some() { + let directories = saturating_fetch_add(&self.directories_started, directories_started); + if self + .max_directories + .is_some_and(|max_directories| directories > max_directories) + { + self.cancel_for(ScannerCycleBudgetReason::Directories); + } + } + } + + pub(crate) fn cancel_after_unreported_remote_progress(&self) { + if self.max_objects.is_some() { + self.cancel_for(ScannerCycleBudgetReason::Objects); + } else if self.max_directories.is_some() { + self.cancel_for(ScannerCycleBudgetReason::Directories); + } + } + + pub(crate) fn try_start_directory(&self) -> bool { + if !self.track_progress && self.max_directories.is_none() { return true; } - self.cancel_for(ScannerCycleBudgetReason::Directories); - false + let directories = saturating_fetch_add(&self.directories_started, 1); + if self + .max_directories + .is_some_and(|max_directories| directories > max_directories) + { + self.cancel_for(ScannerCycleBudgetReason::Directories); + return false; + } + + true } pub(crate) fn record_object_scanned(&self) { - let Some(max_objects) = self.max_objects else { + if !self.track_progress && self.max_objects.is_none() { return; - }; + } - let objects = self.objects_scanned.fetch_add(1, Ordering::Relaxed) + 1; - if objects >= max_objects { + let objects = saturating_fetch_add(&self.objects_scanned, 1); + if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { self.cancel_for(ScannerCycleBudgetReason::Objects); } } @@ -163,6 +248,17 @@ impl ScannerCycleBudget { } } +fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 { + let mut current = value.load(Ordering::Relaxed); + loop { + let next = current.saturating_add(delta); + match value.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => return next, + Err(observed) => current = observed, + } + } +} + impl Drop for ScannerCycleBudget { fn drop(&mut self) { self.token.cancel(); @@ -250,4 +346,119 @@ mod tests { assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects)); assert!(budget.token().is_cancelled()); } + + #[tokio::test] + async fn remaining_config_accounts_for_local_and_remote_progress() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + max_objects: Some(10), + max_directories: Some(5), + }, + ); + + budget.record_object_scanned(); + assert!(budget.try_start_directory()); + budget.record_remote_progress(3, 2); + + let remaining = budget.remaining_config(); + assert!( + remaining + .max_duration + .is_some_and(|duration| duration <= Duration::from_secs(60)) + ); + assert_eq!(remaining.max_objects, Some(6)); + assert_eq!(remaining.max_directories, Some(2)); + assert_eq!(budget.progress(), (4, 3)); + assert!(!budget.budget_elapsed()); + } + + #[test] + fn remote_progress_preserves_object_and_directory_limit_semantics() { + let parent = CancellationToken::new(); + let object_budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(2), + ..Default::default() + }, + ); + object_budget.record_remote_progress(2, 0); + assert_eq!(object_budget.reason(), Some(ScannerCycleBudgetReason::Objects)); + + let directory_budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(2), + ..Default::default() + }, + ); + directory_budget.record_remote_progress(0, 2); + assert!(!directory_budget.budget_elapsed()); + directory_budget.record_remote_progress(0, 1); + assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories)); + } + + #[test] + fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default()); + + assert!(budget.try_start_directory()); + budget.record_object_scanned(); + budget.record_remote_progress(2, 3); + + assert_eq!(budget.progress(), (3, 4)); + assert!(!budget.budget_elapsed()); + assert!(!budget.token().is_cancelled()); + } + + #[test] + fn unreported_remote_progress_cancels_count_budget() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(10), + ..Default::default() + }, + ); + + budget.cancel_after_unreported_remote_progress(); + + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects)); + assert!(budget.token().is_cancelled()); + } + + #[tokio::test] + async fn count_budgets_require_serial_progress_accounting() { + let parent = CancellationToken::new(); + let runtime_only = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(1)), + ..Default::default() + }, + ); + let object_limited = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(1), + ..Default::default() + }, + ); + let directory_limited = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(1), + ..Default::default() + }, + ); + + assert!(!runtime_only.requires_serial_progress_accounting()); + assert!(object_limited.requires_serial_progress_accounting()); + assert!(directory_limited.requires_serial_progress_accounting()); + } } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index bb5c67c15..ad05c1cfc 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -38,8 +38,8 @@ use rustfs_common::heal_channel::{ HealRequestSource, HealScanMode, send_heal_request_with_admission, }; use rustfs_common::metrics::{ - IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, - current_path_updater, global_metrics, + CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, + UpdateCurrentPathFn, current_path_updater, global_metrics, }; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; @@ -76,6 +76,8 @@ const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000; const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4; const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000; const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60); +const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32; +const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30); const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024; const ENV_DATA_USAGE_UPDATE_DIR_CYCLES: &str = "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES"; const ENV_HEAL_OBJECT_SELECT_PROB: &str = "RUSTFS_HEAL_OBJECT_SELECT_PROB"; @@ -411,24 +413,36 @@ fn non_negative_i64_to_u64(value: i64) -> u64 { } fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) { - into.size += summary.total_size; - into.versions += summary.versions; - into.delete_markers += summary.delete_markers; - into.obj_sizes.add(summary.total_size as u64); - into.obj_versions.add(summary.versions as u64); + into.size = into.size.saturating_add(summary.total_size); + into.versions = into.versions.saturating_add(summary.versions); + into.delete_markers = into.delete_markers.saturating_add(summary.delete_markers); + into.obj_sizes.add(u64::try_from(summary.total_size).unwrap_or(u64::MAX)); + into.obj_versions.add(u64::try_from(summary.versions).unwrap_or(u64::MAX)); let replication_stats = into.replication_stats.get_or_insert_with(Default::default); - replication_stats.replica_size += non_negative_i64_to_u64(summary.replica_size); - replication_stats.replica_count += summary.replica_count as u64; + replication_stats.replica_size = replication_stats + .replica_size + .saturating_add(non_negative_i64_to_u64(summary.replica_size)); + replication_stats.replica_count = replication_stats + .replica_count + .saturating_add(u64::try_from(summary.replica_count).unwrap_or(u64::MAX)); for (arn, st) in &summary.repl_target_stats { let tgt_stat = replication_stats.targets.entry(arn.clone()).or_default(); - tgt_stat.pending_size += non_negative_i64_to_u64(st.pending_size); - tgt_stat.failed_size += non_negative_i64_to_u64(st.failed_size); - tgt_stat.replicated_size += non_negative_i64_to_u64(st.replicated_size); - tgt_stat.replicated_count += st.replicated_count as u64; - tgt_stat.failed_count += st.failed_count as u64; - tgt_stat.pending_count += st.pending_count as u64; + tgt_stat.pending_size = tgt_stat.pending_size.saturating_add(non_negative_i64_to_u64(st.pending_size)); + tgt_stat.failed_size = tgt_stat.failed_size.saturating_add(non_negative_i64_to_u64(st.failed_size)); + tgt_stat.replicated_size = tgt_stat + .replicated_size + .saturating_add(non_negative_i64_to_u64(st.replicated_size)); + tgt_stat.replicated_count = tgt_stat + .replicated_count + .saturating_add(u64::try_from(st.replicated_count).unwrap_or(u64::MAX)); + tgt_stat.failed_count = tgt_stat + .failed_count + .saturating_add(u64::try_from(st.failed_count).unwrap_or(u64::MAX)); + tgt_stat.pending_count = tgt_stat + .pending_count + .saturating_add(u64::try_from(st.pending_count).unwrap_or(u64::MAX)); } } @@ -1036,22 +1050,23 @@ impl ScannerItem { if let Some(repl_target_size_summary) = size_summary.repl_target_stats.get_mut(arn.as_str()) { match target_status { ReplicationStatusType::Pending => { - repl_target_size_summary.pending_size += roi.size; - repl_target_size_summary.pending_count += 1; - size_summary.pending_size += roi.size; - size_summary.pending_count += 1; + repl_target_size_summary.pending_size = repl_target_size_summary.pending_size.saturating_add(roi.size); + repl_target_size_summary.pending_count = repl_target_size_summary.pending_count.saturating_add(1); + size_summary.pending_size = size_summary.pending_size.saturating_add(roi.size); + size_summary.pending_count = size_summary.pending_count.saturating_add(1); } ReplicationStatusType::Failed => { - repl_target_size_summary.failed_size += roi.size; - repl_target_size_summary.failed_count += 1; - size_summary.failed_size += roi.size; - size_summary.failed_count += 1; + repl_target_size_summary.failed_size = repl_target_size_summary.failed_size.saturating_add(roi.size); + repl_target_size_summary.failed_count = repl_target_size_summary.failed_count.saturating_add(1); + size_summary.failed_size = size_summary.failed_size.saturating_add(roi.size); + size_summary.failed_count = size_summary.failed_count.saturating_add(1); } ReplicationStatusType::Completed | ReplicationStatusType::CompletedLegacy => { - repl_target_size_summary.replicated_size += roi.size; - repl_target_size_summary.replicated_count += 1; - size_summary.replicated_size += roi.size; - size_summary.replicated_count += 1; + repl_target_size_summary.replicated_size = + repl_target_size_summary.replicated_size.saturating_add(roi.size); + repl_target_size_summary.replicated_count = repl_target_size_summary.replicated_count.saturating_add(1); + size_summary.replicated_size = size_summary.replicated_size.saturating_add(roi.size); + size_summary.replicated_count = size_summary.replicated_count.saturating_add(1); } _ => {} } @@ -1059,8 +1074,8 @@ impl ScannerItem { } if oi.replication_status == ReplicationStatusType::Replica { - size_summary.replica_size += roi.size; - size_summary.replica_count += 1; + size_summary.replica_size = size_summary.replica_size.saturating_add(roi.size); + size_summary.replica_count = size_summary.replica_count.saturating_add(1); } } @@ -1835,21 +1850,10 @@ impl FolderScanner { let mut dir_reader = match tokio::fs::read_dir(&dir_path).await { Ok(dir_reader) => dir_reader, - Err(e) if e.kind() == ErrorKind::NotFound => { - debug!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_FOLDER_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_FOLDER, - dir_path = %dir_path, - state = "dir_missing_before_read", - error = %e, - "Scanner folder state updated" - ); - return Ok(()); - } Err(e) => return Err(ScannerError::Io(e)), }; + let mut pending_entry_progress = 0_u64; + let mut last_entry_progress = Instant::now(); loop { let entry = match dir_reader.next_entry().await { @@ -1883,6 +1887,14 @@ impl FolderScanner { } Err(e) => return Err(ScannerError::Io(e)), }; + pending_entry_progress = pending_entry_progress.saturating_add(1); + if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH + || last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL + { + self.budget.record_entries_visited(pending_entry_progress); + pending_entry_progress = 0; + last_entry_progress = Instant::now(); + } let file_name = entry.file_name().to_string_lossy().to_string(); if file_name.is_empty() || file_name == "." || file_name == ".." { continue; @@ -2122,6 +2134,7 @@ impl FolderScanner { global_metrics().record_scanner_yield(yield_start.elapsed()); } } + self.budget.record_entries_visited(pending_entry_progress); if ctx.is_cancelled() { return Err(ScannerError::Other("Operation cancelled".to_string())); @@ -2759,7 +2772,8 @@ pub async fn scan_data_folder( // Get disk path let base_path = local_disk.path().to_string_lossy().to_string(); - let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name); + let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name).await; + let mut close_disk_guard = CloseDiskGuard::new(close_disk); // Create skip_heal flag let is_erasure_mode = scanner_is_erasure().await; @@ -2829,6 +2843,8 @@ pub async fn scan_data_folder( new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN); new_cache.info.last_update = Some(SystemTime::now()); new_cache.info.next_cycle = cache.info.next_cycle; + let unresolved_objects = root.failed_objects > 0 || !new_cache.info.failed_objects.is_empty(); + new_cache.info.snapshot_complete = !unresolved_objects; let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some(); new_cache.info.scan_resume_after = None; new_cache.info.scan_checkpoint = None; @@ -2836,8 +2852,12 @@ pub async fn scan_data_folder( global_metrics().record_scanner_checkpoint_cleared(); } - close_disk().await; - Ok(new_cache.clone()) + close_disk_guard.close().await; + if unresolved_objects { + Err(ScannerError::PartialCache(Box::new(new_cache.clone()))) + } else { + Ok(new_cache.clone()) + } } Err(e) => { if ctx.is_cancelled() { @@ -2857,14 +2877,23 @@ pub async fn scan_data_folder( } new_cache.info.last_update = Some(SystemTime::now()); new_cache.info.next_cycle = cache.info.next_cycle; + new_cache.info.snapshot_complete = false; if root_has_progress { set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason())); } - close_disk().await; + close_disk_guard.close().await; return Err(ScannerError::PartialCache(Box::new(new_cache.clone()))); } } - close_disk().await; + if matches!(&e, ScannerError::Io(io) if io.kind() == ErrorKind::NotFound) { + let mut partial_cache = scanner.old_cache.clone(); + partial_cache.info.last_update = Some(SystemTime::now()); + partial_cache.info.next_cycle = cache.info.next_cycle; + partial_cache.info.snapshot_complete = false; + close_disk_guard.close().await; + return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache))); + } + close_disk_guard.close().await; // No useful information, return original cache Err(e) } @@ -2877,7 +2906,7 @@ mod tests { use super::*; use crate::storage_api::VersionPurgeStatusType; - use crate::{DiskOption, Endpoint, new_disk}; + use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk}; use rustfs_filemeta::{FileInfo, FileMeta}; use serial_test::serial; #[cfg(unix)] @@ -2886,6 +2915,76 @@ mod tests { use temp_env::{with_var, with_var_unset}; use uuid::Uuid; + #[test] + fn scanner_size_summary_application_saturates_usage_counters() { + let target = "arn:minio:replication::target".to_string(); + let mut entry = DataUsageEntry { + size: usize::MAX, + versions: usize::MAX, + delete_markers: usize::MAX, + replication_stats: Some(Default::default()), + ..Default::default() + }; + let replication_stats = entry + .replication_stats + .as_mut() + .expect("replication statistics should be initialized"); + replication_stats.replica_size = u64::MAX; + replication_stats.replica_count = u64::MAX; + let target_stats = replication_stats.targets.entry(target.clone()).or_default(); + target_stats.pending_size = u64::MAX; + target_stats.failed_size = u64::MAX; + target_stats.replicated_size = u64::MAX; + target_stats.replicated_count = u64::MAX; + target_stats.failed_count = u64::MAX; + target_stats.pending_count = u64::MAX; + + let mut summary = SizeSummary { + total_size: 1, + versions: 1, + delete_markers: 1, + replica_size: 1, + replica_count: 1, + ..Default::default() + }; + summary.repl_target_stats.insert( + target.clone(), + ReplTargetSizeSummary { + replicated_size: 1, + replicated_count: 1, + pending_size: 1, + failed_size: 1, + pending_count: 1, + failed_count: 1, + }, + ); + + apply_scanner_size_summary(&mut entry, &summary); + + assert_eq!(entry.size, usize::MAX); + assert_eq!(entry.versions, usize::MAX); + assert_eq!(entry.delete_markers, usize::MAX); + assert_eq!(entry.obj_sizes.to_map()["LESS_THAN_1024_B"], 1); + assert_eq!(entry.obj_versions.to_map()["SINGLE_VERSION"], 1); + + let replication_stats = entry + .replication_stats + .as_ref() + .expect("replication statistics should remain present"); + assert_eq!(replication_stats.replica_size, u64::MAX); + assert_eq!(replication_stats.replica_count, u64::MAX); + let target_stats = replication_stats + .targets + .get(&target) + .expect("replication target statistics should remain present"); + assert_eq!(target_stats.pending_size, u64::MAX); + assert_eq!(target_stats.failed_size, u64::MAX); + assert_eq!(target_stats.replicated_size, u64::MAX); + assert_eq!(target_stats.replicated_count, u64::MAX); + assert_eq!(target_stats.failed_count, u64::MAX); + assert_eq!(target_stats.pending_count, u64::MAX); + } + async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) { let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-test-{}", Uuid::new_v4())); tokio::fs::create_dir_all(&temp_dir) @@ -4145,7 +4244,7 @@ mod tests { scanner.update_cache.info.name = "bucket".to_string(); let parent = CancellationToken::new(); - let budget = ScannerCycleBudget::new( + let budget = ScannerCycleBudget::new_with_progress_tracking( &parent, crate::scanner_budget::ScannerCycleBudgetConfig { max_directories: Some(1), @@ -4168,6 +4267,7 @@ mod tests { assert!(budget.budget_elapsed()); assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories)); assert!(budget.token().is_cancelled()); + assert!(budget.entries_visited() >= 1); } #[tokio::test] @@ -4212,6 +4312,51 @@ mod tests { assert!(update.compacted, "partial update should preserve compacted state"); } + #[tokio::test] + #[serial] + async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() { + let (scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(temp_dir), + }; + let parent = CancellationToken::new(); + parent.cancel(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let cache = DataUsageCache { + info: crate::data_usage_define::DataUsageCacheInfo { + name: "bucket".to_string(), + ..Default::default() + }, + ..Default::default() + }; + let disk_path = scanner.local_disk.path().to_string_lossy().to_string(); + + let result = scan_data_folder( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + scanner.local_disk, + cache, + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + + assert!(matches!(result, Err(ScannerError::Other(message)) if message == "Operation cancelled")); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let report = global_metrics().report().await; + if report.active_paths.iter().all(|path| !path.starts_with(&disk_path)) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled scan should deregister its active path"); + } + #[tokio::test] #[serial] async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() { @@ -4262,6 +4407,7 @@ mod tests { assert!(partial_cache.info.last_update.is_some()); assert_eq!(partial_cache.info.next_cycle, 7); + assert!(!partial_cache.info.snapshot_complete); assert!(partial_cache.root().is_some(), "partial cache should keep completed scan progress"); assert!(budget.budget_elapsed()); assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories)); @@ -4386,6 +4532,104 @@ mod tests { assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories)); } + #[tokio::test] + #[serial] + async fn scan_data_folder_missing_bucket_returns_partial() { + let (scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(temp_dir), + }; + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let mut cache = DataUsageCache { + info: crate::data_usage_define::DataUsageCacheInfo { + name: "missing-bucket".to_string(), + next_cycle: 9, + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "missing-bucket", + crate::data_usage_define::DATA_USAGE_ROOT, + DataUsageEntry { + objects: 7, + size: 70, + ..Default::default() + }, + ); + + let result = scan_data_folder( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + scanner.local_disk, + cache, + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + + let partial = match result { + Err(ScannerError::NamespaceNotFoundCache(partial)) => partial, + other => panic!("missing bucket should keep the scan incomplete, got {other:?}"), + }; + assert!(!partial.info.snapshot_complete); + assert_eq!(partial.info.next_cycle, 9); + let root = partial + .checked_flatten("missing-bucket") + .expect("missing bucket partial must retain the last durable usage"); + assert_eq!(root.objects, 7); + assert_eq!(root.size, 70); + } + + #[tokio::test] + #[serial] + async fn scan_data_folder_missing_scan_root_returns_partial() { + let (scanner, temp_dir) = build_test_scanner().await; + tokio::fs::remove_dir_all(&temp_dir) + .await + .expect("failed to remove scanner root"); + let _guard = TestGuard { + temp_dir: Some(temp_dir), + }; + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let cache = DataUsageCache { + info: crate::data_usage_define::DataUsageCacheInfo { + name: "missing-bucket".to_string(), + next_cycle: 9, + ..Default::default() + }, + ..Default::default() + }; + + let result = scan_data_folder( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + scanner.local_disk, + cache, + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + + let partial = match result { + Err(ScannerError::NamespaceNotFoundCache(partial)) => partial, + other => panic!("missing scan root should keep the scan incomplete, got {other:?}"), + }; + assert!(!partial.info.snapshot_complete); + assert_eq!(partial.info.next_cycle, 9); + let root = partial + .checked_flatten("missing-bucket") + .expect("missing scan root partial should retain a non-publishable root"); + assert_eq!(root.objects, 0); + assert_eq!(root.size, 0); + } + #[tokio::test] #[serial] async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() { @@ -4533,6 +4777,7 @@ mod tests { .size_recursive("bucket") .expect("completed cache should retain bucket usage"); assert_eq!(root.objects, 5); + assert!(result.info.snapshot_complete); assert!(result.info.scan_resume_after.is_none()); assert!(result.info.scan_checkpoint.is_none()); } @@ -4626,6 +4871,53 @@ mod tests { assert_eq!(result.info.next_cycle, 11); } + #[tokio::test] + #[serial] + async fn test_scan_data_folder_keeps_unresolved_objects_partial() { + let (scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(temp_dir.clone()), + }; + write_test_object_metadata(&temp_dir, "bucket", "object").await; + + let failed_path = temp_dir + .join("bucket") + .join("object") + .join(STORAGE_FORMAT_FILE) + .to_string_lossy() + .into_owned(); + let mut cache = DataUsageCache { + info: crate::data_usage_define::DataUsageCacheInfo { + name: "bucket".to_string(), + next_cycle: 12, + ..Default::default() + }, + ..Default::default() + }; + cache.info.failed_objects.insert(failed_path, FolderScanner::now_secs()); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let result = scan_data_folder( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + scanner.local_disk.clone(), + cache, + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + + let partial = match result { + Err(ScannerError::PartialCache(partial)) => partial, + other => panic!("expected unresolved object to keep the cache partial, got {other:?}"), + }; + assert!(!partial.info.snapshot_complete); + assert!(!partial.info.failed_objects.is_empty()); + } + #[tokio::test] #[serial] #[cfg(unix)] diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 056a227f3..e2219301b 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -16,8 +16,9 @@ use crate::scanner_budget::ScannerCycleBudget; use crate::scanner_folder::{ScannerItem, scan_data_folder}; use crate::sleeper::SCANNER_SLEEPER; use crate::{ - DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo, - DataUsageInfo, ScannerError, ScannerObjectIO, SizeSummary, TierStats, + DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome, + DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, ScannerError, SizeSummary, + TierStats, }; use futures::future::join_all; use metrics::counter; @@ -26,11 +27,15 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics}; #[cfg(test)] use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS}; +use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo}; use rustfs_filemeta::FileMeta; use rustfs_utils::path::path_join_buf; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration}; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::path::Path; +use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{LazyLock, Mutex as StdMutex, MutexGuard}; use std::time::{Instant, SystemTime}; @@ -42,13 +47,13 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; -use crate::storage_api::scanner_io::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi}; +use crate::storage_api::scan::NamespaceLocking as _; +use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, - ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, + RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version, - get_lifecycle_config, get_object_lock_config, get_replication_config, list_runtime_tiers, - resolve_scanner_object_store_handle, storageclass, + get_lifecycle_config, get_object_lock_config, get_replication_config, list_runtime_tiers, storageclass, }; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; @@ -60,6 +65,12 @@ const EVENT_SCANNER_DISK_BUCKET_STATE: &str = "scanner_disk_bucket_state"; const EVENT_SCANNER_DATA_USAGE_STREAM: &str = "scanner_data_usage_stream"; const EVENT_SCANNER_CACHE_PERSIST_STATE: &str = "scanner_cache_persist_state"; const EVENT_SCANNER_SET_STATE: &str = "scanner_set_state"; +const SCANNER_CACHE_LOCK_SUFFIX: &str = ".scanner-cycle.lock"; +const SCANNER_CACHE_LOCK_POLL_INTERVAL: Duration = Duration::from_millis(250); +#[cfg(not(test))] +const SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] +const SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(50); const METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_set_scan_concurrency_limit"; const METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_disk_scan_concurrency_limit"; @@ -117,28 +128,58 @@ fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool { .is_some_and(|enabled| enabled.as_str() == ObjectLockEnabled::ENABLED) } -#[derive(Clone)] pub struct ScannerBucketScanPlan { buckets: Vec, + all_buckets: Arc>, + digest: DataUsageScanPlanDigest, + leader_epoch: u64, dirty_usage_buckets: Arc, - failed_dirty_buckets: Arc>>, + bucket_failures: ScannerBucketFailureState, pending_maintenance_work: Arc, + cache_cycle_floor: Arc, } -impl ScannerBucketScanPlan { - fn new( - buckets: Vec, - dirty_usage_buckets: Arc, - failed_dirty_buckets: Arc>>, - pending_maintenance_work: Arc, - ) -> Self { - Self { - buckets, - dirty_usage_buckets, - failed_dirty_buckets, - pending_maintenance_work, +#[derive(Clone, Default)] +struct ScannerBucketFailureState { + hard: Arc>>, + partial: Arc>>, + namespace_not_found: Arc>>, +} + +fn scanner_bucket_plan_digest(buckets: &[BucketInfo], activity_digest: [u8; 32]) -> DataUsageScanPlanDigest { + let mut buckets = buckets.iter().collect::>(); + buckets.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + + let mut hasher = Sha256::new(); + hasher.update(activity_digest); + hasher.update(u64::try_from(buckets.len()).unwrap_or(u64::MAX).to_be_bytes()); + for bucket in buckets { + let name = bucket.name.as_bytes(); + hasher.update(u64::try_from(name.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(name); + match bucket.created { + Some(created) => { + hasher.update([1]); + hasher.update(created.unix_timestamp_nanos().to_be_bytes()); + } + None => hasher.update([0]), } } + DataUsageScanPlanDigest(hasher.finalize().into()) +} + +fn scanner_bucket_cache_digest( + scan_plan_digest: DataUsageScanPlanDigest, + dirty_generation: Option, +) -> DataUsageScanPlanDigest { + let Some(dirty_generation) = dirty_generation else { + return scan_plan_digest; + }; + + let mut hasher = Sha256::new(); + hasher.update(scan_plan_digest.0); + hasher.update(dirty_generation.to_be_bytes()); + DataUsageScanPlanDigest(hasher.finalize().into()) } static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0); @@ -148,6 +189,20 @@ static SCANNER_ACTIVITY_EPOCH: LazyLock = LazyLock::new(|| format!("{:03 static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0); static SCANNER_MAINTENANCE_NOTIFY: LazyLock = LazyLock::new(Notify::new); +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScannerDirtyUsageState { + pub generation: u64, + pub pending: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ScannerDirtyUsageAckError { + #[error("scanner process instance changed before dirty usage acknowledgement")] + ProcessChanged, + #[error("scanner dirty usage generation cannot be acknowledged")] + InvalidGeneration, +} + fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } @@ -156,6 +211,12 @@ fn usize_to_u64_saturated(value: usize) -> u64 { u64::try_from(value).unwrap_or(u64::MAX) } +fn advance_generation(generation: &AtomicU64) -> u64 { + generation + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1))) + .map_or_else(|current| current, |previous| previous.saturating_add(1)) +} + pub fn record_dirty_usage_bucket(bucket: &str) { if bucket.is_empty() { return; @@ -163,7 +224,7 @@ pub fn record_dirty_usage_bucket(bucket: &str) { let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); - let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1; + let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); dirty_buckets.insert(bucket.to_string(), generation); dirty_buckets.len() }; @@ -176,7 +237,7 @@ pub fn record_scanner_maintenance_change(bucket: &str) { return; } - SCANNER_MAINTENANCE_GENERATION.fetch_add(1, Ordering::AcqRel); + advance_generation(&SCANNER_MAINTENANCE_GENERATION); SCANNER_MAINTENANCE_NOTIFY.notify_one(); record_dirty_usage_bucket(bucket); } @@ -193,6 +254,42 @@ pub fn scanner_activity_epoch() -> &'static str { SCANNER_ACTIVITY_EPOCH.as_str() } +pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState { + let dirty_buckets = dirty_usage_buckets(); + ScannerDirtyUsageState { + generation: DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire), + pending: !dirty_buckets.is_empty(), + } +} + +pub fn acknowledge_dirty_usage_generation( + instance_id: &str, + generation: u64, +) -> std::result::Result<(), ScannerDirtyUsageAckError> { + if instance_id != scanner_activity_epoch() { + return Err(ScannerDirtyUsageAckError::ProcessChanged); + } + + let (cleared_buckets, pending_buckets) = { + let mut dirty_buckets = dirty_usage_buckets(); + let current_generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire); + if generation == 0 || generation == u64::MAX || current_generation == u64::MAX || generation > current_generation { + return Err(ScannerDirtyUsageAckError::InvalidGeneration); + } + + let before = dirty_buckets.len(); + dirty_buckets.retain(|_, dirty_generation| *dirty_generation > generation); + let cleared_buckets = before.saturating_sub(dirty_buckets.len()); + if cleared_buckets > 0 { + advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); + } + (cleared_buckets, dirty_buckets.len()) + }; + global_metrics() + .record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets)); + Ok(()) +} + pub(crate) fn dirty_usage_generation() -> u64 { DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) } @@ -205,7 +302,7 @@ pub fn clear_dirty_usage_bucket(bucket: &str) { let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); dirty_buckets.remove(bucket); - DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel); + advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets)); @@ -259,7 +356,7 @@ fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) { } } if cleared_buckets > 0 { - DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel); + advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); } (cleared_buckets, dirty_buckets.len()) }; @@ -279,10 +376,11 @@ fn should_clear_dirty_usage_snapshot( result_ok: bool, completed_all_sets: bool, budget_elapsed: bool, + activity_and_generation_current: bool, dirty_buckets: &DirtyUsageBuckets, failed_buckets: &HashSet, ) -> Option { - if result_ok && completed_all_sets && !budget_elapsed { + if result_ok && completed_all_sets && !budget_elapsed && activity_and_generation_current { return Some(dirty_usage_buckets_excluding_failed(dirty_buckets, failed_buckets)); } @@ -293,8 +391,55 @@ async fn record_failed_dirty_bucket(failed_buckets: &Arc>> failed_buckets.lock().await.insert(bucket.to_string()); } -fn dirty_usage_snapshot_covers_current(snapshot: &DirtyUsageSnapshot) -> bool { - snapshot.covers_all_pending && DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire) == snapshot.generation +async fn record_partial_dirty_bucket(partial_buckets: &Arc>>, bucket: &str) { + partial_buckets.lock().await.insert(bucket.to_string()); +} + +async fn requeue_bucket_work( + bucket_tx: &mpsc::Sender, + bucket: &BucketInfo, + work_guard: &mut BucketWorkGuard, +) -> bool { + if bucket_tx.send(bucket.clone()).await.is_err() { + return false; + } + + work_guard.mark_requeued(); + true +} + +async fn mark_unprocessed_bucket_work_failed( + bucket_rx: &Mutex>, + remaining: &Arc, + complete: &CancellationToken, + failed_buckets: &Arc>>, +) -> usize { + let mut failed_count = 0; + let mut receiver = bucket_rx.lock().await; + while let Some(bucket) = receiver.recv().await { + record_failed_dirty_bucket(failed_buckets, &bucket.name).await; + drop(BucketWorkGuard::new(remaining.clone(), complete.clone())); + failed_count += 1; + } + failed_count +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DirtyUsageSnapshotStatus { + Current, + Changed, + Unverified, +} + +fn dirty_usage_snapshot_status(snapshot: &DirtyUsageSnapshot) -> DirtyUsageSnapshotStatus { + let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire); + if generation == u64::MAX { + DirtyUsageSnapshotStatus::Unverified + } else if snapshot.covers_all_pending && generation == snapshot.generation { + DirtyUsageSnapshotStatus::Current + } else { + DirtyUsageSnapshotStatus::Changed + } } #[cfg(test)] @@ -400,6 +545,34 @@ struct DiskBucketScanActiveGuard { set: String, } +struct BucketWorkGuard { + remaining: Arc, + complete: CancellationToken, + requeued: bool, +} + +impl BucketWorkGuard { + fn new(remaining: Arc, complete: CancellationToken) -> Self { + Self { + remaining, + complete, + requeued: false, + } + } + + fn mark_requeued(&mut self) { + self.requeued = true; + } +} + +impl Drop for BucketWorkGuard { + fn drop(&mut self) { + if !self.requeued && self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + self.complete.cancel(); + } + } +} + impl DiskBucketScanActiveGuard { fn new(active: Arc, pool: String, set: String) -> Self { let active_count = active.fetch_add(1, Ordering::Relaxed) + 1; @@ -461,6 +634,13 @@ fn decrement_atomic_usize(counter: &AtomicUsize) -> usize { .unwrap_or_else(|current| current) } +fn increment_atomic_usize(counter: &AtomicUsize) -> usize { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(1))) + .map(|previous| previous.saturating_add(1)) + .unwrap_or_else(|current| current) +} + fn record_disk_bucket_scans_queued(count: usize, pool: &str, set: &str) { metrics::gauge!( METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED, @@ -476,6 +656,11 @@ fn decrement_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &s record_disk_bucket_scans_queued(queued_count, pool, set); } +fn increment_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) { + let queued_count = increment_atomic_usize(counter); + record_disk_bucket_scans_queued(queued_count, pool, set); +} + fn reset_set_scan_gauges() { record_set_scan_concurrency_limit(0); record_set_scans_queued(0); @@ -513,6 +698,14 @@ fn scanner_max_concurrent_disk_scans(available: usize) -> usize { scanner_concurrency_limit(crate::runtime_config::scanner_max_concurrent_disk_scans_configured(), available) } +fn scanner_budgeted_concurrency_limit(configured_limit: usize, requires_serial_progress_accounting: bool) -> usize { + if requires_serial_progress_accounting { + 1 + } else { + configured_limit + } +} + fn record_set_scan_failure(first_err: &mut Option, err: Error) { if first_err.is_none() { *first_err = Some(err); @@ -535,17 +728,79 @@ fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option ScannerBucketScanStatus { + if has_failed { + ScannerBucketScanStatus::Failed + } else if has_partial { + ScannerBucketScanStatus::Partial + } else if has_namespace_not_found { + ScannerBucketScanStatus::NamespaceNotFound + } else { + ScannerBucketScanStatus::Complete + } +} + fn classify_nsscanner_cycle( completed_all_sets: bool, budget_elapsed: bool, cancelled: bool, - has_failed_buckets: bool, - dirty_usage_current: bool, + bucket_scan_status: ScannerBucketScanStatus, + dirty_usage_status: DirtyUsageSnapshotStatus, + activity_status: ScannerCycleActivityStatus, ) -> ScannerCycleStatus { - if completed_all_sets && !budget_elapsed && !cancelled && !has_failed_buckets && dirty_usage_current { - ScannerCycleStatus::Complete - } else { - ScannerCycleStatus::Incomplete + if budget_elapsed + || cancelled + || !matches!(bucket_scan_status, ScannerBucketScanStatus::Complete) + || dirty_usage_status == DirtyUsageSnapshotStatus::Unverified + { + return ScannerCycleStatus::Incomplete; + } + if !completed_all_sets { + return ScannerCycleStatus::Incomplete; + } + + match (activity_status, dirty_usage_status) { + (ScannerCycleActivityStatus::Unchanged, DirtyUsageSnapshotStatus::Current) => ScannerCycleStatus::Complete, + (ScannerCycleActivityStatus::Unverified, _) => ScannerCycleStatus::Incomplete, + _ => ScannerCycleStatus::Superseded, + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ScannerCycleActivityStatus { + Unchanged, + Changed, + Unverified, +} + +async fn scanner_cycle_activity_status( + store: &ECStore, + distributed: bool, + before: &crate::scanner::ScannerActivitySnapshot, +) -> ScannerCycleActivityStatus { + match crate::scanner::probe_scanner_activity(store, distributed).await { + Ok(after) if after == *before => ScannerCycleActivityStatus::Unchanged, + Ok(_) => ScannerCycleActivityStatus::Changed, + Err(err) => { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + state = "cycle_activity_probe_failed", + error = %err, + "Scanner cycle activity verification failed" + ); + ScannerCycleActivityStatus::Unverified + } } } @@ -570,14 +825,19 @@ fn is_xl_meta_path(path: &str) -> bool { .is_some_and(|name| name == STORAGE_FORMAT_FILE) } -fn cache_root_entry_info(cache: &DataUsageCache) -> DataUsageEntryInfo { - let entry = cache.root().map(|root| cache.flatten(&root)).unwrap_or_default(); +pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Result { + if cache.info.name.is_empty() { + return Err(ScannerError::Other("scanner cache root name is empty".to_string())); + } + let entry = cache + .checked_flatten(&cache.info.name) + .ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?; - DataUsageEntryInfo { + Ok(DataUsageEntryInfo { name: cache.info.name.clone(), parent: DATA_USAGE_ROOT.to_string(), entry, - } + }) } fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEntryInfo, update_time: SystemTime) { @@ -589,32 +849,192 @@ fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, total_count > 0 && completed_count == total_count && !budget_elapsed && !cancelled } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum NamespaceScannerWorkerMode { + Coordinator, + RemoteV3(uuid::Uuid), +} + +fn namespace_scanner_workers( + coordinator_disks: Vec, + remote_disks: Vec<(T, uuid::Uuid)>, +) -> Vec<(T, NamespaceScannerWorkerMode)> { + let mut workers = Vec::with_capacity(coordinator_disks.len() + remote_disks.len()); + workers.extend( + coordinator_disks + .into_iter() + .map(|disk| (disk, NamespaceScannerWorkerMode::Coordinator)), + ); + workers.extend( + remote_disks + .into_iter() + .map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV3(server_epoch))), + ); + workers +} + +fn group_remote_disks_by_peer(disks: Vec, peer_key: impl Fn(&T) -> String) -> Vec> { + let mut groups = HashMap::>::new(); + for disk in disks { + groups.entry(peer_key(&disk)).or_default().push(disk); + } + groups.into_values().collect() +} + +fn scanner_results_match_scan_scope(results: &[DataUsageCache], expected_sources: &HashSet) -> bool { + if results.is_empty() { + return false; + } + + let sources_match_topology = results + .iter() + .map(|result| result.info.source) + .collect::>>() + .is_some_and(|sources| sources.len() == results.len() && sources == *expected_sources); + let plan_digests = results + .iter() + .map(|result| result.info.scan_plan_digest) + .collect::>>(); + let cycles = results.iter().map(|result| result.info.next_cycle).collect::>(); + let leader_epochs = results.iter().map(|result| result.info.leader_epoch).collect::>(); + + sources_match_topology + && plan_digests.is_some_and(|digests| digests.len() == 1) + && cycles.len() == 1 + && leader_epochs.len() == 1 +} + +fn scanner_results_form_complete_snapshot(results: &[DataUsageCache], expected_sources: &HashSet) -> bool { + results + .iter() + .all(|result| result.info.last_update.is_some() && result.info.snapshot_complete) + && scanner_results_match_scan_scope(results, expected_sources) +} + +fn checked_bucket_usage_info(entry: &DataUsageEntry) -> Option { + let mut usage = BucketUsageInfo { + size: u64::try_from(entry.size).ok()?, + versions_count: u64::try_from(entry.versions).ok()?, + objects_count: u64::try_from(entry.objects).ok()?, + delete_markers_count: u64::try_from(entry.delete_markers).ok()?, + object_size_histogram: entry.obj_sizes.to_map(), + object_versions_histogram: entry.obj_versions.to_map(), + ..Default::default() + }; + + if let Some(replication) = &entry.replication_stats { + usage.replica_size = replication.replica_size; + usage.replica_count = replication.replica_count; + for (target, stats) in &replication.targets { + usage.replication_info.insert( + target.clone(), + BucketTargetUsageInfo { + replication_pending_size: stats.pending_size, + replicated_size: stats.replicated_size, + replication_failed_size: stats.failed_size, + replication_pending_count: stats.pending_count, + replication_failed_count: stats.failed_count, + replicated_count: stats.replicated_count, + ..Default::default() + }, + ); + } + } + Some(usage) +} + +pub(crate) fn scanner_cache_lock_resource(cache_name: &str) -> String { + path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, SCANNER_CACHE_LOCK_SUFFIX]) +} + +pub(crate) fn scanner_cache_lock_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5)) +} + +async fn await_scanner_disk_shutdown(scan: Pin<&mut F>) +where + F: Future, +{ + let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await; +} + +pub(crate) fn cache_snapshot_is_current( + cache: &DataUsageCache, + name: &str, + source: DataUsageCacheSource, + next_cycle: u64, + leader_epoch: u64, + scan_plan_digest: DataUsageScanPlanDigest, +) -> bool { + cache.info.name == name + && cache.info.source == Some(source) + && cache.info.snapshot_complete + && cache.info.scan_plan_digest == Some(scan_plan_digest) + && cache.info.last_update.is_some() + && cache.info.next_cycle == next_cycle + && cache.info.leader_epoch == leader_epoch +} + fn completed_data_usage_info( results: &[DataUsageCache], + expected_sources: &HashSet, all_buckets: &[String], + bucket_plan_complete: bool, budget_elapsed: bool, cancelled: bool, - dirty_usage_current: bool, ) -> Option<(DataUsageInfo, SystemTime)> { + if !bucket_plan_complete { + return None; + } let completed_set_count = results.iter().filter(|result| result.info.last_update.is_some()).count(); - if !should_publish_completed_snapshot(completed_set_count, results.len(), budget_elapsed, cancelled) || !dirty_usage_current { + if !should_publish_completed_snapshot(completed_set_count, results.len(), budget_elapsed, cancelled) { + return None; + } + if !scanner_results_form_complete_snapshot(results, expected_sources) { return None; } - let mut all_merged = DataUsageCache::default(); - for result in results.iter() { - all_merged.merge(result); + if results.iter().any(|result| result.root().is_none()) { + return None; } - let merged_last_update = all_merged.info.last_update.unwrap_or(SystemTime::UNIX_EPOCH); - all_merged.root()?; + let mut total = DataUsageEntry::default(); + let mut buckets_usage = HashMap::with_capacity(all_buckets.len()); + for bucket in all_buckets { + let mut merged = DataUsageEntry::default(); + for result in results { + let entry = result.checked_flatten(bucket)?; + if !merged.checked_merge(&entry) { + return None; + } + } + if !total.checked_merge(&merged) { + return None; + } + buckets_usage.insert(bucket.clone(), checked_bucket_usage_info(&merged)?); + } - Some((all_merged.dui(&all_merged.info.name, all_buckets), merged_last_update)) + let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?; + let data_usage_info = DataUsageInfo { + last_update: Some(merged_last_update), + scanner_cycle: Some(results.first()?.info.next_cycle), + objects_total_count: u64::try_from(total.objects).ok()?, + versions_total_count: u64::try_from(total.versions).ok()?, + delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, + objects_total_size: u64::try_from(total.size).ok()?, + buckets_count: u64::try_from(all_buckets.len()).ok()?, + buckets_usage, + ..Default::default() + }; + Some((data_usage_info, merged_last_update)) } #[cfg(test)] mod publish_gate_tests { use super::*; + use rustfs_data_usage::{ReplicationAllStats, ReplicationStats}; + + const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]); #[test] fn should_publish_completed_snapshot_requires_full_clean_cycle() { @@ -625,11 +1045,54 @@ mod publish_gate_tests { assert!(!should_publish_completed_snapshot(0, 0, false, false)); } - fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64) -> DataUsageCache { + fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache { + DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(TEST_PLAN_DIGEST), + ..Default::default() + }, + ..Default::default() + } + } + + #[test] + fn incomplete_scan_scope_requires_every_expected_set_marker() { + let first_source = DataUsageCacheSource::new(0, 0); + let second_source = DataUsageCacheSource::new(1, 0); + let expected_sources = HashSet::from([first_source, second_source]); + let first = incomplete_scope_cache(first_source); + let second = incomplete_scope_cache(second_source); + + assert!(scanner_results_match_scan_scope(&[first.clone(), second], &expected_sources)); + assert!(!scanner_results_form_complete_snapshot( + &[first.clone(), incomplete_scope_cache(second_source)], + &expected_sources + )); + assert!(!scanner_results_match_scan_scope( + &[first.clone(), DataUsageCache::default()], + &expected_sources + )); + assert!(!scanner_results_match_scan_scope( + &[first.clone(), incomplete_scope_cache(first_source)], + &expected_sources + )); + + let mut mismatched_plan = incomplete_scope_cache(second_source); + mismatched_plan.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32])); + assert!(!scanner_results_match_scan_scope(&[first, mismatched_plan], &expected_sources)); + } + + fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64, source: DataUsageCacheSource) -> DataUsageCache { let mut cache = DataUsageCache { info: DataUsageCacheInfo { name: DATA_USAGE_ROOT.to_string(), last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(update_secs)), + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(TEST_PLAN_DIGEST), ..Default::default() }, ..Default::default() @@ -639,66 +1102,609 @@ mod publish_gate_tests { DATA_USAGE_ROOT, DataUsageEntry { objects, - size: objects * 10, + size: objects.saturating_mul(10), ..Default::default() }, ); cache } + fn completed_data_usage_info_for_test( + results: &[DataUsageCache], + all_buckets: &[String], + budget_elapsed: bool, + cancelled: bool, + ) -> Option<(DataUsageInfo, SystemTime)> { + let expected_sources = results.iter().filter_map(|result| result.info.source).collect::>(); + completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled) + } + #[test] fn completed_data_usage_info_requires_every_set_before_publish() { let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()]; - let first_set = completed_root_cache("bucket-a", 1, 10); - let second_set = completed_root_cache("bucket-b", 2, 20); + let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default()); + let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0)); + second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default()); assert!( - completed_data_usage_info(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false, true) + completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false) .is_none() ); - assert!(completed_data_usage_info(&[first_set.clone(), second_set.clone()], &all_buckets, true, false, true).is_none()); - assert!(completed_data_usage_info(&[first_set.clone(), second_set.clone()], &all_buckets, false, true, true).is_none()); - assert!(completed_data_usage_info(&[first_set.clone(), second_set.clone()], &all_buckets, false, false, false).is_none()); + assert!( + completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, true, false).is_none() + ); + assert!( + completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, false, true).is_none() + ); let (data_usage_info, last_update) = - completed_data_usage_info(&[first_set, second_set], &all_buckets, false, false, true) + completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false) .expect("all completed sets should produce a publishable data usage snapshot"); assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20)); + assert_eq!(data_usage_info.scanner_cycle, Some(0)); assert_eq!(data_usage_info.objects_total_count, 3); assert_eq!(data_usage_info.buckets_usage.len(), 2); } + + #[test] + fn complete_usage_candidate_with_changed_generation_is_superseded() { + let all_buckets = vec!["bucket".to_string()]; + let completed_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let completed_usage = completed_data_usage_info_for_test(&[completed_set], &all_buckets, false, false); + + assert!(completed_usage.is_some()); + assert_eq!( + classify_nsscanner_cycle( + completed_usage.is_some(), + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Unchanged, + ), + ScannerCycleStatus::Superseded + ); + } + + #[test] + fn completed_data_usage_info_adds_same_bucket_across_unique_sets() { + let all_buckets = vec!["bucket".to_string()]; + let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + let second_entry = second_set.find("bucket").cloned().expect("second set bucket entry"); + second_set.replace( + "bucket", + DATA_USAGE_ROOT, + DataUsageEntry { + versions: 4, + delete_markers: 1, + ..second_entry + }, + ); + + let (data_usage_info, last_update) = + completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false) + .expect("unique completed set snapshots should be aggregated"); + let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage"); + + assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20)); + assert_eq!(data_usage_info.objects_total_count, 5); + assert_eq!(data_usage_info.objects_total_size, 50); + assert_eq!(bucket.objects_count, 5); + assert_eq!(bucket.size, 50); + assert_eq!(bucket.versions_count, 4); + assert_eq!(bucket.delete_markers_count, 1); + } + + #[test] + fn completed_data_usage_info_flattens_nested_bucket_entries() { + let all_buckets = vec!["bucket".to_string()]; + let mut first_set = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0)); + let mut nested = DataUsageEntry { + objects: 2, + versions: 3, + size: 2048, + replication_stats: Some(ReplicationAllStats { + targets: HashMap::from([( + "arn:target".to_string(), + ReplicationStats { + replicated_size: 2048, + replicated_count: 2, + ..Default::default() + }, + )]), + replica_size: 2048, + replica_count: 2, + }), + ..Default::default() + }; + nested.obj_sizes.add(2048); + nested.obj_versions.add(3); + first_set.replace("bucket/prefix", "bucket", nested); + let second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + + let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false) + .expect("nested bucket entries should be flattened before aggregation"); + let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage"); + + assert_eq!(data_usage_info.objects_total_count, 6); + assert_eq!(data_usage_info.objects_total_size, 2088); + assert_eq!(bucket.objects_count, 6); + assert_eq!(bucket.versions_count, 3); + assert_eq!(bucket.object_size_histogram["BETWEEN_1024_B_AND_64_KB"], 1); + assert_eq!(bucket.object_versions_histogram["BETWEEN_2_AND_10"], 1); + assert_eq!(bucket.replica_size, 2048); + assert_eq!(bucket.replica_count, 2); + assert_eq!(bucket.replication_info["arn:target"].replicated_size, 2048); + assert_eq!(bucket.replication_info["arn:target"].replicated_count, 2); + } + + #[test] + fn completed_data_usage_info_rejects_cyclic_bucket_entries() { + let all_buckets = vec!["bucket".to_string()]; + let mut cache = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0)); + cache.replace( + "bucket/prefix", + "bucket", + DataUsageEntry { + objects: 1, + size: 10, + ..Default::default() + }, + ); + cache + .cache + .get_mut(&crate::hash_path("bucket/prefix").key()) + .expect("nested entry") + .add_child(&crate::hash_path("bucket")); + + assert!(completed_data_usage_info_for_test(&[cache], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_duplicate_or_incomplete_set_sources() { + let all_buckets = vec!["bucket".to_string()]; + let source = DataUsageCacheSource::new(0, 0); + let first_set = completed_root_cache("bucket", 2, 10, source); + let duplicate_set = completed_root_cache("bucket", 3, 20, source); + + assert!(completed_data_usage_info_for_test(&[first_set.clone(), duplicate_set], &all_buckets, false, false).is_none()); + + let mut incomplete_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + incomplete_set.info.snapshot_complete = false; + assert!(completed_data_usage_info_for_test(&[first_set, incomplete_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_requires_exact_topology_sources() { + let all_buckets = vec!["bucket".to_string()]; + let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let unexpected_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(99, 99)); + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]); + + assert!( + completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, true, false, false) + .is_none() + ); + } + + #[test] + fn completed_data_usage_info_rejects_incomplete_bucket_plan() { + let all_buckets = vec!["bucket".to_string()]; + let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]); + + assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, false, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_missing_bucket_from_any_set() { + let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()]; + let mut complete_set = completed_root_cache("bucket-a", 2, 10, DataUsageCacheSource::new(0, 0)); + complete_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default()); + let missing_bucket_set = completed_root_cache("bucket-a", 3, 20, DataUsageCacheSource::new(1, 0)); + + assert!(completed_data_usage_info_for_test(&[complete_set, missing_bucket_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_mixed_scan_plans() { + let all_buckets = vec!["bucket".to_string()]; + let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + second_set.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32])); + + assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_mixed_scanner_cycles() { + let all_buckets = vec!["bucket".to_string()]; + let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + first_set.info.next_cycle = 12; + second_set.info.next_cycle = 13; + + assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_mixed_leader_epochs() { + let all_buckets = vec!["bucket".to_string()]; + let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); + let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0)); + first_set.info.leader_epoch = 11; + second_set.info.leader_epoch = 12; + + assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn completed_data_usage_info_rejects_counter_overflow() { + let all_buckets = vec!["bucket".to_string()]; + let first_set = completed_root_cache("bucket", usize::MAX, 10, DataUsageCacheSource::new(0, 0)); + let second_set = completed_root_cache("bucket", 1, 20, DataUsageCacheSource::new(1, 0)); + + assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none()); + } + + #[test] + fn current_cache_snapshot_requires_matching_complete_source_and_cycle() { + let source = DataUsageCacheSource::new(1, 2); + let mut cache = completed_root_cache("bucket", 1, 10, source); + cache.info.next_cycle = 10; + + assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 0, TEST_PLAN_DIGEST)); + assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST)); + assert!(!cache_snapshot_is_current( + &cache, + DATA_USAGE_ROOT, + DataUsageCacheSource::new(2, 1), + 10, + 0, + TEST_PLAN_DIGEST + )); + assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 11, 0, TEST_PLAN_DIGEST)); + assert!(!cache_snapshot_is_current( + &cache, + DATA_USAGE_ROOT, + source, + 10, + 0, + DataUsageScanPlanDigest([8; 32]) + )); + cache.info.leader_epoch = 2; + assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 1, TEST_PLAN_DIGEST)); + assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST)); + cache.info.next_cycle = 11; + assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST)); + } + + #[test] + fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() { + let server_epoch = uuid::Uuid::new_v4(); + let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v3", server_epoch)]); + + assert_eq!( + workers, + vec![ + ("local", NamespaceScannerWorkerMode::Coordinator), + ("legacy-remote", NamespaceScannerWorkerMode::Coordinator), + ("v3", NamespaceScannerWorkerMode::RemoteV3(server_epoch)), + ] + ); + assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty()); + } + + #[test] + fn remote_scanner_capability_probes_are_grouped_by_peer() { + let mut groups = group_remote_disks_by_peer(vec![("node-a", 0), ("node-b", 0), ("node-a", 1), ("node-b", 1)], |disk| { + disk.0.to_string() + }); + groups.sort_by_key(|group| group[0].0); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0], vec![("node-a", 0), ("node-a", 1)]); + assert_eq!(groups[1], vec![("node-b", 0), ("node-b", 1)]); + } + + #[test] + fn scanner_bucket_plan_digest_is_order_independent_and_membership_sensitive() { + let activity_digest = [4; 32]; + let buckets = vec![ + BucketInfo { + name: "photos".to_string(), + ..Default::default() + }, + BucketInfo { + name: "videos".to_string(), + ..Default::default() + }, + ]; + let reversed = vec![buckets[1].clone(), buckets[0].clone()]; + let changed = vec![ + buckets[0].clone(), + BucketInfo { + name: "archives".to_string(), + ..Default::default() + }, + ]; + let mut regenerated = buckets.clone(); + regenerated[0].created = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)); + + assert_eq!( + scanner_bucket_plan_digest(&buckets, activity_digest), + scanner_bucket_plan_digest(&reversed, activity_digest) + ); + assert_ne!( + scanner_bucket_plan_digest(&buckets, activity_digest), + scanner_bucket_plan_digest(&changed, activity_digest) + ); + assert_ne!( + scanner_bucket_plan_digest(&buckets, activity_digest), + scanner_bucket_plan_digest(®enerated, activity_digest) + ); + assert_ne!( + scanner_bucket_plan_digest(&buckets, activity_digest), + scanner_bucket_plan_digest(&buckets, [5; 32]) + ); + } + + #[test] + fn dirty_bucket_cache_digest_changes_with_generation() { + let source = DataUsageCacheSource::new(0, 0); + let plan = DataUsageScanPlanDigest([9; 32]); + let first = scanner_bucket_cache_digest(plan, Some(7)); + let second = scanner_bucket_cache_digest(plan, Some(8)); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "photos".to_string(), + next_cycle: 11, + last_update: Some(SystemTime::now()), + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(first), + ..Default::default() + }, + ..Default::default() + }; + cache.replace("photos", "", DataUsageEntry::default()); + + assert_eq!(scanner_bucket_cache_digest(plan, None), plan); + assert!(cache_snapshot_is_current(&cache, "photos", source, 11, 0, first)); + assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second)); + } + + #[test] + fn count_budget_serializes_set_and_disk_work() { + assert_eq!(scanner_budgeted_concurrency_limit(8, true), 1); + assert_eq!(scanner_budgeted_concurrency_limit(8, false), 8); + } + + #[test] + fn requeued_bucket_work_is_only_completed_after_retry() { + let remaining = Arc::new(AtomicUsize::new(1)); + let complete = CancellationToken::new(); + let mut first = BucketWorkGuard::new(remaining.clone(), complete.clone()); + first.mark_requeued(); + drop(first); + assert_eq!(remaining.load(Ordering::Acquire), 1); + assert!(!complete.is_cancelled()); + + drop(BucketWorkGuard::new(remaining.clone(), complete.clone())); + assert_eq!(remaining.load(Ordering::Acquire), 0); + assert!(complete.is_cancelled()); + } + + #[tokio::test] + async fn exhausted_workers_mark_all_queued_bucket_work_failed() { + let (tx, rx) = mpsc::channel(2); + tx.send(BucketInfo { + name: "photos".to_string(), + ..Default::default() + }) + .await + .expect("queue photos"); + tx.send(BucketInfo { + name: "videos".to_string(), + ..Default::default() + }) + .await + .expect("queue videos"); + drop(tx); + + let receiver = Mutex::new(rx); + let remaining = Arc::new(AtomicUsize::new(2)); + let complete = CancellationToken::new(); + let failed = Arc::new(Mutex::new(HashSet::new())); + + let failed_count = mark_unprocessed_bucket_work_failed(&receiver, &remaining, &complete, &failed).await; + + assert_eq!(failed_count, 2); + assert_eq!(remaining.load(Ordering::Acquire), 0); + assert!(complete.is_cancelled()); + assert_eq!(*failed.lock().await, HashSet::from(["photos".to_string(), "videos".to_string()])); + } + + #[tokio::test] + async fn requeued_bucket_remains_pending_for_another_worker() { + let (tx, mut rx) = mpsc::channel(1); + let remaining = Arc::new(AtomicUsize::new(1)); + let complete = CancellationToken::new(); + let mut guard = BucketWorkGuard::new(remaining.clone(), complete.clone()); + let bucket = BucketInfo { + name: "photos".to_string(), + ..Default::default() + }; + + assert!(requeue_bucket_work(&tx, &bucket, &mut guard).await); + drop(guard); + + assert_eq!(remaining.load(Ordering::Acquire), 1); + assert!(!complete.is_cancelled()); + assert_eq!(rx.recv().await.expect("requeued bucket").name, "photos"); + } } async fn send_cache_root_entry_info( - bucket_result_tx: &Arc>>, + bucket_result_tx: &mpsc::Sender, cache: &DataUsageCache, pending_maintenance_work: &AtomicBool, -) -> std::result::Result<(), mpsc::error::SendError> { +) -> std::result::Result<(), ScannerError> { + let root = cache_root_entry_info(cache)?; record_bucket_pending_maintenance_work(cache, pending_maintenance_work); - bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await + bucket_result_tx + .send(root) + .await + .map_err(|err| ScannerError::Other(format!("scanner cache root channel closed: {err}"))) } -async fn persist_and_publish_cache_snapshot( - store: Arc, +async fn persist_and_publish_cache_snapshot( + store: Arc, updates: &mpsc::Sender, - cache_snapshot: DataUsageCache, + mut cache_snapshot: DataUsageCache, + cache_cycle_floor: &AtomicU64, ) -> Option { - let last_update = cache_snapshot.info.last_update; + let source = cache_snapshot.info.source?; + let lock_resource = scanner_cache_lock_resource(DATA_USAGE_CACHE_NAME); + let ns_lock = match store.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await { + Ok(lock) => lock, + Err(err) => { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "lock_create_failed", + error = %err, + "Scanner cache snapshot lock creation failed" + ); + return None; + } + }; + let guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await { + Ok(guard) => guard, + Err(err) => { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "lock_acquire_failed", + error = %err, + "Scanner cache snapshot lock acquisition failed" + ); + return None; + } + }; - let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await { + let mut persisted = DataUsageCache::default(); + let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await { + Ok(revisions) => revisions, + Err(err) => { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "load_or_revision_lookup_failed", + error = %err, + "Scanner cache snapshot load or revision lookup failed" + ); + return None; + } + }; + let scan_plan_digest = cache_snapshot.info.scan_plan_digest?; + if persisted.info.next_cycle > cache_snapshot.info.next_cycle { + cache_cycle_floor.fetch_max(persisted.info.next_cycle, Ordering::AcqRel); + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + requested_cycle = cache_snapshot.info.next_cycle, + persisted_cycle = persisted.info.next_cycle, + state = "stale_cycle_rejected", + "Scanner rejected a set cache cycle regression" + ); + return None; + } + if persisted.info.leader_epoch > cache_snapshot.info.leader_epoch { error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_CACHE_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, cache_name = DATA_USAGE_CACHE_NAME, - state = "save_failed", - error = %e, - "Scanner cache snapshot persistence failed" + requested_epoch = cache_snapshot.info.leader_epoch, + persisted_epoch = persisted.info.leader_epoch, + state = "stale_leader_rejected", + "Scanner rejected a set cache snapshot from an older leader epoch" ); + return None; } - done_save(); + if cache_snapshot_is_current( + &persisted, + DATA_USAGE_ROOT, + source, + cache_snapshot.info.next_cycle, + cache_snapshot.info.leader_epoch, + scan_plan_digest, + ) { + cache_snapshot = persisted; + } else { + if guard.is_lock_lost() { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "lock_lost", + "Scanner cache snapshot save skipped after lock loss" + ); + return None; + } + + let done_save = Metrics::time(Metric::SaveUsage); + if let Err(e) = cache_snapshot + .save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions) + .await + { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "save_failed", + error = %e, + "Scanner cache snapshot persistence failed" + ); + done_save(); + return None; + } + done_save(); + } + if guard.is_lock_lost() { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "lock_lost_after_save", + "Scanner cache snapshot publish skipped after lock loss" + ); + return None; + } + drop(guard); + let last_update = cache_snapshot.info.last_update; if let Err(e) = updates.send(cache_snapshot).await { error!( @@ -716,18 +1722,19 @@ async fn persist_and_publish_cache_snapshot( last_update } -async fn send_merged_data_usage_update(updates: &mpsc::Sender, data_usage_info: DataUsageInfo) { - if let Err(e) = updates.send(data_usage_info).await { +async fn send_data_usage_update(updates: &mpsc::Sender, data_usage_info: DataUsageInfo) -> Result<()> { + updates.send(data_usage_info).await.map_err(|e| { error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DATA_USAGE_STREAM, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, - state = "send_merged_failed", + state = "send_failed", error = %e, - "Scanner merged data usage publish failed" + "Scanner data usage publish failed" ); - } + StorageError::other("scanner data usage receiver closed before update delivery") + }) } #[async_trait::async_trait] @@ -750,6 +1757,7 @@ pub(crate) trait ScannerIOCycle: Send + Sync + Debug + 'static { budget: Arc, updates: mpsc::Sender, want_cycle: u64, + leader_epoch: u64, scan_mode: HealScanMode, ) -> Result; } @@ -770,9 +1778,10 @@ pub trait ScannerIOCache: Send + Sync + Debug + 'static { #[async_trait::async_trait] pub trait ScannerIODisk: Send + Sync + Debug + 'static { async fn nsscanner_disk( - &self, + self: Arc, ctx: CancellationToken, budget: Arc, + set_disks: Vec>, cache: DataUsageCache, updates: Option>, scan_mode: HealScanMode, @@ -785,20 +1794,40 @@ pub trait ScannerIODisk: Send + Sync + Debug + 'static { pub enum ScannerDiskScanOutcome { Complete(DataUsageCache), Partial(DataUsageCache), + NamespaceNotFound(DataUsageCache), +} + +pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec> { + let membership = set.drive_membership_snapshot().await; + let capacity = membership + .online + .len() + .saturating_add(membership.suspect.len()) + .saturating_add(membership.returning.len()) + .saturating_add(membership.offline.len()); + let mut disks = Vec::with_capacity(capacity); + disks.extend(membership.online); + disks.extend(membership.suspect); + disks.extend(membership.returning); + disks.extend(membership.offline); + disks } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ScannerCycleStatus { Complete, Incomplete, + Superseded, } #[derive(Debug)] pub(crate) struct ScannerCycleResult { pub(crate) status: ScannerCycleStatus, dirty_usage_clear: Option, + remote_dirty_usage_acknowledgements: Vec, failed_dirty_usage: bool, pending_maintenance_work: bool, + required_cycle_floor: Option, } impl ScannerCycleResult { @@ -806,8 +1835,10 @@ impl ScannerCycleResult { Self { status, dirty_usage_clear, + remote_dirty_usage_acknowledgements: Vec::new(), failed_dirty_usage: false, pending_maintenance_work: false, + required_cycle_floor: None, } } @@ -821,14 +1852,29 @@ impl ScannerCycleResult { self } - pub(crate) fn acknowledge_durable_usage(self) { + fn with_required_cycle_floor(mut self, required_cycle_floor: Option) -> Self { + self.required_cycle_floor = required_cycle_floor; + self + } + + pub(crate) fn with_remote_dirty_usage_acknowledgements( + mut self, + acknowledgements: Vec, + ) -> Self { + self.remote_dirty_usage_acknowledgements = acknowledgements; + self + } + + pub(crate) fn acknowledge_durable_usage(self) -> Vec { if let Some(snapshot) = self.dirty_usage_clear { clear_dirty_usage_buckets(&snapshot); } + self.remote_dirty_usage_acknowledgements } pub(crate) fn has_dirty_usage_to_acknowledge(&self) -> bool { self.dirty_usage_clear.as_ref().is_some_and(|snapshot| !snapshot.is_empty()) + || !self.remote_dirty_usage_acknowledgements.is_empty() } pub(crate) fn has_failed_dirty_usage(&self) -> bool { @@ -838,6 +1884,10 @@ impl ScannerCycleResult { pub(crate) fn has_pending_maintenance_work(&self) -> bool { self.pending_maintenance_work } + + pub(crate) fn required_cycle_floor(&self) -> Option { + self.required_cycle_floor + } } #[async_trait::async_trait] @@ -850,12 +1900,13 @@ impl ScannerIO for ECStore { want_cycle: u64, scan_mode: HealScanMode, ) -> Result<()> { - // Preserve the public API's pre-existing completion semantics for - // embedders. The main scanner uses nsscanner_with_status so it can - // delay this acknowledgement until usage persistence succeeds. - ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, scan_mode) - .await? - .acknowledge_durable_usage(); + // This public path can prove delivery to the receiver, but not that + // the receiver persisted the update. Keep dirty usage pending unless + // the main scanner confirms durability through nsscanner_with_status. + let leader_epoch = crate::scanner::current_scanner_leader_epoch() + .await + .map_err(|err| StorageError::other(format!("failed to resolve scanner leader epoch: {err}")))?; + ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, leader_epoch, scan_mode).await?; Ok(()) } } @@ -869,45 +1920,98 @@ impl ScannerIOCycle for ECStore { budget: Arc, updates: mpsc::Sender, want_cycle: u64, + leader_epoch: u64, scan_mode: HealScanMode, ) -> Result { let child_token = ctx.child_token(); - let dirty_generation_before_bucket_list = dirty_usage_generation(); - let all_buckets = self.list_bucket(&BucketOptions::default()).await?; - let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); - - if all_buckets.is_empty() { - reset_set_scan_gauges(); - let empty_usage = DataUsageInfo { - last_update: Some(SystemTime::now()), - ..Default::default() - }; - if let Err(e) = updates.send(empty_usage).await { - error!( + let distributed = self.setup_is_dist_erasure().await; + let activity_before = match crate::scanner::probe_scanner_activity(self, distributed).await { + Ok(snapshot) => snapshot, + Err(err) => { + warn!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, - state = "empty_bucket_publish_failed", - error = %e, - "Scanner set state update failed" + state = "cycle_activity_baseline_failed", + error = %err, + "Scanner cycle skipped because cluster activity could not be baselined" ); + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); } - let status = if dirty_usage_snapshot_covers_current(&dirty_usage_snapshot) { - ScannerCycleStatus::Complete - } else { - ScannerCycleStatus::Incomplete + }; + if !crate::scanner::scanner_activity_allows_usage_publication(&activity_before) { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + state = "cycle_data_movement_active", + "Scanner cycle deferred while rebalance or decommission data movement is active" + ); + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + } + let dirty_generation_before_bucket_list = dirty_usage_generation(); + let bucket_listing = self.list_bucket_for_scanner(&BucketOptions::default()).await?; + let mut bucket_plan_complete = bucket_listing.topology_complete; + let all_buckets = Arc::new(bucket_listing.buckets); + let expected_sources = Arc::new( + self.pools + .iter() + .flat_map(|pool| { + pool.disk_set + .iter() + .map(|set| DataUsageCacheSource::new(set.pool_index, set.set_index)) + }) + .collect::>(), + ); + let mut buckets_by_source = HashMap::with_capacity(bucket_listing.set_buckets.len()); + for scope in bucket_listing.set_buckets { + let source = DataUsageCacheSource::new(scope.pool_index, scope.set_index); + if buckets_by_source.insert(source, scope.buckets).is_some() { + bucket_plan_complete = false; + } + } + bucket_plan_complete &= buckets_by_source.keys().copied().collect::>() == *expected_sources; + let scan_plan_digest = + scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before)); + let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); + let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle)); + + if all_buckets.is_empty() { + reset_set_scan_gauges(); + if !bucket_plan_complete { + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + } + let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await; + let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); + let status = classify_nsscanner_cycle( + true, + false, + ctx.is_cancelled(), + ScannerBucketScanStatus::Complete, + dirty_usage_status, + activity_status, + ); + if status != ScannerCycleStatus::Complete { + return Ok(ScannerCycleResult::new(status, None)); + } + let empty_usage = DataUsageInfo { + last_update: Some(SystemTime::now()), + scanner_cycle: Some(want_cycle), + ..Default::default() }; - let dirty_usage_clear = - (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); - return Ok(ScannerCycleResult::new(status, dirty_usage_clear)); + send_data_usage_update(&updates, empty_usage).await?; + let dirty_usage_clear = Some(dirty_usage_snapshot.buckets.as_ref().clone()); + return Ok( + ScannerCycleResult::new(status, dirty_usage_clear).with_remote_dirty_usage_acknowledgements( + crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before), + ), + ); } - let mut total_results = 0; - for pool in self.pools.iter() { - total_results += pool.disk_set.len(); - } + let total_results = expected_sources.len(); if total_results == 0 { warn!( target: "rustfs::scanner::io", @@ -922,8 +2026,11 @@ impl ScannerIOCycle for ECStore { return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); } - let set_scan_limit = scanner_max_concurrent_set_scans(total_results); - let failed_dirty_buckets = Arc::new(Mutex::new(HashSet::::new())); + let set_scan_limit = scanner_budgeted_concurrency_limit( + scanner_max_concurrent_set_scans(total_results), + budget.requires_serial_progress_accounting(), + ); + let bucket_failures = ScannerBucketFailureState::default(); let pending_maintenance_work = Arc::new(AtomicBool::new(false)); record_set_scan_concurrency_limit(set_scan_limit); debug!( @@ -954,6 +2061,8 @@ impl ScannerIOCycle for ECStore { results_index += 1; // Clone the Arc to move it into the spawned task let set_clone: Arc = Arc::clone(set); + let source = DataUsageCacheSource::new(set.pool_index, set.set_index); + let set_buckets = buckets_by_source.remove(&source).unwrap_or_default(); let pool_label = set.pool_index.to_string(); let set_label = set.set_index.to_string(); @@ -978,12 +2087,16 @@ impl ScannerIOCycle for ECStore { }); wait_futs.push(receiver_fut); - let scan_plan = ScannerBucketScanPlan::new( - all_buckets.clone(), - dirty_usage_snapshot.buckets.clone(), - failed_dirty_buckets.clone(), - pending_maintenance_work.clone(), - ); + let scan_plan = ScannerBucketScanPlan { + buckets: set_buckets, + all_buckets: Arc::clone(&all_buckets), + digest: scan_plan_digest, + leader_epoch, + dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), + bucket_failures: bucket_failures.clone(), + pending_maintenance_work: pending_maintenance_work.clone(), + cache_cycle_floor: cache_cycle_floor.clone(), + }; // Spawn task to run the scanner let scanner_fut = tokio::spawn(async move { let permit_wait = child_token_clone.clone(); @@ -1052,70 +2165,6 @@ impl ScannerIOCycle for ECStore { } } - let (update_tx, mut update_rx) = tokio::sync::oneshot::channel::<()>(); - - let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::>(); - let results_mutex_for_updates = results_mutex.clone(); - let budget_for_updates = budget.clone(); - let child_token_for_updates = child_token.clone(); - let dirty_usage_snapshot_for_updates = dirty_usage_snapshot.clone(); - tokio::spawn(async move { - let mut last_update = SystemTime::UNIX_EPOCH; - let mut has_sent_once = false; - - let mut ticker = tokio::time::interval(Duration::from_secs(30)); - loop { - tokio::select! { - _ = child_token_for_updates.cancelled() => { - break; - } - res = &mut update_rx => { - if res.is_err() { - break; - } - - let data_usage_update = { - let results = results_mutex_for_updates.lock().await; - completed_data_usage_info( - &results, - &all_buckets_clone, - budget_for_updates.budget_elapsed(), - child_token_for_updates.is_cancelled(), - dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()), - ) - }; - - if let Some((data_usage_info, merged_last_update)) = data_usage_update - && (!has_sent_once || merged_last_update > last_update) - { - send_merged_data_usage_update(&updates, data_usage_info).await; - } - break; - } - _ = ticker.tick() => { - let data_usage_update = { - let results = results_mutex_for_updates.lock().await; - completed_data_usage_info( - &results, - &all_buckets_clone, - budget_for_updates.budget_elapsed(), - child_token_for_updates.is_cancelled(), - dirty_usage_snapshot_covers_current(dirty_usage_snapshot_for_updates.as_ref()), - ) - }; - - if let Some((data_usage_info, merged_last_update)) = data_usage_update - && (!has_sent_once || merged_last_update > last_update) - { - send_merged_data_usage_update(&updates, data_usage_info).await; - has_sent_once = true; - last_update = merged_last_update; - } - } - } - } - }); - for join_result in join_all(wait_futs).await { if let Err(err) = join_result { error!( @@ -1135,34 +2184,68 @@ impl ScannerIOCycle for ECStore { record_set_scans_queued(0); record_set_scans_active(0); - let _ = update_tx.send(()); - let first_err = first_err_mutex.lock().await.take(); let results = results_mutex.lock().await.clone(); - let completed_all_sets = results.iter().all(|result| result.info.last_update.is_some()); + let completed_all_sets = bucket_plan_complete && scanner_results_form_complete_snapshot(&results, &expected_sources); let result = finalize_nsscanner_result(&results, first_err); - let failed_buckets = failed_dirty_buckets.lock().await.clone(); + let failed_buckets = bucket_failures.hard.lock().await.clone(); + let partial_buckets = bucket_failures.partial.lock().await.clone(); + let namespace_not_found_buckets = bucket_failures.namespace_not_found.lock().await.clone(); + let scan_scope_matches = scanner_results_match_scan_scope(&results, &expected_sources); + let bucket_scan_status = scanner_bucket_scan_status( + !failed_buckets.is_empty(), + scan_scope_matches && !partial_buckets.is_empty(), + scan_scope_matches && !namespace_not_found_buckets.is_empty(), + ); let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results); + let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire); + let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor); let budget_elapsed = budget.budget_elapsed(); - let dirty_usage_current = dirty_usage_snapshot_covers_current(&dirty_usage_snapshot); - let cycle_status = classify_nsscanner_cycle( - completed_all_sets, + let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); + let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current; + let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await; + let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::>(); + let completed_usage = completed_data_usage_info( + &results, + &expected_sources, + &all_bucket_names, + bucket_plan_complete, budget_elapsed, ctx.is_cancelled(), - !failed_buckets.is_empty(), - dirty_usage_current, ); + let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some(); + let cycle_status = classify_nsscanner_cycle( + structurally_complete_snapshot, + budget_elapsed, + ctx.is_cancelled(), + bucket_scan_status, + dirty_usage_status, + activity_status, + ); + if cycle_status == ScannerCycleStatus::Complete + && let Some((data_usage_info, _)) = completed_usage + { + send_data_usage_update(&updates, data_usage_info).await?; + } let dirty_usage_clear = should_clear_dirty_usage_snapshot( result.is_ok(), - completed_all_sets, + structurally_complete_snapshot, budget_elapsed, + activity_status == ScannerCycleActivityStatus::Unchanged && dirty_usage_current, &dirty_usage_snapshot.buckets, &failed_buckets, ); result?; + let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete { + crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before) + } else { + Vec::new() + }; Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) + .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) .with_failed_dirty_usage(!failed_buckets.is_empty()) - .with_pending_maintenance_work(pending_maintenance_work)) + .with_pending_maintenance_work(pending_maintenance_work) + .with_required_cycle_floor(required_cycle_floor)) } } @@ -1180,16 +2263,42 @@ impl ScannerIOCache for SetDisks { ) -> Result<()> { let ScannerBucketScanPlan { buckets, + all_buckets, + digest: scan_plan_digest, + leader_epoch, dirty_usage_buckets, - failed_dirty_buckets, + bucket_failures, pending_maintenance_work, + cache_cycle_floor, } = scan_plan; let pool_label = self.pool_index.to_string(); let set_label = self.set_index.to_string(); + let source = DataUsageCacheSource::new(self.pool_index, self.set_index); if buckets.is_empty() { + let now = SystemTime::now(); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: want_cycle, + last_update: Some(now), + leader_epoch, + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(scan_plan_digest), + ..Default::default() + }, + cache: HashMap::new(), + }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + for bucket in all_buckets.iter() { + cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + } reset_disk_bucket_scan_gauges(&pool_label, &set_label); - return Ok(()); + return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref()) + .await + .map(|_| ()) + .ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); } let (disks, healing) = self.get_online_disks_with_healing(false).await; @@ -1207,7 +2316,107 @@ impl ScannerIOCache for SetDisks { reset_disk_bucket_scan_gauges(&pool_label, &set_label); return Ok(()); } - let disk_scan_limit = scanner_max_concurrent_disk_scans(disks.len()); + // Preserve the original set topology across capability filtering. During + // rolling upgrades, an old remote peer must not make a distributed set + // look local and allow an unscoped legacy cache to be adopted. + let require_cache_source = disks.iter().any(|disk| !disk.is_local()); + let mut coordinator_disks = Vec::new(); + let mut remote_candidates = Vec::new(); + for disk in disks { + if disk.is_local() { + coordinator_disks.push(disk); + } else { + remote_candidates.push(disk); + } + } + let remote_groups = group_remote_disks_by_peer(remote_candidates, |disk| disk.host_name()); + let capability_results = join_all(remote_groups.into_iter().map(|disks| async move { + let server_epoch = match disks.first() { + Some(disk) => disk.ns_scanner_server_epoch().await, + None => Ok(None), + }; + (disks, server_epoch) + })) + .await; + let mut remote_disks = Vec::new(); + let mut unsupported_remote_disks = 0_usize; + for (disks, server_epoch) in capability_results { + match server_epoch { + Ok(Some(server_epoch)) => { + remote_disks.extend(disks.into_iter().map(|disk| (disk, server_epoch))); + } + Ok(None) => { + let disk_count = disks.len(); + let peer = disks.first().map(|disk| disk.host_name()).unwrap_or_default(); + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + peer = %peer, + disk_count, + state = "remote_scanner_unsupported", + "Scanner found a peer without remote namespace scanner support" + ); + unsupported_remote_disks = unsupported_remote_disks.saturating_add(disk_count); + coordinator_disks.extend(disks); + } + Err(err) => { + let peer = disks.first().map(|disk| disk.host_name()).unwrap_or_default(); + let disk_count = disks.len(); + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + peer = %peer, + disk_count, + state = "remote_scanner_probe_failed", + error = %err, + "Scanner skipped a peer whose remote namespace scanner capability could not be confirmed" + ); + } + } + } + let remote_disk_count = remote_disks.len(); + let workers = namespace_scanner_workers(coordinator_disks, remote_disks); + if workers.is_empty() { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + state = "no_compatible_disks", + "Scanner set state found no usable namespace scanner disks" + ); + reset_disk_bucket_scan_gauges(&pool_label, &set_label); + return Ok(()); + } + let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await); + if unsupported_remote_disks > 0 { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + v3_disks = remote_disk_count, + unsupported_remote_disks, + state = "unsupported_remote_disks_using_coordinator", + "Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers" + ); + } + let disk_scan_limit = scanner_budgeted_concurrency_limit( + scanner_max_concurrent_disk_scans(workers.len()), + budget.requires_serial_progress_accounting(), + ); record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit); debug!( target: "rustfs::scanner::io", @@ -1216,7 +2425,7 @@ impl ScannerIOCache for SetDisks { subsystem = LOG_SUBSYSTEM_IO, pool = self.pool_index, set = self.set_index, - online_disks = disks.len(), + online_disks = workers.len(), concurrency_limit = disk_scan_limit, state = "disk_concurrency_budget", "Scanner disk concurrency budget resolved" @@ -1243,15 +2452,66 @@ impl ScannerIOCache for SetDisks { "Scanner old data usage cache load failed; rebuilding from bucket caches" ); } + match old_cache.prepare_for_scan( + DATA_USAGE_ROOT, + want_cycle, + leader_epoch, + source, + scan_plan_digest, + require_cache_source, + ) { + DataUsageCachePrepareOutcome::RejectedNewerCycle => { + cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel); + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + requested_cycle = want_cycle, + cached_cycle = old_cache.info.next_cycle, + state = "stale_cycle_rejected", + "Scanner rejected a set cache cycle regression" + ); + return Ok(()); + } + DataUsageCachePrepareOutcome::RejectedNewerLeader => { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + requested_epoch = leader_epoch, + cached_epoch = old_cache.info.leader_epoch, + state = "stale_leader_rejected", + "Scanner rejected work from an older leader epoch" + ); + return Ok(()); + } + DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {} + } let mut cache = DataUsageCache { info: DataUsageCacheInfo { name: DATA_USAGE_ROOT.to_string(), - next_cycle: old_cache.info.next_cycle, + next_cycle: want_cycle, + leader_epoch, + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(scan_plan_digest), ..Default::default() }, cache: HashMap::new(), }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + for bucket in all_buckets.iter() { + cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + } let (bucket_tx, bucket_rx) = mpsc::channel::(buckets.len()); @@ -1265,7 +2525,7 @@ impl ScannerIOCache for SetDisks { } if let Err(e) = bucket_tx.send(bucket.clone()).await { - record_failed_dirty_bucket(&failed_dirty_buckets, &bucket.name).await; + record_failed_dirty_bucket(&bucket_failures.hard, &bucket.name).await; error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, @@ -1279,11 +2539,9 @@ impl ScannerIOCache for SetDisks { } } - drop(bucket_tx); - let cache_mutex: Arc> = Arc::new(Mutex::new(cache)); - let (bucket_result_tx, mut bucket_result_rx) = mpsc::channel::(disks.len()); + let (bucket_result_tx, mut bucket_result_rx) = mpsc::channel::(workers.len()); let cache_mutex_clone = cache_mutex.clone(); let ctx_clone = ctx.clone(); @@ -1313,31 +2571,49 @@ impl ScannerIOCache for SetDisks { let mut futs = Vec::new(); let bucket_rx_mutex: Arc>> = Arc::new(Mutex::new(bucket_rx)); - let bucket_result_tx_clone: Arc>> = Arc::new(Mutex::new(bucket_result_tx)); - for disk in disks.into_iter() { + let remaining_bucket_work = Arc::new(AtomicUsize::new(buckets.len())); + let bucket_work_complete = CancellationToken::new(); + for (disk, worker_mode) in workers { let bucket_rx_mutex_clone = bucket_rx_mutex.clone(); + let bucket_tx_clone = bucket_tx.clone(); + let remaining_bucket_work_clone = remaining_bucket_work.clone(); + let bucket_work_complete_clone = bucket_work_complete.clone(); let ctx_clone = ctx.clone(); let budget_clone = budget.clone(); let store_clone_clone = self.clone(); - let bucket_result_tx_clone_clone = bucket_result_tx_clone.clone(); + let bucket_result_tx_clone = bucket_result_tx.clone(); let disk_clone = disk.clone(); + let set_disk_inventory_clone = set_disk_inventory.clone(); let disk_scan_semaphore_clone = disk_scan_semaphore.clone(); let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone(); let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone(); let pool_label_clone = pool_label.clone(); let set_label_clone = set_label.clone(); - let failed_dirty_buckets_clone = failed_dirty_buckets.clone(); + let failed_dirty_buckets_clone = bucket_failures.hard.clone(); + let partial_dirty_buckets_clone = bucket_failures.partial.clone(); let pending_maintenance_work_clone = pending_maintenance_work.clone(); + let dirty_usage_buckets_clone = dirty_usage_buckets.clone(); + let cache_cycle_floor_clone = cache_cycle_floor.clone(); + let remote_server_epoch = match worker_mode { + NamespaceScannerWorkerMode::RemoteV3(server_epoch) => Some(server_epoch), + NamespaceScannerWorkerMode::Coordinator => None, + }; futs.push(tokio::spawn(async move { + let remote_session_id = uuid::Uuid::new_v4(); + let mut remote_session_sequence = 0_u64; loop { - let Some(bucket) = bucket_rx_mutex_clone.lock().await.recv().await else { - break; + let bucket = tokio::select! { + _ = bucket_work_complete_clone.cancelled() => break, + _ = ctx_clone.cancelled() => break, + bucket = async { bucket_rx_mutex_clone.lock().await.recv().await } => { + let Some(bucket) = bucket else { + break; + }; + bucket + } }; - - if ctx_clone.is_cancelled() { - decrement_disk_bucket_scans_queued(&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone); - break; - } + let mut work_guard = + BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone()); let permit_wait = ctx_clone.clone(); let permit_wait_start = Instant::now(); @@ -1386,35 +2662,327 @@ impl ScannerIOCache for SetDisks { ); let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]); + let bucket_scan_plan_digest = + scanner_bucket_cache_digest(scan_plan_digest, dirty_usage_buckets_clone.get(&bucket.name).copied()); - let mut cache = DataUsageCache::default(); - if let Err(e) = cache.load(store_clone_clone.clone(), &cache_name).await { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, + if let Some(server_epoch) = remote_server_epoch { + let request_sequence = remote_session_sequence; + let Some(next_sequence) = remote_session_sequence.checked_add(1) else { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "remote_session_sequence_exhausted", + "Remote scanner session sequence exhausted" + ); + break; + }; + remote_session_sequence = next_sequence; + let remote_outcome = crate::remote_scanner::scan_remote_bucket( + &disk_clone, + ctx_clone.clone(), + budget_clone.clone(), + crate::remote_scanner::RemoteScannerScanSpec { + bucket: &bucket.name, + next_cycle: want_cycle, + leader_epoch, + server_epoch, + session_id: remote_session_id, + session_sequence: request_sequence, + scan_plan_digest: bucket_scan_plan_digest, + skip_healing: healing, + scan_mode, + }, + ) + .await; + match remote_outcome { + Ok(crate::remote_scanner::RemoteScannerOutcome::Complete { + usage, + pending_maintenance_work, + }) => { + if pending_maintenance_work { + pending_maintenance_work_clone.store(true, Ordering::Release); + } + if let Err(e) = bucket_result_tx_clone.send(*usage).await { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DATA_USAGE_STREAM, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "send_remote_root_failed", + error = %e, + "Remote scanner root entry publish failed" + ); + } + } + Ok(crate::remote_scanner::RemoteScannerOutcome::Partial) => { + record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await; + } + Ok(crate::remote_scanner::RemoteScannerOutcome::NamespaceNotFound) => { + if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await { + increment_disk_bucket_scans_queued( + &queued_disk_bucket_scans_clone, + &pool_label_clone, + &set_label_clone, + ); + } else { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + } + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "remote_namespace_missing_requeued", + "Remote scanner requeued a bucket missing from this disk" + ); + break; + } + Ok(crate::remote_scanner::RemoteScannerOutcome::CycleAhead(required_cycle)) => { + cache_cycle_floor_clone.fetch_max(required_cycle, Ordering::AcqRel); + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + requested_cycle = want_cycle, + cached_cycle = required_cycle, + state = "remote_stale_cycle_rejected", + "Remote scanner rejected a bucket cache cycle regression" + ); + } + Err(e) => { + if e.retry_bucket_work() { + if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await { + increment_disk_bucket_scans_queued( + &queued_disk_bucket_scans_clone, + &pool_label_clone, + &set_label_clone, + ); + } else { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + } + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "remote_zero_progress_requeued", + error = %e, + "Remote scanner requeued bucket rejected before execution" + ); + break; + } + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + if ctx_clone.is_cancelled() { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "remote_cancelled", + error = %e, + "Remote scanner bucket scan cancelled" + ); + } else { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "remote_scan_failed", + error = %e, + "Remote scanner bucket scan failed" + ); + } + if e.retire_worker() { + break; + } + } + } + continue; + } + + let _local_admission = if disk_clone.is_local() { + match crate::remote_scanner::try_admit_remote_scanner(&disk_clone) { + Ok(admission) => Some(admission), + Err(e) => { + if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await { + increment_disk_bucket_scans_queued( + &queued_disk_bucket_scans_clone, + &pool_label_clone, + &set_label_clone, + ); + } else { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + } + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "local_disk_busy", + error = %e, + "Scanner local disk is already serving namespace scanner work" + ); + break; + } + } + } else { + None + }; + + // Lock order: scanner leader fence -> per-bucket cache lock -> + // cache object read/write. The cache lock stays outermost for + // local and rolling-upgrade workers so leader failover cannot + // execute the same bucket concurrently. + let lock_resource = scanner_cache_lock_resource(&cache_name); + let ns_lock = match store_clone_clone.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await { + Ok(lock) => lock, + Err(e) => { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, bucket = %bucket.name, cache_name = %cache_name, - state = "cache_load_failed", + state = "lock_create_failed", error = %e, - "Scanner disk bucket cache load failed" - ); + "Scanner bucket cache lock creation failed" + ); + continue; + } + }; + let cache_guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await { + Ok(guard) => guard, + Err(e) => { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "lock_acquire_failed", + error = %e, + "Scanner bucket cache lock acquisition failed" + ); + continue; + } + }; + + let mut cache = DataUsageCache::default(); + let revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await { + Ok(revisions) => revisions, + Err(e) => { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "load_or_revision_lookup_failed", + error = %e, + "Scanner bucket cache load or revision lookup failed" + ); + continue; + } + }; + if cache_snapshot_is_current(&cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest) + { + if cache_guard.is_lock_lost() { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "lock_lost_before_reuse", + "Current scanner bucket cache root publish skipped after lock loss" + ); + continue; + } + if let Err(e) = + send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await + { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DATA_USAGE_STREAM, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "send_current_root_failed", + error = %e, + "Current scanner bucket cache root entry publish failed" + ); + } + continue; } - if cache.info.name.is_empty() { - cache.info.name = bucket.name.clone(); + match cache.prepare_for_scan( + &bucket.name, + want_cycle, + leader_epoch, + source, + bucket_scan_plan_digest, + require_cache_source, + ) { + DataUsageCachePrepareOutcome::RejectedNewerCycle => { + cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel); + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + requested_cycle = want_cycle, + cached_cycle = cache.info.next_cycle, + state = "stale_cycle_rejected", + "Scanner rejected a bucket cache cycle regression" + ); + continue; + } + DataUsageCachePrepareOutcome::RejectedNewerLeader => { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + requested_epoch = leader_epoch, + cached_epoch = cache.info.leader_epoch, + state = "stale_leader_rejected", + "Scanner rejected bucket work from an older leader epoch" + ); + continue; + } + DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {} } - cache.info.skip_healing = healing; - cache.info.next_cycle = want_cycle; - if cache.info.name != bucket.name { - cache.info = DataUsageCacheInfo { - name: bucket.name.clone(), - next_cycle: want_cycle, - ..Default::default() - }; - } debug!( target: "rustfs::scanner::io", @@ -1429,10 +2997,31 @@ impl ScannerIOCache for SetDisks { let before = cache.info.last_update; - let scan_outcome = match disk_clone - .nsscanner_disk(ctx_clone.clone(), budget_clone.clone(), cache.clone(), None, scan_mode) - .await - { + let scan_ctx = ctx_clone.child_token(); + let scan = disk_clone.clone().nsscanner_disk( + scan_ctx.clone(), + budget_clone.clone(), + set_disk_inventory_clone.as_ref().clone(), + cache.clone(), + None, + scan_mode, + ); + tokio::pin!(scan); + let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL); + lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let scan_result = loop { + tokio::select! { + result = &mut scan => break result, + _ = lock_watch.tick() => { + if cache_guard.is_lock_lost() { + scan_ctx.cancel(); + await_scanner_disk_shutdown(scan.as_mut()).await; + break Err(Error::other("scanner bucket cache lock was lost during bucket scan")); + } + } + } + }; + let scan_outcome = match scan_result { Ok(scan_outcome) => scan_outcome, Err(e) => { record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; @@ -1460,11 +3049,15 @@ impl ScannerIOCache for SetDisks { ); } - if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before) + if !cache_guard.is_lock_lost() + && let (Some(last_update), Some(before_update)) = (cache.info.last_update, before) && last_update > before_update { let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await { + if let Err(e) = cache + .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .await + { error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_CACHE_PERSIST_STATE, @@ -1484,12 +3077,44 @@ impl ScannerIOCache for SetDisks { } }; - cache = match scan_outcome { - ScannerDiskScanOutcome::Complete(cache) => cache, - ScannerDiskScanOutcome::Partial(cache) => { - record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; - let done_save = Metrics::time(Metric::SaveUsage); - let partial_saved = match cache.save(store_clone_clone.clone(), cache_name.as_str()).await { + let partial = match scan_outcome { + ScannerDiskScanOutcome::Complete(completed_cache) => { + cache = completed_cache; + None + } + ScannerDiskScanOutcome::Partial(partial_cache) => Some((partial_cache, &partial_dirty_buckets_clone)), + ScannerDiskScanOutcome::NamespaceNotFound(_) => { + if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await { + increment_disk_bucket_scans_queued( + &queued_disk_bucket_scans_clone, + &pool_label_clone, + &set_label_clone, + ); + } else { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + } + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_DISK_BUCKET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + state = "local_namespace_missing_requeued", + "Scanner requeued a bucket missing from this local disk" + ); + break; + } + }; + if let Some((partial_cache, failure_buckets)) = partial { + record_partial_dirty_bucket(failure_buckets, &bucket.name).await; + let done_save = Metrics::time(Metric::SaveUsage); + let partial_saved = if cache_guard.is_lock_lost() { + false + } else { + match partial_cache + .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .await + { Ok(()) => true, Err(e) => { error!( @@ -1505,24 +3130,27 @@ impl ScannerIOCache for SetDisks { ); false } - }; - done_save(); - if partial_saved { - debug!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket.name, - cache_name = %cache_name, - state = "partial_saved_not_published", - "Scanner partial bucket cache saved without publishing usage aggregate" - ); } - - continue; + }; + done_save(); + if !partial_saved { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; } - }; + if partial_saved { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "partial_saved_not_published", + "Scanner partial bucket cache saved without publishing usage aggregate" + ); + } + + continue; + } debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DISK_BUCKET_STATE, @@ -1538,6 +3166,58 @@ impl ScannerIOCache for SetDisks { break; } + if cache_guard.is_lock_lost() { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "lock_lost", + "Scanner bucket cache save skipped after lock loss" + ); + continue; + } + + let done_save = Metrics::time(Metric::SaveUsage); + if let Err(e) = cache + .save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions) + .await + { + done_save(); + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "save_failed", + error = %e, + "Scanner bucket cache save failed" + ); + continue; + } + done_save(); + + if cache_guard.is_lock_lost() { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "lock_lost_after_save", + "Scanner bucket cache root publish skipped after lock loss" + ); + continue; + } + debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DATA_USAGE_STREAM, @@ -1550,7 +3230,7 @@ impl ScannerIOCache for SetDisks { ); if let Err(e) = - send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache, &pending_maintenance_work_clone).await + send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await { record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; error!( @@ -1564,26 +3244,11 @@ impl ScannerIOCache for SetDisks { "Scanner root entry publish failed" ); } - - let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await { - record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket.name, - cache_name = %cache_name, - state = "save_failed", - error = %e, - "Scanner bucket cache save failed" - ); - } - done_save(); } })); } + drop(bucket_tx); + drop(bucket_result_tx); let mut first_join_err = None; for join_result in join_all(futs).await { @@ -1602,12 +3267,30 @@ impl ScannerIOCache for SetDisks { record_set_scan_failure(&mut first_join_err, scanner_task_join_error("scanner disk bucket", err)); } } + let unprocessed_buckets = mark_unprocessed_bucket_work_failed( + bucket_rx_mutex.as_ref(), + &remaining_bucket_work, + &bucket_work_complete, + &bucket_failures.hard, + ) + .await; + if unprocessed_buckets > 0 { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_SET_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + unprocessed_buckets, + state = "workers_exhausted", + "Scanner marked queued bucket work failed after all disk workers exited" + ); + } record_disk_scan_concurrency_limit(&pool_label, &set_label, 0); record_disk_bucket_scans_queued(0, &pool_label, &set_label); record_disk_bucket_scans_active(0, &pool_label, &set_label); - drop(bucket_result_tx_clone); - if let Err(err) = collect_bucket_results_fut.await { return Err(scanner_task_join_error("scanner bucket result collector", err)); } @@ -1622,10 +3305,36 @@ impl ScannerIOCache for SetDisks { let mut cache = cache_mutex.lock().await; cache.info.next_cycle = want_cycle; cache.info.last_update.get_or_insert_with(SystemTime::now); + cache.info.snapshot_complete = true; cache.clone() }; - let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot).await; + let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await; } else { + let incomplete_scope = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: want_cycle, + leader_epoch, + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(scan_plan_digest), + ..Default::default() + }, + cache: HashMap::new(), + }; + if let Err(e) = updates.send(incomplete_scope).await { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + state = "incomplete_scope_publish_failed", + error = %e, + "Scanner incomplete set scope publish failed" + ); + } debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_CACHE_PERSIST_STATE, @@ -1758,9 +3467,10 @@ impl ScannerIODisk for Disk { #[tracing::instrument(skip(self, budget, updates, cache))] async fn nsscanner_disk( - &self, + self: Arc, ctx: CancellationToken, budget: Arc, + set_disks: Vec>, cache: DataUsageCache, updates: Option>, scan_mode: HealScanMode, @@ -1803,68 +3513,17 @@ impl ScannerIODisk for Disk { cache.info.object_lock = Some(Arc::new(object_lock_config)); } - let Some(ecstore) = resolve_scanner_object_store_handle() else { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket, - state = "ecstore_unavailable", - "Scanner disk bucket missing object layer" - ); - return Err(StorageError::other("ECStore not available".to_string())); - }; - - let disk_location = self.get_disk_location(); - - let (Some(pool_idx), Some(set_idx)) = (disk_location.pool_idx, disk_location.set_idx) else { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket, - state = "disk_location_unavailable", - "Scanner disk bucket missing disk location" - ); - return Err(StorageError::other("Disk location not available".to_string())); - }; - - let disks_result = StorageAdminApi::disk_set_inventory(ecstore.as_ref(), DiskSetSelector::new(pool_idx, set_idx)).await?; - - let Some(disk_idx) = disk_location.disk_idx else { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket, - state = "disk_index_unavailable", - "Scanner disk bucket missing disk index" - ); - return Err(StorageError::other("Disk index not available".to_string())); - }; - - let local_disk = if let Some(Some(local_disk)) = disks_result.get(disk_idx) { - local_disk.clone() - } else { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket, - state = "local_disk_unavailable", - "Scanner disk bucket missing local disk" - ); - return Err(StorageError::other("Local disk not available".to_string())); - }; - - let disks = disks_result.into_iter().flatten().collect::>>(); - - let result = - scan_data_folder(ctx.clone(), budget, disks, local_disk, cache, updates, scan_mode, SCANNER_SLEEPER.clone()).await; + let result = scan_data_folder( + ctx.clone(), + budget, + set_disks, + self.clone(), + cache, + updates, + scan_mode, + SCANNER_SLEEPER.clone(), + ) + .await; match result { Ok(mut data_usage_info) => { @@ -1881,6 +3540,13 @@ impl ScannerIODisk for Disk { failure_guard.mark_not_failed(); Ok(ScannerDiskScanOutcome::Partial(*partial_cache)) } + Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => { + done_drive(); + emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed()); + partial_cache.info.last_update.get_or_insert_with(SystemTime::now); + failure_guard.mark_not_failed(); + Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache)) + } Err(e) => { if ctx.is_cancelled() { emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed()); @@ -1898,8 +3564,14 @@ impl ScannerIODisk for Disk { #[cfg(test)] mod tests { use super::*; + use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_folder::ScannerItem; - use crate::{DiskOption, Endpoint, new_disk, path2_bucket_object_with_base_path}; + use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _}; + use crate::{ + DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions, + ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, + init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path, + }; use rustfs_filemeta::FileInfo; use serial_test::serial; use temp_env::with_var; @@ -1916,6 +3588,150 @@ mod tests { } } + async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc) { + init_ecstore_config_for_scanner_tests(); + let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created"); + let mut pools = Vec::new(); + for pool_index in 0..2 { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("multi-pool scanner test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pools.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: format!("scanner-cycle-pool-{pool_index}"), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }); + } + + let endpoint_pools = EndpointServerPools::from(pools); + let instance_ctx = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("multi-pool local disks should initialize"); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + CancellationToken::new(), + instance_ctx, + ) + .await + .expect("multi-pool ECStore should initialize"); + init_bucket_metadata_sys_for_scanner_tests(store.clone()).await; + + (temp_dir, store) + } + + #[tokio::test] + async fn data_usage_publish_fails_when_receiver_is_closed() { + let (updates, receiver) = mpsc::channel(1); + drop(receiver); + + let err = send_data_usage_update(&updates, DataUsageInfo::default()) + .await + .expect_err("closed usage receiver must reject the scanner update"); + + assert!(err.to_string().contains("receiver closed")); + } + + #[tokio::test] + #[serial] + async fn multi_pool_scanner_cycle_publishes_combined_usage() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let bucket = format!("scanner-union-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created across both pools"); + + for (pool_index, (object, body)) in [("pool-a", b"first".as_slice()), ("pool-b", b"second".as_slice())] + .into_iter() + .enumerate() + { + let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); + store.pools[pool_index].disk_set[0] + .put_object(&bucket, object, &mut reader, &ScannerObjectOptions::default()) + .await + .expect("object should be written to its selected pool"); + } + + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); + let (updates, mut receiver) = mpsc::channel(1); + let result = tokio::time::timeout( + Duration::from_secs(30), + ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal), + ) + .await + .expect("multi-pool scanner cycle should finish") + .expect("multi-pool scanner cycle should succeed"); + + assert_eq!(result.status, ScannerCycleStatus::Complete); + let usage = receiver.recv().await.expect("complete scanner cycle should publish usage"); + let bucket_usage = usage + .buckets_usage + .get(&bucket) + .expect("combined bucket usage should be present"); + assert_eq!(bucket_usage.objects_count, 2); + assert_eq!(bucket_usage.size, 11); + assert_eq!(usage.objects_total_count, 2); + assert_eq!(usage.objects_total_size, 11); + } + + #[tokio::test] + #[serial] + async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple()); + store.pools[1].disk_set[0] + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created only in the second pool"); + let body = b"second-only"; + let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); + store.pools[1].disk_set[0] + .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default()) + .await + .expect("object should be written only to the second pool"); + + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); + let (updates, mut receiver) = mpsc::channel(1); + let result = tokio::time::timeout( + Duration::from_secs(30), + ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal), + ) + .await + .expect("second-pool-only scanner cycle should finish") + .expect("second-pool-only scanner cycle should succeed"); + + assert_eq!(result.status, ScannerCycleStatus::Complete); + let usage = receiver.recv().await.expect("complete scanner cycle should publish usage"); + let bucket_usage = usage + .buckets_usage + .get(&bucket) + .expect("second-pool-only bucket usage should be present"); + assert_eq!(bucket_usage.objects_count, 1); + assert_eq!(bucket_usage.size, u64::try_from(body.len()).expect("test body length should fit u64")); + assert_eq!(usage.objects_total_count, 1); + assert_eq!( + usage.objects_total_size, + u64::try_from(body.len()).expect("test body length should fit u64") + ); + } + #[tokio::test] async fn scanner_item_object_lock_uses_cached_config() { let temp_dir = std::env::temp_dir(); @@ -1972,6 +3788,62 @@ mod tests { clear_dirty_usage_buckets_for_tests(); } + #[test] + #[serial] + fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket("photos"); + let acknowledged_generation = scanner_dirty_usage_state().generation; + record_dirty_usage_bucket("videos"); + + acknowledge_dirty_usage_generation(scanner_activity_epoch(), acknowledged_generation) + .expect("a matching process and prior generation should be acknowledged"); + acknowledge_dirty_usage_generation(scanner_activity_epoch(), acknowledged_generation) + .expect("replaying an acknowledged generation should be idempotent"); + + let pending = dirty_usage_buckets_for_tests(); + assert!(!pending.contains_key("photos")); + assert!(pending.contains_key("videos")); + assert!(scanner_dirty_usage_state().pending); + drop(pending); + + let remaining_generation = scanner_dirty_usage_state().generation; + acknowledge_dirty_usage_generation(scanner_activity_epoch(), remaining_generation) + .expect("the remaining generation should be acknowledged"); + assert!(!scanner_dirty_usage_state().pending); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket("photos"); + let generation = scanner_dirty_usage_state().generation; + + assert_eq!( + acknowledge_dirty_usage_generation("stale-process", generation), + Err(ScannerDirtyUsageAckError::ProcessChanged) + ); + assert_eq!( + acknowledge_dirty_usage_generation(scanner_activity_epoch(), 0), + Err(ScannerDirtyUsageAckError::InvalidGeneration) + ); + assert_eq!( + acknowledge_dirty_usage_generation(scanner_activity_epoch(), u64::MAX), + Err(ScannerDirtyUsageAckError::InvalidGeneration) + ); + assert_eq!( + acknowledge_dirty_usage_generation( + scanner_activity_epoch(), + generation.checked_add(1).expect("test generation should not be exhausted") + ), + Err(ScannerDirtyUsageAckError::InvalidGeneration) + ); + assert!(dirty_usage_buckets_for_tests().contains_key("photos")); + clear_dirty_usage_buckets_for_tests(); + } + #[test] #[serial] fn dirty_usage_snapshot_detects_uncovered_generation() { @@ -1980,14 +3852,23 @@ mod tests { let buckets = vec![bucket_info("photos")]; let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation()); - assert!(dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current); record_dirty_usage_bucket("photos"); - assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed); clear_dirty_usage_buckets_for_tests(); } + #[test] + fn generation_saturates_instead_of_wrapping() { + let generation = AtomicU64::new(u64::MAX - 1); + + assert_eq!(advance_generation(&generation), u64::MAX); + assert_eq!(advance_generation(&generation), u64::MAX); + assert_eq!(generation.load(Ordering::Acquire), u64::MAX); + } + #[test] #[serial] fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() { @@ -2001,10 +3882,11 @@ mod tests { assert!(snapshot.buckets.contains_key("photos")); assert!(snapshot.buckets.contains_key("temporarily-omitted")); assert!(dirty_usage_buckets().contains_key("temporarily-omitted")); - assert!(dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current); - ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())) + let acknowledgements = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())) .acknowledge_durable_usage(); + assert!(acknowledgements.is_empty()); assert!(!dirty_usage_buckets().contains_key("temporarily-omitted")); clear_dirty_usage_buckets_for_tests(); } @@ -2019,7 +3901,7 @@ mod tests { let snapshot = snapshot_dirty_usage_buckets(&[], generation_before_bucket_list); assert!(!snapshot.buckets.contains_key("new-or-racing-bucket")); - assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed); assert!(dirty_usage_buckets().contains_key("new-or-racing-bucket")); clear_dirty_usage_buckets_for_tests(); } @@ -2029,11 +3911,11 @@ mod tests { fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() { clear_dirty_usage_buckets_for_tests(); let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); - assert!(dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current); record_dirty_usage_bucket("photos"); - assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed); assert!(dirty_usage_buckets().contains_key("photos")); clear_dirty_usage_buckets_for_tests(); } @@ -2047,7 +3929,7 @@ mod tests { record_dirty_usage_bucket("photos"); let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list); - assert!(!dirty_usage_snapshot_covers_current(&snapshot)); + assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed); assert!(dirty_usage_buckets().contains_key("photos")); clear_dirty_usage_buckets_for_tests(); } @@ -2090,7 +3972,7 @@ mod tests { let snapshot = DirtyUsageBuckets::from([("photos".to_string(), 1), ("videos".to_string(), 2)]); let failed_buckets = HashSet::from(["videos".to_string()]); - let clear_snapshot = should_clear_dirty_usage_snapshot(true, true, false, &snapshot, &failed_buckets) + let clear_snapshot = should_clear_dirty_usage_snapshot(true, true, false, true, &snapshot, &failed_buckets) .expect("successful completed cycle should produce a clear snapshot"); assert!(clear_snapshot.contains_key("photos")); @@ -2109,7 +3991,8 @@ mod tests { assert!(dirty_usage_buckets().contains_key("photos")); let confirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone())); - confirmed.acknowledge_durable_usage(); + let acknowledgements = confirmed.acknowledge_durable_usage(); + assert!(acknowledgements.is_empty()); assert!(!dirty_usage_buckets().contains_key("photos")); clear_dirty_usage_buckets_for_tests(); } @@ -2194,19 +4077,152 @@ mod tests { #[test] fn scanner_cycle_status_requires_a_clean_complete_snapshot() { - assert_eq!(classify_nsscanner_cycle(true, false, false, false, true), ScannerCycleStatus::Complete); + assert_eq!( + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Unchanged, + ), + ScannerCycleStatus::Complete + ); + assert_eq!( + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Unchanged, + ), + ScannerCycleStatus::Superseded + ); + assert_eq!( + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Changed, + ), + ScannerCycleStatus::Superseded + ); + assert_eq!( + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Unverified, + ), + ScannerCycleStatus::Incomplete + ); for status in [ - classify_nsscanner_cycle(false, false, false, false, true), - classify_nsscanner_cycle(true, true, false, false, true), - classify_nsscanner_cycle(true, false, true, false, true), - classify_nsscanner_cycle(true, false, false, true, true), - classify_nsscanner_cycle(true, false, false, false, false), + classify_nsscanner_cycle( + false, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Unchanged, + ), + classify_nsscanner_cycle( + true, + true, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Unchanged, + ), + classify_nsscanner_cycle( + true, + false, + true, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Unchanged, + ), + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Failed, + DirtyUsageSnapshotStatus::Current, + ScannerCycleActivityStatus::Changed, + ), + classify_nsscanner_cycle( + false, + false, + false, + ScannerBucketScanStatus::Partial, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Changed, + ), ] { assert_eq!(status, ScannerCycleStatus::Incomplete); } } + #[test] + fn scanner_cycle_fails_closed_for_namespace_disappearance() { + for activity_status in [ + ScannerCycleActivityStatus::Changed, + ScannerCycleActivityStatus::Unchanged, + ScannerCycleActivityStatus::Unverified, + ] { + assert_eq!( + classify_nsscanner_cycle( + false, + false, + false, + ScannerBucketScanStatus::NamespaceNotFound, + DirtyUsageSnapshotStatus::Changed, + activity_status, + ), + ScannerCycleStatus::Incomplete + ); + } + assert_eq!( + classify_nsscanner_cycle( + true, + true, + false, + ScannerBucketScanStatus::NamespaceNotFound, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Changed, + ), + ScannerCycleStatus::Incomplete + ); + } + + #[test] + fn scanner_cycle_fails_closed_when_dirty_generation_is_unverified() { + assert_eq!( + classify_nsscanner_cycle( + true, + false, + false, + ScannerBucketScanStatus::Complete, + DirtyUsageSnapshotStatus::Unverified, + ScannerCycleActivityStatus::Unchanged, + ), + ScannerCycleStatus::Incomplete + ); + } + + #[test] + fn scanner_bucket_failure_status_preserves_the_strongest_failure() { + assert_eq!(scanner_bucket_scan_status(false, false, false), ScannerBucketScanStatus::Complete); + assert_eq!(scanner_bucket_scan_status(false, false, true), ScannerBucketScanStatus::NamespaceNotFound); + assert_eq!(scanner_bucket_scan_status(false, true, true), ScannerBucketScanStatus::Partial); + assert_eq!(scanner_bucket_scan_status(true, true, true), ScannerBucketScanStatus::Failed); + } + #[test] fn scanner_cycle_surfaces_persisted_pending_heal_work() { let clean = DataUsageCache::default(); @@ -2232,7 +4248,14 @@ mod tests { #[tokio::test] async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() { let pending_maintenance_work = Arc::new(AtomicBool::new(false)); - let mut bucket_cache = DataUsageCache::default(); + let mut bucket_cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "photos".to_string(), + ..Default::default() + }, + ..Default::default() + }; + bucket_cache.replace("photos", DATA_USAGE_ROOT, DataUsageEntry::default()); bucket_cache.info.pending_heals.push(crate::PendingScannerHeal { kind: crate::PendingScannerHealKind::Object, bucket: "photos".to_string(), @@ -2246,7 +4269,6 @@ mod tests { last_admission_reason: "capacity".to_string(), }); let (sender, mut receiver) = mpsc::channel(1); - let sender = Arc::new(Mutex::new(sender)); send_cache_root_entry_info(&sender, &bucket_cache, &pending_maintenance_work) .await @@ -2319,6 +4341,13 @@ mod tests { assert_eq!(decrement_atomic_usize(&counter), 0); } + #[test] + fn increment_atomic_usize_saturates_at_max() { + let counter = AtomicUsize::new(usize::MAX); + assert_eq!(increment_atomic_usize(&counter), usize::MAX); + assert_eq!(counter.load(Ordering::Relaxed), usize::MAX); + } + #[test] #[serial] fn scanner_max_concurrent_set_scans_uses_env_cap() { @@ -2571,7 +4600,7 @@ mod tests { }, ); - let info = cache_root_entry_info(&cache); + let info = cache_root_entry_info(&cache).expect("valid cache should flatten"); assert_eq!(info.name, "bucket"); assert_eq!(info.parent, DATA_USAGE_ROOT); @@ -2580,6 +4609,24 @@ mod tests { assert!(info.entry.children.is_empty()); } + #[test] + fn cache_root_entry_info_rejects_missing_or_dangling_roots() { + let missing_root = DataUsageCache { + info: DataUsageCacheInfo { + name: "bucket".to_string(), + ..Default::default() + }, + ..Default::default() + }; + assert!(cache_root_entry_info(&missing_root).is_err()); + + let mut dangling = missing_root; + let mut root = DataUsageEntry::default(); + root.add_child(&crate::hash_path("bucket/missing")); + dangling.replace("bucket", DATA_USAGE_ROOT, root); + assert!(cache_root_entry_info(&dangling).is_err()); + } + #[test] fn apply_bucket_result_to_cache_updates_bucket_entry() { let mut cache = DataUsageCache { diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index fd1ea75e4..6efc1d2a0 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -27,6 +27,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{ Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts, TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE, object_opts_from_object_info as ecstore_object_opts_from_object_info, }; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config, @@ -62,8 +64,8 @@ pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, pub(crate) use rustfs_ecstore::api::disk::{ BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI, DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation, - RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE, - ScanGuard as EcstoreScanGuard, + NsScannerOpenRequest as EcstoreNsScannerOpenRequest, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, + STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE, ScanGuard as EcstoreScanGuard, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::disk::{ @@ -72,6 +74,12 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::error::{ Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, }; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::layout::{ + EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, +}; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext; pub(crate) use rustfs_ecstore::api::runtime::{ expiry_state_handle as ecstore_expiry_state_handle, global_tier_config_mgr as ecstore_get_global_tier_config_mgr, object_store_handle as ecstore_resolve_object_store_handle, setup_is_erasure as ecstore_is_erasure, @@ -79,30 +87,39 @@ pub(crate) use rustfs_ecstore::api::runtime::{ }; pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks; pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx; pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig; use rustfs_storage_api as storage_contracts; pub(crate) mod owner { - pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete}; + pub(crate) use super::storage_contracts::{ + HTTPPreconditions, HTTPRangeSpec, NS_SCANNER_PROTOCOL_VERSION, ObjectIO, ObjectOperations, ObjectToDelete, + }; pub(crate) use super::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, - EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, - EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, - EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult, - ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle, - ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config, - ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, - ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, - ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, - ecstore_resolve_object_store_handle, ecstore_save_config, scanner_replication_config_for_lifecycle_eval, + EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, + EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, + EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, + ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, + ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, + ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, + ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, + ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, + ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, + ecstore_save_config, scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] - pub(crate) use super::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk}; + pub(crate) use super::{ + EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, + EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, + ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk, + }; } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -244,11 +261,17 @@ impl From for ScannerReplicationHealResult { } pub(crate) mod scan { - pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, NamespaceLocking}; + pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION; + pub(crate) use super::storage_contracts::{ + BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + }; + #[cfg(test)] + pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO}; } pub(crate) mod scanner_io { - pub(crate) use super::storage_contracts::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi}; + pub(crate) use super::storage_contracts::{BucketInfo, BucketOptions}; #[cfg(test)] pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO}; } diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index e54d36d86..1747b3424 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -17,6 +17,27 @@ pub const WALK_DIR_STREAM_COMPLETION_QUERY: &str = "walk_dir_stream_completion"; pub const WALK_DIR_STREAM_COMPLETION_V1: &str = "error-v1"; pub const WALK_DIR_BODY_SHA256_QUERY: &str = "walk_dir_body_sha256"; +pub const NS_SCANNER_BODY_SHA256_QUERY: &str = "ns_scanner_body_sha256"; +pub const NS_SCANNER_CAPABILITY_CHALLENGE_QUERY: &str = "ns_scanner_challenge"; +pub const NS_SCANNER_CYCLE_QUERY: &str = "ns_scanner_cycle"; +pub const NS_SCANNER_LEADER_EPOCH_QUERY: &str = "ns_scanner_leader_epoch"; +pub const NS_SCANNER_REQUEST_ID_QUERY: &str = "ns_scanner_request_id"; +pub const NS_SCANNER_SERVER_EPOCH_QUERY: &str = "ns_scanner_server_epoch"; +pub const NS_SCANNER_SESSION_ID_QUERY: &str = "ns_scanner_session_id"; +pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence"; +pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol"; +pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3; +pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0; +pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 4; +pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 5; + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub struct NsScannerCapabilityResponse { + pub version: u16, + pub server_epoch: uuid::Uuid, + pub proof: Vec, +} pub mod admin; pub mod bucket; diff --git a/crates/test-utils/src/data_usage_snapshot_tests.rs b/crates/test-utils/src/data_usage_snapshot_tests.rs index 94ed7694a..2d9be2341 100644 --- a/crates/test-utils/src/data_usage_snapshot_tests.rs +++ b/crates/test-utils/src/data_usage_snapshot_tests.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo}; +use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo}; use std::time::{Duration, SystemTime}; use crate::TestECStoreEnv; @@ -48,7 +48,7 @@ async fn data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache() .init_bucket_metadata(false) .build() .await; - let primary = format!("{BUCKET_META_PREFIX}/.usage.json"); + let primary = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}"); let backup = format!("{primary}.bkp"); let last_update = SystemTime::UNIX_EPOCH + Duration::from_secs(100); let expected = snapshot("bucket-a", last_update); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 93a39aaa4..47128351b 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,8 @@ for later deletion. ## Open Items +- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while the authoritative `.usage.v2.json` pair is absent and continue removing deleted buckets from legacy copies that still exist. Remove the fallback and legacy cleanup after every supported direct-upgrade source version writes `.usage.v2.json`. +- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version. - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability. - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request. - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus. diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 0881848e8..2499f2647 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -58,8 +58,8 @@ The `/v3/scanner/status` response reports each effective runtime value with a | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | | `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` | Enables scanner sleeps and cooperative throttling. | | `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `30` | Timeout for saving scanner cache; runtime enforces a minimum of `1`. | -| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `0` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | -| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `0` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | +| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | +| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | | `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` | Controls how often object loops yield to the async runtime. `0` disables this extra yield. | | `scanner.alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | versions | `100` | Version count threshold for scanner alerts. | | `scanner.alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | bytes | `1099511627776` | Retained version byte threshold for scanner alerts. | diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 575384980..be25c50be 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -259,8 +259,10 @@ impl Operation for AccountInfoHandler { .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; - // Serve the last persisted scanner snapshot plus the in-memory overlay. - // This request path must never trigger a live full-version listing + // Serve the last persisted scanner snapshot plus an authoritative + // single-process overlay. Distributed nodes must not promote their + // process-local absolute counters to cluster-wide bucket totals. + // This path never triggers a live full-version listing // (rustfs/backlog#1306); freshness is owned by the scanner. let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?; apply_bucket_usage_memory_overlay(&mut data_usage_info).await; diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index 0b6853832..b4d7e5f9f 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::handlers::supervise_admin_mutation; use crate::admin::handlers::target_descriptor::AdminTargetSpec; use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; use crate::admin::storage_api::config::{ - read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock, - with_admin_server_config_write_lock, + read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, }; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::DEFAULT_DELIMITER; @@ -96,20 +96,20 @@ where let Some(store) = current_object_store_handle_for_context(context) else { return Err(s3_error!(InternalError, "server storage not initialized")); }; - let specs = specs.to_vec(); - let lock_store = store.clone(); - with_admin_server_config_write_lock(lock_store, move || async move { - let mut config = read_admin_config_without_migrate_no_lock(store.clone()) + supervise_admin_mutation("audit config update", async move { + let snapshot = read_admin_server_config_snapshot(store.clone()) .await .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; + let mut config = snapshot.config.clone(); if !modifier(&mut config) { return Ok(()); } - save_admin_server_config_no_lock(store, &config) + save_admin_server_config_snapshot(store, &config, &snapshot) .await + .map(|_| ()) .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; // Keep persistence and runtime publication in one detached, serialized @@ -118,8 +118,6 @@ where apply_audit_runtime_config(&specs, config).await }) .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; - Ok(()) } pub(crate) async fn update_audit_config_and_reload(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()> diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index f2e1506dd..b9d409090 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -21,15 +21,14 @@ use crate::admin::runtime_sources::{ }; use crate::admin::service::config::{ CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN, - LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, - preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked, - signal_dynamic_config_reload_checked, + LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload, + prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload, + signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked, }; use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_api::config::{ - RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, - read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock, - with_admin_server_config_write_lock, + AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, + read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot, }; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; @@ -64,7 +63,7 @@ use rustfs_config::oidc::{ }; use rustfs_config::server_config::{Config as ServerConfig, DEFAULT_KVS, KV, KVS}; use rustfs_config::{ - COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS, + BASE_DSN_STRING, COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS, ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE, @@ -86,7 +85,6 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error} use serde::Serialize; use std::collections::{BTreeSet, HashMap}; use std::env; -use std::future::Future; use std::mem::size_of; use time::OffsetDateTime; use tracing::warn; @@ -750,14 +748,6 @@ async fn load_server_config_from_store() -> S3Result { .map_err(Into::into) } -async fn load_server_config_from_store_locked() -> S3Result { - let store = object_store()?; - read_admin_config_without_migrate_no_lock(store) - .await - .map_err(ApiError::from) - .map_err(Into::into) -} - async fn load_active_server_config() -> S3Result { if let Ok(config) = load_server_config_from_store().await { return Ok(config); @@ -768,9 +758,17 @@ async fn load_active_server_config() -> S3Result { .ok_or_else(|| s3_error!(InternalError, "server config is not initialized")) } -async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> { +async fn load_server_config_snapshot_from_store() -> S3Result { let store = object_store()?; - save_admin_server_config_no_lock(store, config) + read_admin_server_config_snapshot(store) + .await + .map_err(ApiError::from) + .map_err(Into::into) +} + +async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result { + let store = object_store()?; + save_admin_server_config_snapshot(store, config, snapshot) .await .map_err(ApiError::from) .map_err(Into::into) @@ -864,7 +862,17 @@ fn encode_config_history_recovery(config: &ServerConfig) -> Vec { fn encode_config_history_snapshot(config: &ServerConfig) -> S3Result> { let recovery = encode_config_history_recovery(config); - let display = render_full_config(config, true); + let mut display_config = config.clone(); + for targets in display_config.0.values_mut() { + for kvs in targets.values_mut() { + for entry in &mut kvs.0 { + if (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() { + entry.value = REDACTED_VALUE.to_string(); + } + } + } + } + let display = render_full_config(&display_config, false); let recovery_len = u64::try_from(recovery.len()).map_err(|_| s3_error!(InternalError, "config history recovery snapshot is too large"))?; let mut snapshot = @@ -1267,6 +1275,7 @@ fn is_sensitive_key_name(key: &str) -> bool { || normalized.ends_with("_tls_client_key") || normalized == "private_key" || normalized.ends_with("_private_key") + || normalized == BASE_DSN_STRING } fn is_sensitive_env_key_name(key: &str) -> bool { @@ -1415,7 +1424,7 @@ fn apply_delete_directives(config: &mut ServerConfig, directives: &[ConfigDirect } fn render_entry_value(entry: &KV, redact_secrets: bool) -> String { - if redact_secrets && (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() { + if redact_secrets && is_sensitive_key_name(&entry.key) && !entry.value.trim().is_empty() { REDACTED_VALUE.to_string() } else { entry.value.clone() @@ -1824,22 +1833,22 @@ fn build_help_response(sub_system: Option<&str>, key: Option<&str>, env_only: bo }) } +async fn reject_lost_config_transaction(snapshot: AdminServerConfigSnapshot, phase: &'static str) -> S3Result { + drop(snapshot); + reload_runtime_config_snapshot().await?; + signal_config_snapshot_reload().await; + Err(s3_error!( + InternalError, + "server config transaction lock was lost {phase}; runtime state was reloaded from durable config" + )) +} + fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> { prepared.publish_storage_class()?; publish_server_config(config); Ok(()) } -async fn commit_prepared_config( - config: ServerConfig, - prepared: PreparedRuntimeConfig, - persist: impl Future>, - publish: impl FnOnce(ServerConfig, PreparedRuntimeConfig) -> S3Result<()>, -) -> S3Result<()> { - persist.await?; - publish(config, prepared) -} - /// Re-apply local mutable worker families after a full-config replacement. /// Peers receive one full-snapshot signal after this returns; signaling each /// family here as well would recreate audit/scanner targets twice per peer. @@ -1851,9 +1860,26 @@ fn publish_notify_config_intent( .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone())) } -async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> { - if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) { - preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?; +fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> { + if let Some(sub_system) = sub_system { + return is_dynamic_config_subsystem(sub_system) + .then_some(sub_system) + .into_iter() + .collect(); + } + + let mut sub_systems = vec![NOTIFY_WEBHOOK_SUB_SYS]; + for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { + if !NOTIFY_SUB_SYSTEMS.contains(&sub_system) { + sub_systems.push(sub_system); + } + } + sub_systems +} + +async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> { + for sub_system in config_preflight_subsystems(sub_system) { + preflight_dynamic_config_reload(sub_system).await?; } Ok(()) } @@ -1869,13 +1895,13 @@ async fn wait_notify_config_intent(transition: Option Vec { +async fn reload_non_notify_dynamic_subsystems() -> Vec { let mut failures = Vec::new(); for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { if NOTIFY_SUB_SYSTEMS.contains(&sub_system) { continue; } - if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() { + if reload_dynamic_config_runtime_state(sub_system).await.is_err() { failures.push(format!("local {sub_system}")); warn!( event = EVENT_CONFIG_WORKER_RELOAD_FAILED, @@ -1904,7 +1930,6 @@ fn finish_config_reconciliation(errors: Vec) -> S3Result<()> { } async fn reconcile_targeted_config( - config: ServerConfig, sub_system: Option, storage_class_applied: bool, notify_transition: Option, @@ -1933,8 +1958,8 @@ async fn reconcile_targeted_config( } else if let Some(sub_system) = sub_system.as_deref() && is_dynamic_config_subsystem(sub_system) { - let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await { - Ok(applied) => applied, + let config_applied = match reload_dynamic_config_runtime_state(sub_system).await { + Ok(()) => true, Err(_) => { warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed"); errors.push(format!("local {sub_system}")); @@ -1958,10 +1983,7 @@ async fn reconcile_targeted_config( Ok(config_applied) } -async fn reconcile_full_config( - config: ServerConfig, - notify_transition: Option, -) -> S3Result<()> { +async fn reconcile_full_config(notify_transition: Option) -> S3Result<()> { let mut errors = Vec::new(); if let Err(err) = wait_notify_config_intent(notify_transition).await { warn!(error = %err, "Local notification config failed to converge"); @@ -1971,7 +1993,7 @@ async fn reconcile_full_config( warn!(error = %err, "Peer storage-class reload failed"); errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); } - errors.extend(apply_non_notify_dynamic_subsystems(&config).await); + errors.extend(reload_non_notify_dynamic_subsystems().await); if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await { warn!(error = %err, "Peer notification config reload failed"); errors.push("peer notify".to_string()); @@ -1983,6 +2005,90 @@ async fn reconcile_full_config( finish_config_reconciliation(errors) } +struct PersistedConfigTransaction { + previous_config: ServerConfig, + history_restore_id: Option, + storage_class_applied: bool, + notify_transition: Option, +} + +async fn persist_server_config_transaction( + config: ServerConfig, + prepared: PreparedRuntimeConfig, + snapshot: AdminServerConfigSnapshot, + sub_system: Option<&str>, +) -> S3Result { + snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?; + let previous_config = snapshot.config.clone(); + let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; + if snapshot.is_lock_lost() { + cleanup_failed_config_history_snapshot(&history_restore_id).await; + return reject_lost_config_transaction(snapshot, "before persistence").await; + } + + let persisted = match save_server_config_to_store(&config, &snapshot).await { + Ok(persisted) => persisted, + Err(err) => { + cleanup_failed_config_history_snapshot(&history_restore_id).await; + return Err(err); + } + }; + let history_restore_id = if persisted { + Some(history_restore_id) + } else { + cleanup_failed_config_history_snapshot(&history_restore_id).await; + None + }; + if snapshot.is_lock_lost() { + return reject_lost_config_transaction(snapshot, "after persistence").await; + } + + let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS); + if storage_class_applied { + if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) { + return Err(match history_restore_id.as_deref() { + Some(restore_id) => s3_error!( + InternalError, + "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", + restore_id, + err + ), + None => err, + }); + } + } else { + publish_server_config(config.clone()); + } + let notify_transition = publish_notify_config_intent(&config, sub_system); + if snapshot.is_lock_lost() { + return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await; + } + + drop(snapshot); + Ok(PersistedConfigTransaction { + previous_config, + history_restore_id, + storage_class_applied, + notify_transition, + }) +} + +async fn commit_server_config_transaction( + config: ServerConfig, + prepared: PreparedRuntimeConfig, + snapshot: AdminServerConfigSnapshot, + sub_system: Option, +) -> S3Result { + let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?; + + if sub_system.is_none() { + reconcile_full_config(transaction.notify_transition).await?; + return Ok(false); + } + + reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await +} + pub struct GetConfigKVHandler {} #[async_trait::async_trait] @@ -2016,41 +2122,13 @@ impl Operation for SetConfigKVHandler { validate_config_directives(&directives)?; let sub_system = config_update_sub_system(&directives)?.map(str::to_owned); - let transaction_sub_system = sub_system.clone(); - let config_store = object_store()?; let config_applied = supervise_admin_mutation("config mutation", async move { - preflight_notify_config_intent(sub_system.as_deref()).await?; - let (config, storage_class_applied, notify_transition) = - with_admin_server_config_write_lock(config_store, move || async move { - let sub_system = transaction_sub_system.as_deref(); - let previous_config = load_server_config_from_store_locked().await?; - let mut config = previous_config.clone(); - apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, sub_system).await?; - let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; - if let Err(err) = save_server_config_to_store_locked(&config).await { - cleanup_failed_config_history_snapshot(&history_restore_id).await; - return Err(err); - } - if sub_system == Some(STORAGE_CLASS_SUB_SYS) { - publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| { - s3_error!( - InternalError, - "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - history_restore_id, - err - ) - })?; - } else { - publish_server_config(config.clone()); - } - let notify_transition = publish_notify_config_intent(&config, sub_system); - Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition)) - }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; - - reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await + preflight_config_intent(sub_system.as_deref()).await?; + let snapshot = load_server_config_snapshot_from_store().await?; + let mut config = snapshot.config.clone(); + apply_set_directives(&mut config, &directives)?; + let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, prepared, snapshot, sub_system).await }) .await?; @@ -2072,41 +2150,13 @@ impl Operation for DelConfigKVHandler { validate_config_directives(&directives)?; let sub_system = config_update_sub_system(&directives)?.map(str::to_owned); - let transaction_sub_system = sub_system.clone(); - let config_store = object_store()?; let config_applied = supervise_admin_mutation("config mutation", async move { - preflight_notify_config_intent(sub_system.as_deref()).await?; - let (config, storage_class_applied, notify_transition) = - with_admin_server_config_write_lock(config_store, move || async move { - let sub_system = transaction_sub_system.as_deref(); - let previous_config = load_server_config_from_store_locked().await?; - let mut config = previous_config.clone(); - apply_delete_directives(&mut config, &directives); - let prepared = prepare_server_config(&config, sub_system).await?; - let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; - if let Err(err) = save_server_config_to_store_locked(&config).await { - cleanup_failed_config_history_snapshot(&history_restore_id).await; - return Err(err); - } - if sub_system == Some(STORAGE_CLASS_SUB_SYS) { - publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| { - s3_error!( - InternalError, - "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - history_restore_id, - err - ) - })?; - } else { - publish_server_config(config.clone()); - } - let notify_transition = publish_notify_config_intent(&config, sub_system); - Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition)) - }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; - - reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await + preflight_config_intent(sub_system.as_deref()).await?; + let snapshot = load_server_config_snapshot_from_store().await?; + let mut config = snapshot.config.clone(); + apply_delete_directives(&mut config, &directives); + let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, prepared, snapshot, sub_system).await }) .await?; @@ -2186,79 +2236,108 @@ impl Operation for RestoreConfigHistoryKVHandler { .ok_or_else(|| s3_error!(InvalidRequest, "missing restoreId query parameter"))?; let history = read_server_config_history(restore_id).await?; let config = decode_config_history_snapshot(&history)?; - let prepared = prepare_server_config(&config, None).await?; - let config_store = object_store()?; supervise_admin_mutation("config mutation", async move { - preflight_notify_config_intent(None).await?; - let persisted_config = config.clone(); + preflight_config_intent(None).await?; + let snapshot = load_server_config_snapshot_from_store().await?; + let prepared = prepare_server_config(&config, None).await?; let restored_config = config.clone(); - let rollback_store = config_store.clone(); - let (previous_config, recovery_restore_id, notify_transition) = - with_admin_server_config_write_lock(config_store, move || async move { - let previous_config = load_server_config_from_store_locked().await?; - let recovery_restore_id = save_server_config_history_snapshot(&previous_config).await?; - if let Err(err) = save_server_config_to_store_locked(&persisted_config).await { - cleanup_failed_config_history_snapshot(&recovery_restore_id).await; - return Err(err); - } - publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| { - s3_error!( - InternalError, - "config restore persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - recovery_restore_id, - err - ) - })?; - let notify_transition = publish_notify_config_intent(&persisted_config, None); - Ok::<_, S3Error>((previous_config, recovery_restore_id, notify_transition)) - }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??; - - let Err(restore_error) = reconcile_full_config(config, notify_transition).await else { + let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?; + let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else { return Ok(()); }; - let rollback_config = previous_config.clone(); - let rollback_expected_config = restored_config.clone(); - let rollback_transition = with_admin_server_config_write_lock(rollback_store, move || async move { - let current_config = load_server_config_from_store_locked().await?; - validate_restore_rollback_generation(¤t_config, &rollback_expected_config)?; - let rollback_prepared = prepare_server_config(&rollback_config, None).await?; - commit_prepared_config( - rollback_config.clone(), - rollback_prepared, - save_server_config_to_store_locked(&rollback_config), - publish_prepared_config_snapshots, - ) - .await?; - Ok::<_, S3Error>(publish_notify_config_intent(&rollback_config, None)) - }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config rollback: {}", err))?; - - let rollback_transition = match rollback_transition { - Ok(transition) => transition, + let previous_config = transaction.previous_config; + let recovery_restore_id = transaction.history_restore_id; + let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created"); + let rollback_snapshot = match load_server_config_snapshot_from_store().await { + Ok(snapshot) => snapshot, Err(rollback_error) => { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", restore_error, rollback_error, - recovery_restore_id + recovery_reference )); } }; - if let Err(rollback_error) = reconcile_full_config(previous_config, rollback_transition).await { + if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + + let rollback_prepared = prepare_server_config(&previous_config, None).await?; + rollback_snapshot + .ensure_lock_held() + .map_err(ApiError::from) + .map_err(S3Error::from)?; + if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + if rollback_snapshot.is_lock_lost() { + if let Err(rollback_error) = + reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await + { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); + } + + if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + let rollback_transition = publish_notify_config_intent(&previous_config, None); + if rollback_snapshot.is_lock_lost() { + if let Err(rollback_error) = + reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await + { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); + } + drop(rollback_snapshot); + + if let Err(rollback_error) = reconcile_full_config(rollback_transition).await { return Err(s3_error!( InternalError, "config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}", restore_error, rollback_error, - recovery_restore_id + recovery_reference )); } - cleanup_failed_config_history_snapshot(&recovery_restore_id).await; + if let Some(recovery_restore_id) = recovery_restore_id.as_deref() { + cleanup_failed_config_history_snapshot(recovery_restore_id).await; + } Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error)) }) .await?; @@ -2293,34 +2372,14 @@ impl Operation for SetConfigHandler { } validate_config_directives(&directives)?; - let mut config = ServerConfig::new(); - apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, None).await?; - let config_store = object_store()?; supervise_admin_mutation("config mutation", async move { - preflight_notify_config_intent(None).await?; - let persisted_config = config.clone(); - let notify_transition = with_admin_server_config_write_lock(config_store, move || async move { - let previous_config = load_server_config_from_store_locked().await?; - let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; - if let Err(err) = save_server_config_to_store_locked(&persisted_config).await { - cleanup_failed_config_history_snapshot(&history_restore_id).await; - return Err(err); - } - publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| { - s3_error!( - InternalError, - "full config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - history_restore_id, - err - ) - })?; - Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None)) - }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??; - - reconcile_full_config(config, notify_transition).await + preflight_config_intent(None).await?; + let snapshot = load_server_config_snapshot_from_store().await?; + let mut config = ServerConfig::new(); + apply_set_directives(&mut config, &directives)?; + let prepared = prepare_server_config(&config, None).await?; + commit_server_config_transaction(config, prepared, snapshot, None).await?; + Ok(()) }) .await?; @@ -2332,56 +2391,21 @@ impl Operation for SetConfigHandler { mod tests { use super::*; use serial_test::serial; - use std::sync::{Arc, Mutex}; use temp_env::with_vars; - #[tokio::test] - async fn prepared_config_commit_persists_before_publish() { - let events = Arc::new(Mutex::new(Vec::new())); - let persist_events = events.clone(); - let publish_events = events.clone(); - - commit_prepared_config( - ServerConfig::new(), - PreparedRuntimeConfig::default(), - async move { - persist_events.lock().expect("persist events lock").push("persist"); - Ok(()) - }, - move |_, _| { - publish_events.lock().expect("publish events lock").push("publish"); - Ok(()) - }, - ) - .await - .expect("prepared config commit"); - - assert_eq!(*events.lock().expect("result events lock"), ["persist", "publish"]); - } - - #[tokio::test] - async fn prepared_config_commit_does_not_publish_after_persist_failure() { - let events = Arc::new(Mutex::new(Vec::new())); - let persist_events = events.clone(); - let publish_events = events.clone(); - - let err = commit_prepared_config( - ServerConfig::new(), - PreparedRuntimeConfig::default(), - async move { - persist_events.lock().expect("persist events lock").push("persist"); - Err(s3_error!(InternalError, "injected persistence failure")) - }, - move |_, _| { - publish_events.lock().expect("publish events lock").push("publish"); - Ok(()) - }, - ) - .await - .expect_err("persistence failure must propagate"); - - assert_eq!(err.code(), &S3ErrorCode::InternalError); - assert_eq!(*events.lock().expect("result events lock"), ["persist"]); + #[test] + fn config_preflight_covers_each_runtime_worker_family() { + assert_eq!(config_preflight_subsystems(Some(SCANNER_SUB_SYS)), [SCANNER_SUB_SYS]); + assert_eq!(config_preflight_subsystems(Some(HEAL_SUB_SYS)), [HEAL_SUB_SYS]); + assert!(config_preflight_subsystems(Some("identity_openid")).is_empty()); + assert_eq!( + config_preflight_subsystems(None), + [ + NOTIFY_WEBHOOK_SUB_SYS, + rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, + SCANNER_SUB_SYS + ] + ); } #[test] @@ -2499,6 +2523,17 @@ mod tests { assert!(!rendered_after_delete.contains("client_secret=")); } + #[test] + fn config_rendering_redacts_database_dsn() { + let entry = KV { + key: BASE_DSN_STRING.to_string(), + value: "postgres://user:password@db.example/database".to_string(), + hidden_if_empty: true, + }; + + assert_eq!(render_entry_value(&entry, true), REDACTED_VALUE); + } + #[test] fn full_config_export_can_be_reapplied() { crate::admin::storage_api::config::init_admin_config_defaults(); @@ -2621,6 +2656,36 @@ identity_openid config_url="https://issuer.example" client_id="console""#, validate_config_directives(&directives).expect("scanner pacing keys should be supported"); } + #[test] + fn scanner_cycle_config_set_and_get_round_trip() { + let mut config = ServerConfig::new(); + let directives = parse_config_directives("scanner cycle=61", false).expect("parse scanner cycle directive"); + validate_config_directives(&directives).expect("scanner cycle should be supported"); + apply_set_directives(&mut config, &directives).expect("apply scanner cycle directive"); + + assert_eq!( + config + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("scanner KVS should exist") + .lookup(SCANNER_CYCLE) + .as_deref(), + Some("61") + ); + let rendered = String::from_utf8( + render_selected_config( + &config, + &ConfigSelector { + sub_system: SCANNER_SUB_SYS.to_string(), + target: Some(DEFAULT_DELIMITER.to_string()), + }, + true, + ) + .expect("render scanner cycle config"), + ) + .expect("scanner config should be UTF-8"); + assert!(rendered.contains("cycle=\"61\"")); + } + #[test] fn build_help_response_reports_scanner_delay() { let response = build_help_response(Some(SCANNER_SUB_SYS), Some(SCANNER_DELAY), false).expect("scanner help response"); diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index bed7593be..6b088ebc1 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_app_context, current_federated_identity_service, current_object_store_handle_for_context, @@ -20,8 +21,7 @@ use crate::admin::runtime_sources::{ }; use crate::admin::service::federated_identity::DefaultFederatedSessionBinding; use crate::admin::storage_api::config::{ - read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock, - with_admin_server_config_write_lock, + read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, }; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr}; @@ -381,17 +381,13 @@ impl Operation for PutOidcConfigHandler { let provider_id = provider_id.to_owned(); let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?; - let store = oidc_config_store()?; - let lock_store = store.clone(); - with_admin_server_config_write_lock(lock_store, move || async move { - let mut config = load_server_config_from_store_locked(store.clone()).await?; - let existing_secret = persisted_provider_secret(&config, &provider_id); + update_oidc_server_config(move |config| { + let existing_secret = persisted_provider_secret(config, &provider_id); let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?; - upsert_persisted_provider_config(&mut config, &provider_config); - save_server_config_to_store_locked(store, &config).await + upsert_persisted_provider_config(config, &provider_config); + Ok(()) }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; + .await?; json_response( StatusCode::OK, @@ -422,15 +418,11 @@ impl Operation for DeleteOidcConfigHandler { } let provider_id = provider_id.to_owned(); - let store = oidc_config_store()?; - let lock_store = store.clone(); - with_admin_server_config_write_lock(lock_store, move || async move { - let mut config = load_server_config_from_store_locked(store.clone()).await?; - delete_persisted_provider_config(&mut config, &provider_id)?; - save_server_config_to_store_locked(store, &config).await + update_oidc_server_config(move |config| { + delete_persisted_provider_config(config, &provider_id)?; + Ok(()) }) - .await - .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; + .await?; json_response( StatusCode::OK, @@ -916,21 +908,23 @@ fn oidc_config_store() -> S3Result, -) -> S3Result { - read_admin_config_without_migrate_no_lock(store) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}"))) -} - -async fn save_server_config_to_store_locked( - store: std::sync::Arc, - config: &ServerConfig, -) -> S3Result<()> { - save_admin_server_config_no_lock(store, config) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}"))) +async fn update_oidc_server_config(modifier: F) -> S3Result<()> +where + F: FnOnce(&mut ServerConfig) -> S3Result<()> + Send + 'static, +{ + let store = oidc_config_store()?; + supervise_admin_mutation("OIDC config update", async move { + let snapshot = read_admin_server_config_snapshot(store.clone()) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))?; + let mut config = snapshot.config.clone(); + modifier(&mut config)?; + save_admin_server_config_snapshot(store, &config, &snapshot) + .await + .map(|_| ()) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}"))) + }) + .await } fn is_env_managed_provider(provider_id: &str) -> bool { diff --git a/rustfs/src/admin/runtime_sources.rs b/rustfs/src/admin/runtime_sources.rs index 7f02f8c65..4d112e86d 100644 --- a/rustfs/src/admin/runtime_sources.rs +++ b/rustfs/src/admin/runtime_sources.rs @@ -30,7 +30,9 @@ pub(crate) use crate::runtime_sources::{ current_replication_stats_handle_for_context, current_server_config_for_context, current_token_signing_key, }; #[cfg(test)] -pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface}; +pub(crate) use crate::runtime_sources::{ + IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface, +}; use rustfs_config::server_config::Config; use rustfs_kms::KmsServiceManager; use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration}; diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index ab676aa6b..43e500d7c 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -37,9 +37,12 @@ use rustfs_targets::config::{ }; use s3s::{S3Error, S3ErrorCode, S3Result}; use std::future::Future; +use tokio::sync::Mutex as AsyncMutex; use tracing::warn; use url::Url; +static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(()); + pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool { NOTIFY_SUB_SYSTEMS.contains(&sub_system) || matches!( @@ -393,6 +396,7 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste } pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { + let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM { let store = resolve_runtime_config_store_for_context(context)?; let notify_result = reconcile_event_notifier_from_store(store).await; @@ -414,6 +418,14 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}"); internal_error(format!("failed to load server config: {err}")) })?; + + if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) { + validate_server_config_for_context(context, &config, Some(sub_system)).await?; + // Scanner cycles refresh from the process-wide server config before + // each pass. Publish the same validated snapshot first so that refresh + // cannot overwrite this peer's dynamic scanner update with stale data. + publish_server_config_for_context(context, config.clone()); + } apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) .await .inspect_err(|_| { @@ -463,6 +475,7 @@ where } pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> { + let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; let store = resolve_runtime_config_store_for_context(context)?; reload_runtime_config_snapshot_with( @@ -641,13 +654,16 @@ pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface}; + use crate::admin::runtime_sources::{ + IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface, + }; use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; use crate::admin::storage_api::config::{ read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config, save_admin_server_config_no_lock, with_admin_server_config_write_lock, }; use crate::admin::storage_api::error::StorageError; + use crate::admin::storage_api::runtime_sources::NotificationSys; use crate::server::{ ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled, refresh_persisted_module_switches_from, save_persisted_module_switches_to, @@ -656,7 +672,7 @@ mod tests { use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx}; use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES}; - use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; + use rustfs_config::{ENV_SCANNER_CYCLE, HEAL_SUB_SYS, SCANNER_CYCLE, SCANNER_SUB_SYS}; use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; @@ -675,6 +691,36 @@ mod tests { const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const REPLICATION_RELOAD_LABEL: &str = "replication"; + struct TestNotificationSystemInterface(Arc); + + impl NotificationSystemInterface for TestNotificationSystemInterface { + fn handle(&self) -> Option> { + Some(self.0.clone()) + } + } + + #[tokio::test] + async fn checked_scanner_reload_reports_unreachable_peer() { + let temp_dir = TempDir::new().expect("scanner reload temp dir"); + let store = build_isolated_heterogeneous_store(temp_dir.path()).await; + let mut notification_system = NotificationSys::new(EndpointServerPools::default()).await; + notification_system.peer_clients.push(None); + let context = AppContext::new( + store, + Arc::new(TestIamInterface), + Arc::new(TestKmsInterface { + manager: Arc::new(KmsServiceManager::new()), + }), + ) + .with_test_notification_system_interface(Arc::new(TestNotificationSystemInterface(Arc::new(notification_system)))); + + let err = signal_dynamic_config_reload_checked_for_context(Some(&context), SCANNER_SUB_SYS) + .await + .expect_err("scanner config must not report success when a peer did not reload it"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(err.message(), Some("1 peer(s) failed dynamic config reload for scanner")); + } + fn without_storage_class_env(f: impl FnOnce() -> R) -> R { temp_env::with_vars_unset( [ @@ -778,6 +824,16 @@ mod tests { config } + fn scanner_server_config(cycle: &str) -> ServerConfig { + let mut config = ServerConfig::new(); + let mut kvs = KVS::new(); + kvs.insert(SCANNER_CYCLE.to_string(), cycle.to_string()); + config + .0 + .insert(SCANNER_SUB_SYS.to_string(), HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)])); + config + } + async fn build_isolated_heterogeneous_store(temp_dir: &Path) -> Arc { let mut pools = Vec::new(); for (pool_index, drives_per_set) in [4, 2].into_iter().enumerate() { @@ -1124,6 +1180,47 @@ mod tests { .await; } + #[tokio::test] + #[serial_test::serial(scanner_env)] + async fn peer_scanner_reload_publishes_server_snapshot_used_by_cycle_refresh() { + temp_env::async_with_vars([(ENV_SCANNER_CYCLE, None::<&str>)], async { + let fixture = runtime_config_reload_fixture().await; + let candidate = scanner_server_config("61"); + save_admin_server_config(fixture.context.object_store(), &candidate) + .await + .expect("persist scanner config"); + + reload_dynamic_config_runtime_state_for_context(Some(&fixture.context), SCANNER_SUB_SYS) + .await + .expect("scanner peer reload should apply persisted config"); + + assert_eq!(fixture.server_set_calls.load(Ordering::SeqCst), 1); + let snapshot = fixture + .server_snapshot + .lock() + .expect("server config result lock") + .clone() + .expect("scanner peer reload should publish a server config snapshot"); + assert_eq!( + snapshot + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("scanner defaults should be present") + .get(SCANNER_CYCLE), + "61", + "scanner peer reload must keep the process-wide config source current" + ); + let runtime = rustfs_scanner::scanner_runtime_config_status(); + assert_eq!(runtime.cycle_interval_seconds.value, 61); + assert_eq!( + runtime.cycle_interval_seconds.source, + rustfs_scanner::runtime_config::ScannerRuntimeConfigSource::Config + ); + + rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults"); + }) + .await; + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn peer_full_reload_rejects_later_pool_without_publishing() { diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index dabd5724f..73611f152 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -639,10 +639,13 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc) -> Resu ecstore_config::com::read_config_without_migrate(api).await } +#[cfg(test)] pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc) -> Result { ecstore_config::com::read_config_without_migrate_no_lock(api).await } +pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot; + pub(crate) async fn save_admin_config(api: Arc, file: &str, data: Vec) -> Result<()> { ecstore_config::com::save_config(api, file, data).await } @@ -656,6 +659,7 @@ pub(crate) async fn save_admin_server_config(api: Arc, cfg: &rustfs_con ecstore_config::com::save_server_config(api, cfg).await } +#[cfg(test)] pub(crate) async fn save_admin_server_config_no_lock( api: Arc, cfg: &rustfs_config::server_config::Config, @@ -663,6 +667,7 @@ pub(crate) async fn save_admin_server_config_no_lock( ecstore_config::com::save_server_config_no_lock(api, cfg).await } +#[cfg(test)] pub(crate) async fn with_admin_server_config_write_lock(api: Arc, operation: F) -> Result where F: FnOnce() -> Fut + Send + 'static, @@ -672,6 +677,18 @@ where ecstore_config::com::with_server_config_write_lock(api, operation).await } +pub(crate) async fn read_admin_server_config_snapshot(api: Arc) -> Result { + ecstore_config::com::read_server_config_snapshot(api).await +} + +pub(crate) async fn save_admin_server_config_snapshot( + api: Arc, + cfg: &rustfs_config::server_config::Config, + snapshot: &AdminServerConfigSnapshot, +) -> Result { + ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await +} + pub(crate) fn init_admin_config_defaults() { ecstore_config::init(); } @@ -769,13 +786,16 @@ pub(crate) mod cluster { } pub(crate) mod config { - #[cfg(test)] - pub(crate) use super::save_admin_server_config; pub(crate) use super::storageclass; pub(crate) use super::{ - RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config, - read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config, - save_admin_server_config_no_lock, with_admin_server_config_write_lock, + AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, + read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, + save_admin_server_config_snapshot, + }; + #[cfg(test)] + pub(crate) use super::{ + read_admin_config_without_migrate_no_lock, save_admin_server_config, save_admin_server_config_no_lock, + with_admin_server_config_write_lock, }; } diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index ac1898f50..cd859e981 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -247,8 +247,10 @@ impl DefaultAdminUsecase { Self::query_data_usage_info_with_store(store).await } - /// Serve the last persisted scanner snapshot plus the in-memory overlay. - /// This request path must never trigger a live full-version listing + /// Serve the last persisted scanner snapshot plus an authoritative + /// single-process overlay. Distributed nodes must not promote their + /// process-local absolute counters to cluster-wide bucket totals. + /// This path never triggers a live full-version listing /// (rustfs/backlog#1306); freshness is owned by the scanner. pub(crate) async fn query_data_usage_info_with_store(store: Arc) -> AdminUsecaseResult { let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index c7c91c3b1..dd94e0a15 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -47,7 +47,6 @@ use super::storage_api::bucket_usecase::contract::bucket::{ use super::storage_api::bucket_usecase::contract::list::{ ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, }; -use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend; use super::storage_api::bucket_usecase::error::StorageError; use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context}; use super::storage_api::bucket_usecase::object_utils::to_s3s_etag; @@ -120,13 +119,18 @@ use std::{ borrow::Cow, collections::{HashMap, HashSet}, fmt::Display, + future::Future, io::Write, - sync::Arc, + sync::{Arc, LazyLock}, }; -use tracing::{debug, error, info, instrument, warn}; +use tokio::sync::{Semaphore, TryAcquireError}; +use tracing::{Instrument as _, debug, error, info, instrument, warn}; const LOG_COMPONENT_APP: &str = "app"; const LOG_SUBSYSTEM_BUCKET: &str = "bucket"; +const BUCKET_OPERATION_CONCURRENCY: usize = 8; +static BUCKET_OPERATION_ADMISSION: LazyLock> = + LazyLock::new(|| Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY))); use urlencoding::encode; type ListObjectVersionsInfo = StorageListObjectVersionsInfo; @@ -1087,6 +1091,40 @@ pub struct DefaultBucketUsecase { context: Option>, } +async fn await_bucket_usecase_on_fresh_task(operation: &'static str, future: F) -> S3Result +where + T: Send + 'static, + F: Future> + Send + 'static, +{ + await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), future).await +} + +async fn await_bucket_usecase_on_fresh_task_with_admission( + operation: &'static str, + admission: Arc, + future: F, +) -> S3Result +where + T: Send + 'static, + F: Future> + Send + 'static, +{ + let permit = admission.try_acquire_owned().map_err(|err| match err { + TryAcquireError::NoPermits => { + S3Error::with_message(S3ErrorCode::SlowDown, format!("{operation} concurrency limit reached; retry later")) + } + TryAcquireError::Closed => S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} admission closed")), + })?; + tokio::spawn( + async move { + let _permit = permit; + future.await + } + .in_current_span(), + ) + .await + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} task failed: {err}")))? +} + impl DefaultBucketUsecase { #[cfg(test)] pub fn without_context() -> Self { @@ -1121,6 +1159,11 @@ impl DefaultBucketUsecase { fields(start_time=?time::OffsetDateTime::now_utc()) )] pub async fn execute_create_bucket(&self, req: S3Request) -> S3Result> { + let usecase = self.clone(); + await_bucket_usecase_on_fresh_task("bucket creation", async move { usecase.execute_create_bucket_inner(req).await }).await + } + + async fn execute_create_bucket_inner(&self, req: S3Request) -> S3Result> { let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket); let requester_is_owner = match req_info_ref(&req) { Ok(r) => r.is_owner, @@ -1179,7 +1222,15 @@ impl DefaultBucketUsecase { } #[instrument(level = "debug", skip(self, req))] - pub async fn execute_delete_bucket(&self, mut req: S3Request) -> S3Result> { + pub async fn execute_delete_bucket(&self, req: S3Request) -> S3Result> { + let usecase = self.clone(); + await_bucket_usecase_on_fresh_task("bucket deletion", async move { usecase.execute_delete_bucket_inner(req).await }).await + } + + async fn execute_delete_bucket_inner( + &self, + mut req: S3Request, + ) -> S3Result> { let helper = OperationHelper::new(&req, EventName::BucketRemoved, S3Operation::DeleteBucket); let input = req.input.clone(); @@ -1218,12 +1269,8 @@ impl DefaultBucketUsecase { // Invalidate bucket validation cache crate::storage::invalidate_bucket_validation_cache(&input.bucket); - // Re-evaluate lifecycle/replication after bucket removal and keep an - // absent-bucket dirty marker until a complete usage snapshot is durable. + // Re-evaluate lifecycle and replication after bucket removal. rustfs_scanner::record_scanner_maintenance_change(&input.bucket); - if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await { - warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage"); - } if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await { warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed"); @@ -2633,6 +2680,155 @@ mod tests { ServerSideEncryptionConfiguration, Tag, Transition, TransitionStorageClass, }; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + use tokio::sync::Notify; + + #[tokio::test] + async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() { + let admission = Arc::new(Semaphore::new(1)); + let committed = Arc::new(Notify::new()); + let committed_wait = committed.notified(); + let release_hook = Arc::new(Notify::new()); + let finished = Arc::new(Notify::new()); + let finished_wait = finished.notified(); + let hook_ran = Arc::new(AtomicBool::new(false)); + let committed_for_task = committed.clone(); + let release_hook_for_task = release_hook.clone(); + let finished_for_task = finished.clone(); + let hook_ran_for_task = hook_ran.clone(); + let parent = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( + "test bucket operation", + admission.clone(), + async move { + committed_for_task.notify_one(); + release_hook_for_task.notified().await; + hook_ran_for_task.store(true, Ordering::SeqCst); + finished_for_task.notify_one(); + Ok(()) + }, + )); + committed_wait.await; + + parent.abort(); + let _ = parent.await; + release_hook.notify_waiters(); + tokio::time::timeout(Duration::from_secs(1), finished_wait) + .await + .expect("the post-commit hook should finish after request cancellation"); + + assert!( + hook_ran.load(Ordering::SeqCst), + "the fresh task must own both the storage mutation and its post-commit hooks" + ); + assert_eq!(admission.available_permits(), 1); + } + + #[tokio::test] + async fn saturated_bucket_usecase_admission_does_not_start_work() { + let admission = Arc::new(Semaphore::new(1)); + let held = admission + .clone() + .acquire_owned() + .await + .expect("test admission should remain open"); + let started = Arc::new(AtomicBool::new(false)); + let started_for_task = started.clone(); + let result = tokio::time::timeout( + Duration::from_secs(1), + await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async move { + started_for_task.store(true, Ordering::SeqCst); + Ok(()) + }), + ) + .await + .expect("saturated admission must fail within the bounded timeout"); + drop(held); + + assert!( + !started.load(Ordering::SeqCst), + "saturated admission must not start a detached bucket operation" + ); + assert_eq!(result.expect_err("saturated admission must fail fast").code(), &S3ErrorCode::SlowDown); + } + + #[tokio::test] + async fn bucket_usecase_admission_rejects_excess_detached_tasks() { + let admission = Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY)); + let release = Arc::new(Semaphore::new(0)); + let started = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::with_capacity(BUCKET_OPERATION_CONCURRENCY); + + for _ in 0..BUCKET_OPERATION_CONCURRENCY { + let admission_for_task = admission.clone(); + let release_for_task = release.clone(); + let started_for_task = started.clone(); + tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( + "test bucket operation", + admission_for_task, + async move { + started_for_task.fetch_add(1, Ordering::SeqCst); + let _release = release_for_task + .acquire() + .await + .expect("test release gate should remain open"); + Ok(()) + }, + ))); + } + + tokio::time::timeout(Duration::from_secs(1), async { + while started.load(Ordering::SeqCst) != BUCKET_OPERATION_CONCURRENCY { + tokio::task::yield_now().await; + } + }) + .await + .expect("all admitted operations should start"); + + let ninth_started = Arc::new(AtomicBool::new(false)); + let ninth_started_for_task = ninth_started.clone(); + let ninth_result = tokio::time::timeout( + Duration::from_secs(1), + await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission.clone(), async move { + ninth_started_for_task.store(true, Ordering::SeqCst); + Ok(()) + }), + ) + .await + .expect("an excess bucket transaction must fail within the bounded timeout"); + assert!( + !ninth_started.load(Ordering::SeqCst), + "the ninth bucket transaction must not start when admission is saturated" + ); + assert_eq!( + ninth_result.expect_err("the ninth bucket transaction must fail fast").code(), + &S3ErrorCode::SlowDown + ); + + release.add_permits(BUCKET_OPERATION_CONCURRENCY); + for task in tasks { + task.await + .expect("bucket transaction parent should join") + .expect("bucket transaction should succeed"); + } + + await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async { Ok(()) }) + .await + .expect("admission must recover after active transactions finish"); + } + + #[tokio::test] + async fn bucket_usecase_task_maps_panics_to_internal_errors() { + async fn panicking_bucket_operation() -> S3Result<()> { + panic!("injected bucket task panic"); + } + + let result = await_bucket_usecase_on_fresh_task("test bucket operation", panicking_bucket_operation()).await; + + let err = result.expect_err("a bucket task panic must be returned as an error"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert!(err.message().is_some_and(|message| message.contains("task failed"))); + } fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -2662,6 +2858,25 @@ mod tests { &rest[..end] } + #[test] + fn bucket_create_and_delete_use_admitted_fresh_tasks() { + let source = include_str!("bucket_usecase.rs"); + for (method, operation, inner_method) in [ + ("execute_create_bucket", "bucket creation", "execute_create_bucket_inner"), + ("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner"), + ] { + let body = usecase_method_source(source, method); + assert!( + body.contains(&format!("await_bucket_usecase_on_fresh_task(\"{operation}\"")), + "{method} must use bounded detached-task admission" + ); + assert!( + body.contains(&format!("{inner_method}(req).await")), + "{method} must run its mutation inside the admitted task" + ); + } + } + #[test] fn bucket_metadata_config_changes_notify_peer_metadata_reload() { let source = include_str!("bucket_usecase.rs"); diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index 3d0c87992..bd1baf9a6 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -354,6 +354,14 @@ impl AppContext { self.storage_class = storage_class; self } + + pub(crate) fn with_test_notification_system_interface( + mut self, + notification_system: Arc, + ) -> Self { + self.notification_system = notification_system; + self + } } static APP_CONTEXT_SINGLETON: OnceLock> = OnceLock::new(); diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index c0a65439d..f24fef4f4 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -110,13 +110,6 @@ pub(crate) mod data_usage { ) .await; } - - pub(crate) async fn remove_bucket_usage_from_backend( - store: Arc, - bucket: &str, - ) -> Result<(), crate::storage::storage_api::StorageError> { - crate::storage::storage_api::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await - } } pub(crate) mod runtime { @@ -956,7 +949,7 @@ pub(crate) mod bucket_usecase { } } - pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api}; + pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3_api}; pub(crate) use crate::storage::storage_api::{ ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, validate_list_object_unordered_with_delimiter, diff --git a/rustfs/src/runtime_sources.rs b/rustfs/src/runtime_sources.rs index c8575828a..9f8445535 100644 --- a/rustfs/src/runtime_sources.rs +++ b/rustfs/src/runtime_sources.rs @@ -61,7 +61,7 @@ pub(crate) fn set_test_outbound_tls_generation(generation: u64) { pub(crate) use context::install_test_app_context; #[cfg(test)] -pub(crate) use context::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface}; +pub(crate) use context::{IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface}; pub(crate) fn current_app_context() -> Option> { context::get_global_app_context() diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 4e16962e3..69010a02e 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -14,22 +14,26 @@ use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; -#[cfg(test)] -use crate::storage::storage_api::rpc_consumer::http_service::WALK_DIR_BODY_SHA256_QUERY; use crate::storage::storage_api::rpc_consumer::http_service::{ - DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, - verify_rpc_signature, + DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _, + WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature, +}; +#[cfg(test)] +use crate::storage::storage_api::rpc_consumer::http_service::{ + NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, + NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, + WALK_DIR_BODY_SHA256_QUERY, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use bytes::{Bytes, BytesMut}; use futures_util::{Stream, StreamExt, TryStreamExt, stream}; -use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; +use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri}; use http_body_util::{BodyExt, Limited}; use hyper::body::Incoming; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_io_metrics::internode_metrics::{ - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, - INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, }; use rustfs_utils::net::bytes_stream; use s3s::Body; @@ -39,11 +43,12 @@ use serde_urlencoded::from_bytes; use sha2::{Digest, Sha256}; use std::future::Future; use std::pin::Pin; +use std::sync::LazyLock; use std::task::{Context, Poll}; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio::io::{self, AsyncWriteExt}; use tokio::sync::oneshot; -use tokio_util::io::ReaderStream; +use tokio_util::{io::ReaderStream, sync::CancellationToken}; use tower::Service; use tracing::{error, warn}; @@ -52,6 +57,7 @@ type RpcErrorResponse = Box>; const LOG_COMPONENT_INTERNODE_RPC: &str = "internode_rpc"; const LOG_SUBSYSTEM_FILE_TRANSFER: &str = "file_transfer"; const LOG_SUBSYSTEM_DIRECTORY_WALK: &str = "directory_walk"; +const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner"; const LOG_SUBSYSTEM_ROUTING: &str = "routing"; const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected"; const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed"; @@ -60,6 +66,10 @@ const RPC_OPERATION_UNKNOWN: &str = "unknown"; const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream"; const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir"; +const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner"; +const NS_SCANNER_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(15); +const NS_SCANNER_STREAM_BUFFER_SIZE: usize = 64 * 1024; +static NS_SCANNER_SERVER_EPOCH: LazyLock = LazyLock::new(uuid::Uuid::new_v4); macro_rules! log_internode_rpc_response_failure { ($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{ @@ -247,6 +257,34 @@ struct WalkDirQuery { walk_dir_body_sha256: Option, } +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct NsScannerQuery { + disk: String, + ns_scanner_request_id: uuid::Uuid, + ns_scanner_server_epoch: uuid::Uuid, + ns_scanner_session_id: uuid::Uuid, + ns_scanner_session_sequence: u64, + ns_scanner_cycle: u64, + ns_scanner_leader_epoch: u64, + ns_scanner_body_sha256: Option, +} + +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct NsScannerCapabilityQuery { + ns_scanner_protocol: Option, + ns_scanner_challenge: Option, +} + +fn verify_ns_scanner_body_digest(query: &NsScannerQuery, body: &[u8]) -> bool { + let Some(expected) = query.ns_scanner_body_sha256.as_deref() else { + return false; + }; + let actual = hex_simd::encode_to_string(Sha256::digest(body), hex_simd::AsciiCase::Lower); + expected == actual +} + fn supports_walk_dir_stream_completion(query: &WalkDirQuery) -> bool { query.walk_dir_stream_completion.as_deref() == Some(WALK_DIR_STREAM_COMPLETION_V1) } @@ -322,6 +360,15 @@ async fn handle_internode_rpc(req: Request) -> Response { let response = match (method, path) { (Method::GET, READ_FILE_STREAM_PATH) | (Method::HEAD, READ_FILE_STREAM_PATH) => handle_read_file(req).await, (Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await, + (Method::GET, NS_SCANNER_PATH) => match parse_query::(&req) { + Ok(query) if query.ns_scanner_protocol == Some(NS_SCANNER_PROTOCOL_VERSION) => match query.ns_scanner_challenge { + Some(challenge) if !challenge.is_nil() => ns_scanner_capability_response(challenge), + Some(_) | None => response_with_status(StatusCode::BAD_REQUEST, "namespace scanner challenge is invalid"), + }, + Ok(_) => response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner protocol is unsupported"), + Err(response) => *response, + }, + (Method::POST, NS_SCANNER_PATH) => handle_ns_scanner(req).await, (Method::PUT, PUT_FILE_STREAM_PATH) => handle_put_file(req).await, _ => response_with_status(StatusCode::NOT_FOUND, "internode rpc route not found"), }; @@ -346,6 +393,7 @@ fn internode_http_operation(path: &str) -> Option<&'static str> { READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM), PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM), WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR), + NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER), _ => None, } } @@ -358,6 +406,64 @@ fn record_internode_rpc_error(operation: Option<&'static str>) { } } +fn ns_scanner_capability_response(challenge: uuid::Uuid) -> Response { + let server_epoch = *NS_SCANNER_SERVER_EPOCH; + let proof = match sign_ns_scanner_capability(challenge, server_epoch) { + Ok(proof) => proof, + Err(err) => { + error!( + event = EVENT_RPC_REQUEST_FAILED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "failed", + status_code = StatusCode::UPGRADE_REQUIRED.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::GET, + reason = "capability_authentication_unavailable", + error = %err, + "internode rpc request failed" + ); + return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable"); + } + }; + let body = match rmp_serde::to_vec_named(&NsScannerCapabilityResponse { + version: NS_SCANNER_PROTOCOL_VERSION, + server_epoch, + proof, + }) { + Ok(body) => body, + Err(err) => { + error!( + event = EVENT_RPC_REQUEST_FAILED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "failed", + status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::GET, + reason = "capability_response_encode_failed", + error = %err, + "internode rpc request failed" + ); + return response_with_status( + StatusCode::INTERNAL_SERVER_ERROR, + "namespace scanner capability response encoding failed", + ); + } + }; + let mut response = Response::new(Body::from(Bytes::from(body))); + response + .headers_mut() + .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack")); + response +} + +fn ns_scanner_server_epoch_matches(server_epoch: uuid::Uuid) -> bool { + server_epoch == *NS_SCANNER_SERVER_EPOCH +} + fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> { if method == Method::HEAD { return Ok(()); @@ -602,6 +708,304 @@ async fn handle_walk_dir(req: Request) -> Response { .expect("failed to build walk dir response") } +async fn handle_ns_scanner(req: Request) -> Response { + let query = match parse_query::(&req) { + Ok(query) => query, + Err(response) => return *response, + }; + if !ns_scanner_server_epoch_matches(query.ns_scanner_server_epoch) { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::CONFLICT.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "server_epoch_mismatch", + "internode rpc request rejected" + ); + return response_with_status(StatusCode::CONFLICT, "namespace scanner server epoch is stale"); + } + let Some(disk) = find_local_disk_by_ref(&query.disk).await else { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::BAD_REQUEST.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "disk_not_found", + disk = %query.disk, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::BAD_REQUEST, "disk not found"); + }; + if let Err(err) = rustfs_scanner::preflight_remote_scanner_request( + disk.as_ref(), + query.ns_scanner_session_id, + query.ns_scanner_cycle, + query.ns_scanner_leader_epoch, + query.ns_scanner_session_sequence, + ) { + let (status, reason, message) = remote_scanner_claim_rejection(&err); + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = status.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason, + disk = %query.disk, + error = %err, + "internode rpc request rejected" + ); + return response_with_status(status, message); + } + if let Err(err) = + rustfs_scanner::validate_remote_scanner_request_fence(query.ns_scanner_cycle, query.ns_scanner_leader_epoch).await + { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::CONFLICT.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "scanner_cycle_mismatch", + disk = %query.disk, + error = %err, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::CONFLICT, "namespace scanner cycle does not match persisted state"); + } + // Acquire the per-disk permit before reading the request body. This bounds + // slow or duplicated authenticated uploads to one request per physical disk; + // dropping the request on timeout releases the permit without consuming its + // replay sequence. + let admission = match rustfs_scanner::admit_remote_scanner_request(disk.as_ref()) { + Ok(admission) => admission, + Err(rustfs_scanner::ScannerError::RemoteDiskBusy) => { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::TOO_MANY_REQUESTS.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "disk_scan_already_active", + disk = %query.disk, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::TOO_MANY_REQUESTS, "namespace scanner disk is already active"); + } + Err(err) => { + let (status, reason, message) = remote_scanner_claim_rejection(&err); + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = status.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason, + disk = %query.disk, + error = %err, + "internode rpc request rejected" + ); + return response_with_status(status, message); + } + }; + let body = match tokio::time::timeout( + NS_SCANNER_REQUEST_BODY_TIMEOUT, + Limited::new(req.into_body(), rustfs_scanner::NS_SCANNER_MAX_REQUEST_BODY_SIZE).collect(), + ) + .await + { + Ok(Ok(body)) => body.to_bytes(), + Ok(Err(err)) => { + log_internode_rpc_response_failure!( + StatusCode::PAYLOAD_TOO_LARGE, + NS_SCANNER_PATH, + &Method::POST, + Some(INTERNODE_OPERATION_NS_SCANNER), + "request_body_read_failed", + "rejected", + Some(("disk", query.disk.as_str())), + Some(&err) + ); + return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, "namespace scanner request body is too large"); + } + Err(_) => { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::REQUEST_TIMEOUT.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "request_body_timeout", + disk = %query.disk, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::REQUEST_TIMEOUT, "namespace scanner request body timed out"); + } + }; + + if !verify_ns_scanner_body_digest(&query, &body) { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::FORBIDDEN.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "request_body_digest_mismatch", + disk = %query.disk, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::FORBIDDEN, "invalid request body digest"); + } + + let request = match rustfs_scanner::decode_remote_scanner_request(&body) { + Ok(request) => request, + Err(err) => { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::BAD_REQUEST.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "request_body_decode_failed", + disk = %query.disk, + error = %err, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::BAD_REQUEST, "invalid namespace scanner request"); + } + }; + if !rustfs_scanner::remote_scanner_request_matches_envelope( + &request, + query.ns_scanner_request_id, + query.ns_scanner_server_epoch, + query.ns_scanner_session_id, + query.ns_scanner_session_sequence, + query.ns_scanner_cycle, + query.ns_scanner_leader_epoch, + ) { + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = StatusCode::FORBIDDEN.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason = "request_envelope_mismatch", + disk = %query.disk, + "internode rpc request rejected" + ); + return response_with_status(StatusCode::FORBIDDEN, "namespace scanner request envelope mismatch"); + } + if let Err(err) = rustfs_scanner::claim_remote_scanner_request( + disk.as_ref(), + query.ns_scanner_session_id, + query.ns_scanner_cycle, + query.ns_scanner_leader_epoch, + query.ns_scanner_session_sequence, + ) { + let (status, reason, message) = remote_scanner_claim_rejection(&err); + warn!( + event = EVENT_RPC_REQUEST_REJECTED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "rejected", + status_code = status.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::POST, + reason, + disk = %query.disk, + error = %err, + "internode rpc request rejected" + ); + return response_with_status(status, message); + } + + let metrics = runtime_sources::current_internode_metrics(); + metrics + .record_incoming_request_for_operation_and_backend(INTERNODE_OPERATION_NS_SCANNER, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_NS_SCANNER, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + body.len(), + ); + + let log_disk = query.disk; + let response_body = ns_scanner_response_body(move |writer, disconnect| async move { + let _admission = admission; + rustfs_scanner::serve_remote_scanner_request(disk, request, writer, disconnect) + .await + .map_err(|err| { + error!( + event = EVENT_RPC_BACKGROUND_TASK_FAILED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "failed", + disk = %log_disk, + error = %err, + "internode rpc background task failed" + ); + io::Error::other("remote namespace scanner failed") + }) + }); + + let mut response = Response::new(response_body); + response + .headers_mut() + .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack")); + response +} + +fn remote_scanner_claim_rejection(error: &rustfs_scanner::ScannerError) -> (StatusCode, &'static str, &'static str) { + match error { + rustfs_scanner::ScannerError::RemoteRequestReplay => { + (StatusCode::CONFLICT, "request_replay", "namespace scanner request was already accepted") + } + rustfs_scanner::ScannerError::RemoteReplayCapacity => ( + StatusCode::TOO_MANY_REQUESTS, + "replay_capacity", + "namespace scanner request capacity is temporarily exhausted", + ), + _ => ( + StatusCode::SERVICE_UNAVAILABLE, + "replay_state_unavailable", + "namespace scanner replay state is unavailable", + ), + } +} + fn walk_dir_response_body(propagate_completion_errors: bool, producer: F) -> Body where F: FnOnce(tokio::io::DuplexStream) -> Fut + Send + 'static, @@ -656,6 +1060,52 @@ where ) } +fn ns_scanner_response_body(producer: F) -> Body +where + F: FnOnce(tokio::io::DuplexStream, CancellationToken) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, +{ + let (reader, writer) = tokio::io::duplex(NS_SCANNER_STREAM_BUFFER_SIZE); + let (mut completion_tx, completion_rx) = oneshot::channel(); + let disconnect = CancellationToken::new(); + spawn_traced(async move { + let producer = producer(writer, disconnect.clone()); + tokio::pin!(producer); + tokio::select! { + biased; + result = &mut producer => { + let _ = completion_tx.send(result); + } + _ = completion_tx.closed() => { + disconnect.cancel(); + let _ = producer.await; + } + } + }); + + let metrics = runtime_sources::current_internode_metrics(); + let stream = ReaderStream::with_capacity(reader, NS_SCANNER_STREAM_BUFFER_SIZE).map_ok(move |bytes| { + metrics.record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_NS_SCANNER, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + bytes.len(), + ); + bytes + }); + let completion = stream::once(async move { + match completion_rx.await { + Ok(Ok(())) => None, + Ok(Err(err)) => Some(Err(err)), + Err(err) => Some(Err(io::Error::other(format!( + "remote namespace scanner task ended without a result: {err}" + )))), + } + }) + .filter_map(std::future::ready); + + Body::from(StreamingBlob::wrap(stream.chain(completion))) +} + async fn handle_put_file(req: Request) -> Response { let method = req.method().clone(); let path = req.uri().path().to_string(); @@ -805,6 +1255,7 @@ fn response_with_status(status: StatusCode, message: impl Into) -> Respo fn internode_rpc_subsystem(operation: Option<&'static str>) -> &'static str { match operation { Some(INTERNODE_OPERATION_WALK_DIR) => LOG_SUBSYSTEM_DIRECTORY_WALK, + Some(INTERNODE_OPERATION_NS_SCANNER) => LOG_SUBSYSTEM_NAMESPACE_SCANNER, Some(INTERNODE_OPERATION_READ_FILE_STREAM | INTERNODE_OPERATION_PUT_FILE_STREAM) => LOG_SUBSYSTEM_FILE_TRANSFER, _ => LOG_SUBSYSTEM_ROUTING, } @@ -828,19 +1279,23 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std #[cfg(test)] mod tests { use super::{ - LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery, + LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING, + NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, + NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, + NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery, READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion, - internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, put_body_size_mismatch, - put_file_stage_error_message, read_file_body_stream, supports_walk_dir_stream_completion, - validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_walk_dir_body_digest, - walk_dir_response_body, write_body_chunks_to_writer, + internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body, + ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream, + remote_scanner_claim_rejection, supports_walk_dir_stream_completion, validate_walk_dir_completion_request, + verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body, + write_body_chunks_to_writer, }; use bytes::Bytes; use http::{HeaderMap, Method, StatusCode, Uri}; use http_body_util::BodyExt; use rustfs_io_metrics::internode_metrics::{ - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, - global_internode_metrics, + INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_OPERATION_WALK_DIR, global_internode_metrics, }; use sha2::Digest as _; use tokio::io; @@ -862,6 +1317,7 @@ mod tests { fn internode_rpc_path_matches_rpc_prefix() { assert!(is_internode_rpc_path("/rustfs/rpc/read_file_stream")); assert!(is_internode_rpc_path("/rustfs/rpc/walk_dir")); + assert!(is_internode_rpc_path("/rustfs/rpc/ns_scanner")); assert!(!is_internode_rpc_path("/rustfs/admin/v3/info")); } @@ -873,16 +1329,37 @@ mod tests { ); assert_eq!(internode_http_operation(PUT_FILE_STREAM_PATH), Some(INTERNODE_OPERATION_PUT_FILE_STREAM)); assert_eq!(internode_http_operation(WALK_DIR_PATH), Some(INTERNODE_OPERATION_WALK_DIR)); + assert_eq!(internode_http_operation(NS_SCANNER_PATH), Some(INTERNODE_OPERATION_NS_SCANNER)); assert_eq!(internode_http_operation("/rustfs/rpc/unknown"), None); } #[test] - fn rpc_head_signature_verification_is_skipped() { + fn file_stream_head_signature_verification_is_skipped() { let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri"); let headers = HeaderMap::new(); assert!(verify_internode_rpc_signature(&uri, &Method::HEAD, &headers).is_ok()); } + #[test] + fn namespace_scanner_capability_get_requires_signature() { + let challenge = uuid::Uuid::new_v4(); + let uri: Uri = format!( + "{NS_SCANNER_PATH}?ns_scanner_protocol={}&{NS_SCANNER_CAPABILITY_CHALLENGE_QUERY}={challenge}", + super::NS_SCANNER_PROTOCOL_VERSION + ) + .parse() + .expect("uri"); + let headers = HeaderMap::new(); + let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + + #[test] + fn namespace_scanner_rejects_requests_from_a_prior_server_epoch() { + assert!(ns_scanner_server_epoch_matches(*super::NS_SCANNER_SERVER_EPOCH)); + assert!(!ns_scanner_server_epoch_matches(uuid::Uuid::nil())); + } + #[test] fn rpc_get_request_requires_signature() { let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri"); @@ -941,9 +1418,92 @@ mod tests { LOG_SUBSYSTEM_FILE_TRANSFER ); assert_eq!(internode_rpc_subsystem(Some(INTERNODE_OPERATION_WALK_DIR)), LOG_SUBSYSTEM_DIRECTORY_WALK); + assert_eq!( + internode_rpc_subsystem(Some(INTERNODE_OPERATION_NS_SCANNER)), + LOG_SUBSYSTEM_NAMESPACE_SCANNER + ); assert_eq!(internode_rpc_subsystem(None), LOG_SUBSYSTEM_ROUTING); } + #[test] + fn namespace_scanner_requires_matching_signed_body_digest() { + let body = b"scanner-request"; + let digest = hex_simd::encode_to_string(sha2::Sha256::digest(body), hex_simd::AsciiCase::Lower); + let request_id = uuid::Uuid::new_v4(); + let server_epoch = uuid::Uuid::new_v4(); + let session_id = uuid::Uuid::new_v4(); + let valid: NsScannerQuery = serde_urlencoded::from_str(&format!( + "disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}={digest}" + )) + .expect("query should parse"); + let missing: NsScannerQuery = serde_urlencoded::from_str(&format!( + "disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9" + )) + .expect("query should parse"); + + assert!(verify_ns_scanner_body_digest(&valid, body)); + assert!(!verify_ns_scanner_body_digest(&valid, b"tampered")); + assert!(!verify_ns_scanner_body_digest(&missing, body)); + } + + #[test] + fn namespace_scanner_queries_reject_unknown_fields() { + let request_id = uuid::Uuid::new_v4(); + let server_epoch = uuid::Uuid::new_v4(); + let session_id = uuid::Uuid::new_v4(); + let query = format!( + "disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}=digest&unexpected=true" + ); + assert!(serde_urlencoded::from_str::(&query).is_err()); + assert!(serde_urlencoded::from_str::("ns_scanner_protocol=1&unexpected=true").is_err()); + } + + #[test] + fn namespace_scanner_replay_capacity_is_retryable_without_weakening_replay_rejection() { + let (replay_status, replay_reason, _) = + remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteRequestReplay); + let (capacity_status, capacity_reason, _) = + remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteReplayCapacity); + + assert_eq!(replay_status, StatusCode::CONFLICT); + assert_eq!(replay_reason, "request_replay"); + assert_eq!(capacity_status, StatusCode::TOO_MANY_REQUESTS); + assert_eq!(capacity_reason, "replay_capacity"); + } + + #[tokio::test] + async fn namespace_scanner_body_surfaces_background_failure() { + let body = ns_scanner_response_body(|mut writer, _disconnect| async move { + writer.write_all(b"partial scanner frame").await?; + Err(io::Error::other("remote namespace scanner failed")) + }); + let err = BodyExt::collect(body) + .await + .expect_err("failed completion must fail body collection"); + + assert!(err.to_string().contains("remote namespace scanner failed")); + } + + #[tokio::test] + async fn dropping_namespace_scanner_body_cancels_producer() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let body = ns_scanner_response_body(move |_writer, disconnect| async move { + let _drop_notifier = DropNotifier(Some(dropped_tx)); + let _ = started_tx.send(()); + disconnect.cancelled().await; + Ok(()) + }); + + started_rx.await.expect("namespace scanner producer should start"); + drop(body); + + tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx) + .await + .expect("dropping the response body should cancel the namespace scanner producer") + .expect("drop notifier should send a cancellation signal"); + } + #[tokio::test] async fn write_body_chunks_to_writer_streams_all_chunks() { let (mut reader, mut writer) = tokio::io::duplex(64); diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 2f8d6d411..be247a86c 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -23,8 +23,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType}; use crate::storage::storage_api::rpc_consumer::node_service::{ DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, - find_local_disk_by_ref, reload_transition_tier_config, + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, + SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref, + reload_transition_tier_config, }; use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources}; use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest}; @@ -78,6 +79,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1; const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2; const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3; +fn signal_service_response(success: bool, error_info: Option) -> Response { + Response::new(SignalServiceResponse { + success, + error_info, + protocol_version: rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION, + }) +} + fn supports_dynamic_config_rpc(sub_system: &str) -> bool { NOTIFY_SUB_SYSTEMS.contains(&sub_system) || matches!( @@ -174,11 +183,54 @@ fn validate_admin_heal_control_start(request: &rustfs_common::heal_channel::Heal Ok(()) } -fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse { +fn scanner_activity_response( + namespace_generation: u64, + topology_digest: [u8; 32], + data_movement_active: bool, + dirty_usage: rustfs_scanner::ScannerDirtyUsageState, +) -> ScannerActivityResponse { ScannerActivityResponse { instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), namespace_generation, maintenance_generation: rustfs_scanner::scanner_maintenance_generation(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: topology_digest.to_vec().into(), + data_movement_active, + response_proof: Bytes::new(), + dirty_usage_generation: dirty_usage.generation, + dirty_usage_pending: dirty_usage.pending, + } +} + +fn previous_scanner_activity_response( + namespace_generation: u64, + topology_digest: [u8; 32], + data_movement_active: bool, +) -> ScannerActivityResponse { + ScannerActivityResponse { + instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), + namespace_generation, + maintenance_generation: rustfs_scanner::scanner_maintenance_generation(), + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + topology_digest: topology_digest.to_vec().into(), + data_movement_active, + response_proof: Bytes::new(), + dirty_usage_generation: 0, + dirty_usage_pending: false, + } +} + +fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse { + ScannerActivityResponse { + instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), + namespace_generation, + maintenance_generation: rustfs_scanner::scanner_maintenance_generation(), + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + topology_digest: Bytes::new(), + data_movement_active: false, + response_proof: Bytes::new(), + dirty_usage_generation: 0, + dirty_usage_pending: false, } } @@ -1559,73 +1611,149 @@ impl Node for NodeService { Some(value) => match value.parse::() { Ok(value) => value, Err(_) => { - return Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(format!("invalid dry-run value: {value}")), - })); + return Ok(signal_service_response(false, Some(format!("invalid dry-run value: {value}")))); } }, }; match signal { Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await { - Ok(()) => Ok(Response::new(SignalServiceResponse { - success: true, - error_info: None, - })), - Err(_) => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some("runtime config snapshot reload failed".to_string()), - })), + Ok(()) => Ok(signal_service_response(true, None)), + Err(_) => Ok(signal_service_response(false, Some("runtime config snapshot reload failed".to_string()))), }, Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => { let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system); if !supported { - return Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")), - })); + return Ok(signal_service_response( + false, + Some(format!("unsupported dynamic config subsystem: {sub_system}")), + )); } if dry_run { - return Ok(Response::new(SignalServiceResponse { - success: true, - error_info: None, - })); + return Ok(signal_service_response(true, None)); } match reload_dynamic_config_runtime_state(sub_system).await { - Ok(()) => Ok(Response::new(SignalServiceResponse { - success: true, - error_info: None, - })), - Err(_) => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(format!("dynamic config reload failed for {sub_system}")), - })), + Ok(()) => Ok(signal_service_response(true, None)), + Err(_) => Ok(signal_service_response( + false, + Some(format!("dynamic config reload failed for {sub_system}")), + )), } } - Some(other) => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(format!("unsupported service signal: {other}")), - })), - None if raw_signal.is_some() => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())), - })), - None => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some("missing service signal".to_string()), - })), + Some(other) => Ok(signal_service_response(false, Some(format!("unsupported service signal: {other}")))), + None if raw_signal.is_some() => Ok(signal_service_response( + false, + Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())), + )), + None => Ok(signal_service_response(false, Some("missing service signal".to_string()))), } } async fn scanner_activity( &self, - _request: Request, + request: Request, ) -> Result, Status> { + let request_protocol = request.get_ref().protocol_version; + match request_protocol { + // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy request body is unbound. Remove after protocol v0 peers are unsupported. + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => { + if !request.get_ref().acknowledge_instance_id.is_empty() + || request.get_ref().acknowledge_dirty_usage_generation != 0 + { + return Err(Status::invalid_argument("legacy scanner activity request cannot acknowledge dirty usage")); + } + } + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + verify_tonic_canonical_body_digest(&request, request.get_ref().challenge.as_ref()) + .map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?; + if !request.get_ref().acknowledge_instance_id.is_empty() + || request.get_ref().acknowledge_dirty_usage_generation != 0 + { + return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage")); + } + } + rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => { + let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?; + verify_tonic_canonical_body_digest(&request, &canonical) + .map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?; + let has_acknowledgement = !request.get_ref().acknowledge_instance_id.is_empty(); + if has_acknowledgement != (request.get_ref().acknowledge_dirty_usage_generation != 0) { + return Err(Status::invalid_argument( + "scanner dirty usage acknowledgement requires both instance ID and generation", + )); + } + } + version => { + return Err(Status::failed_precondition(format!( + "unsupported scanner activity request protocol {version}" + ))); + } + } + let challenge_len = request.get_ref().challenge.len(); + if challenge_len != 16 && !(request_protocol == SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION && challenge_len == 0) { + return Err(Status::invalid_argument("scanner activity challenge must be 16 bytes")); + } let store = self .resolve_object_store() .ok_or_else(|| Status::unavailable("storage layer is not initialized"))?; - Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation()))) + if request.get_ref().challenge.is_empty() { + // Older peers send an empty protocol-0 request and cannot establish + // the topology fence required for distributed usage publication. + return Ok(Response::new(legacy_scanner_activity_response( + store.scanner_namespace_mutation_generation(), + ))); + } + let request = request.into_inner(); + let challenge: [u8; 16] = request + .challenge + .as_ref() + .try_into() + .map_err(|_| Status::invalid_argument("scanner activity challenge must be 16 bytes"))?; + if !request.acknowledge_instance_id.is_empty() { + rustfs_scanner::acknowledge_dirty_usage_generation( + &request.acknowledge_instance_id, + request.acknowledge_dirty_usage_generation, + ) + .map_err(|err| Status::failed_precondition(err.to_string()))?; + } + let namespace_generation = store.scanner_namespace_mutation_generation(); + let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref()); + let data_movement_active = store.scanner_data_movement_active().await; + let mut response = match request_protocol { + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active) + } + rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response( + namespace_generation, + topology_digest, + data_movement_active, + rustfs_scanner::scanner_dirty_usage_state(), + ), + version => { + return Err(Status::failed_precondition(format!( + "unsupported scanner activity request protocol {version}" + ))); + } + }; + let canonical = match response.protocol_version { + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response) + } + rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => { + rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response) + } + version => { + return Err(Status::internal(format!( + "scanner activity response selected unsupported protocol {version}" + ))); + } + } + .map_err(|_| Status::internal("scanner activity response is too large to authenticate"))?; + response.response_proof = sign_tonic_rpc_response_proof(&canonical) + .map_err(|_| Status::unavailable("scanner activity response authentication is unavailable"))? + .into(); + Ok(Response::new(response)) } async fn background_heal_status( @@ -1882,11 +2010,13 @@ impl Node for NodeService { mod tests { use super::{ CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService, - PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, + PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message, - execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server, - make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context, - remove_heal_control_replay, scanner_activity_response, stop_rebalance_response, + execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response, + make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context, + make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay, + scanner_activity_response, stop_rebalance_response, }; use crate::storage::rpc::node_service::heal::heal_topology_fingerprint; use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint}; @@ -4296,24 +4426,199 @@ mod tests { } #[tokio::test] - async fn test_scanner_activity_requires_storage_layer() { + async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() { let service = create_test_node_service(); - let err = service - .scanner_activity(Request::new(ScannerActivityRequest {})) + let legacy = service + .scanner_activity(Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + })) .await - .expect_err("activity queries must fail closed before storage is initialized"); + .expect_err("a rolling-upgrade request should pass authentication before storage lookup"); + assert_eq!(legacy.code(), tonic::Code::Unavailable); - assert_eq!(err.code(), tonic::Code::Unavailable); + let unsupported = service + .scanner_activity(Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION + 1, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + })) + .await + .expect_err("an unknown request protocol must fail before storage lookup"); + assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition); + + let malformed_legacy = service + .scanner_activity(Request::new(ScannerActivityRequest { + challenge: vec![7; 15].into(), + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + })) + .await + .expect_err("a malformed legacy challenge must fail before storage lookup"); + assert_eq!(malformed_legacy.code(), tonic::Code::InvalidArgument); + + let unsigned = service + .scanner_activity(Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + })) + .await + .expect_err("unsigned activity queries must fail before storage lookup"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + + let mut malformed_current = Request::new(ScannerActivityRequest { + challenge: vec![7; 15].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + let malformed_canonical = rustfs_protos::canonical_scanner_activity_request_body(malformed_current.get_ref()) + .expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut malformed_current, &malformed_canonical).expect("digest metadata should encode"); + mark_v2_authenticated(&mut malformed_current); + let malformed_current = service + .scanner_activity(malformed_current) + .await + .expect_err("a signed malformed challenge must fail before storage lookup"); + assert_eq!(malformed_current.code(), tonic::Code::InvalidArgument); + + let mut tampered = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + let other = ScannerActivityRequest { + challenge: vec![8; 16].into(), + ..tampered.get_ref().clone() + }; + let other_canonical = + rustfs_protos::canonical_scanner_activity_request_body(&other).expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut tampered, &other_canonical).expect("digest metadata should encode"); + mark_v2_authenticated(&mut tampered); + let tampered = service + .scanner_activity(tampered) + .await + .expect_err("tampered activity challenge must fail before storage lookup"); + assert_eq!(tampered.code(), tonic::Code::PermissionDenied); + + let mut downgraded = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + let current_canonical = rustfs_protos::canonical_scanner_activity_request_body(downgraded.get_ref()) + .expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut downgraded, ¤t_canonical).expect("digest metadata should encode"); + downgraded.get_mut().protocol_version = SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION; + mark_v2_authenticated(&mut downgraded); + let downgraded = service + .scanner_activity(downgraded) + .await + .expect_err("a signed current request must not be downgraded to protocol v4"); + assert_eq!(downgraded.code(), tonic::Code::PermissionDenied); + + let mut incomplete_acknowledgement = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), + acknowledge_dirty_usage_generation: 0, + }); + let acknowledgement_canonical = + rustfs_protos::canonical_scanner_activity_request_body(incomplete_acknowledgement.get_ref()) + .expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut incomplete_acknowledgement, &acknowledgement_canonical) + .expect("digest metadata should encode"); + mark_v2_authenticated(&mut incomplete_acknowledgement); + let incomplete_acknowledgement = service + .scanner_activity(incomplete_acknowledgement) + .await + .expect_err("dirty usage acknowledgements require an instance ID and generation"); + assert_eq!(incomplete_acknowledgement.code(), tonic::Code::InvalidArgument); + + let mut previous = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + set_tonic_canonical_body_digest(&mut previous, &[7; 16]).expect("protocol v4 digest metadata should encode"); + mark_v2_authenticated(&mut previous); + let previous = service + .scanner_activity(previous) + .await + .expect_err("an authenticated protocol v4 request should reach storage lookup during rolling upgrades"); + assert_eq!(previous.code(), tonic::Code::Unavailable); + + let mut signed = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + let signed_canonical = rustfs_protos::canonical_scanner_activity_request_body(signed.get_ref()) + .expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut signed, &signed_canonical).expect("digest metadata should encode"); + mark_v2_authenticated(&mut signed); + let unavailable = service + .scanner_activity(signed) + .await + .expect_err("authenticated activity queries still require initialized storage"); + assert_eq!(unavailable.code(), tonic::Code::Unavailable); } #[test] fn test_scanner_activity_response_uses_process_epoch_and_generations() { - let response = scanner_activity_response(17); + let response = scanner_activity_response( + 17, + [7; 32], + true, + rustfs_scanner::ScannerDirtyUsageState { + generation: 11, + pending: true, + }, + ); assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch()); assert_eq!(response.namespace_generation, 17); assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation()); + assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION); + assert_eq!(response.topology_digest.as_ref(), &[7; 32]); + assert!(response.data_movement_active); + assert_eq!(response.dirty_usage_generation, 11); + assert!(response.dirty_usage_pending); + } + + #[test] + fn test_previous_scanner_activity_response_omits_dirty_usage_fields() { + let response = previous_scanner_activity_response(17, [7; 32], true); + + assert_eq!(response.protocol_version, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION); + assert_eq!(response.topology_digest.as_ref(), &[7; 32]); + assert!(response.data_movement_active); + assert_eq!(response.dirty_usage_generation, 0); + assert!(!response.dirty_usage_pending); + } + + #[test] + fn test_legacy_scanner_activity_response_omits_extended_fields() { + let response = legacy_scanner_activity_response(17); + + assert_eq!(response.namespace_generation, 17); + assert_eq!(response.protocol_version, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION); + assert!(response.topology_digest.is_empty()); + assert!(!response.data_movement_active); + assert!(response.response_proof.is_empty()); + assert_eq!(response.dirty_usage_generation, 0); + assert!(!response.dirty_usage_pending); } #[tokio::test] @@ -4373,6 +4678,7 @@ mod tests { assert!(response.success, "new nodes must advertise notify lifecycle reload support"); assert!(response.error_info.is_none()); + assert_eq!(response.protocol_version, rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION); } #[tokio::test] diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 31f1da99f..e6f54da76 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -698,13 +698,24 @@ impl NodeService { Ok(volume_infos) => { let volume_infos = volume_infos .into_iter() - .filter_map(|volume_info| serde_json::to_string(&volume_info).ok()) - .collect(); - Ok(Response::new(ListVolumesResponse { - success: true, - volume_infos, - error: None, - })) + .enumerate() + .map(|(index, volume_info)| { + serde_json::to_string(&volume_info) + .map_err(|err| DiskError::other(format!("encode list volumes entry {index} failed: {err}"))) + }) + .collect::, DiskError>>(); + match volume_infos { + Ok(volume_infos) => Ok(Response::new(ListVolumesResponse { + success: true, + volume_infos, + error: None, + })), + Err(err) => Ok(Response::new(ListVolumesResponse { + success: false, + volume_infos: Vec::new(), + error: Some(err.into()), + })), + } } Err(err) => Ok(Response::new(ListVolumesResponse { success: false, diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index da959a502..b62e4d810 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -212,12 +212,23 @@ pub(crate) mod rpc_consumer { pub(crate) mod http_service { pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE; #[cfg(test)] - pub(crate) use super::super::storage_contracts::WALK_DIR_BODY_SHA256_QUERY; - pub(crate) use super::super::storage_contracts::WALK_DIR_STREAM_COMPLETION_V1; - pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature}; + pub(crate) use super::super::storage_contracts::{ + NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, + NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, + NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, WALK_DIR_BODY_SHA256_QUERY, + }; + pub(crate) use super::super::storage_contracts::{ + NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, WALK_DIR_STREAM_COMPLETION_V1, + }; + pub(crate) use super::super::{ + StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature, + }; } pub(crate) mod node_service { + pub(crate) use super::super::storage_contracts::{ + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + }; pub(crate) use super::super::{ BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, @@ -404,7 +415,7 @@ pub(crate) mod ecstore_data_usage { apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend, load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, - record_bucket_object_write_unknown_previous_memory, remove_bucket_usage_from_backend, store_compression_total_in_backend, + record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, }; // Test-only observables for the rustfs/backlog#1306 revert detector. #[cfg(test)] @@ -484,7 +495,8 @@ pub(crate) mod ecstore_rpc { pub(crate) use rustfs_ecstore::api::rpc::{ LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, - sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature, + sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, + verify_tonic_rpc_signature, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::{ @@ -1543,6 +1555,10 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h ecstore_rpc::verify_rpc_signature(url, method, headers) } +pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result> { + ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch) +} + pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> { ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers) }