From 8679570c2a328f3095f371be25a9e68997298bf7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 22 Aug 2026 16:23:40 +0800 Subject: [PATCH 01/32] fix(heal): retain displaced task status (#6370) --- crates/heal/src/heal/channel.rs | 54 +++++ crates/heal/src/heal/manager.rs | 115 +++++++++- crates/heal/src/heal/manager/auto_scan.rs | 17 +- crates/heal/src/heal/manager/scheduler.rs | 28 ++- crates/heal/src/heal/manager/tests.rs | 263 +++++++++++++++++++++- 5 files changed, 461 insertions(+), 16 deletions(-) diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index edfcf2613..e00435100 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -1640,6 +1640,60 @@ mod tests { assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0); } + #[tokio::test] + async fn test_process_query_request_reports_displaced_terminal_detail() { + let heal_manager = Arc::new(HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + )); + let mut displaced = HealRequest::new( + HealType::Bucket { + bucket: "displaced-channel".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + displaced.id = "displaced-channel-task".to_string(); + let displaced_id = displaced.id.clone(); + heal_manager + .submit_heal_request(displaced) + .await + .expect("initial channel task should queue"); + heal_manager + .submit_heal_request(HealRequest::new( + HealType::Bucket { + bucket: "successor-channel".to_string(), + }, + HealOptions::default(), + HealPriority::High, + )) + .await + .expect("successor channel task should displace the initial task"); + + let processor = HealChannelProcessor::new(heal_manager); + let (tx, rx) = oneshot::channel(); + processor + .process_query_request("displaced-channel".to_string(), displaced_id, None, tx) + .await + .expect("displaced query should process"); + let response = rx + .await + .expect("query response should be returned") + .expect("displaced query should remain successful"); + let payload: serde_json::Value = serde_json::from_slice(response.data.as_deref().expect("status payload should exist")) + .expect("status payload should be json"); + assert_eq!(payload["summary"], "stopped"); + assert!( + response + .error + .as_deref() + .is_some_and(|detail| detail.contains("reason=displaced")) + ); + } + #[tokio::test] async fn test_process_query_request_reports_running_for_queued_task() { let heal_manager = create_test_heal_manager(); diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index bad6d4b5b..fc8255e0b 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -40,6 +40,7 @@ use tracing::{debug, error, info, warn}; use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); +const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again"; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner"; const LOG_SUBSYSTEM_MANAGER: &str = "manager"; @@ -120,26 +121,30 @@ struct MrfRepairNoticeTarget { version_id: Option<[u8; 16]>, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] struct HealAdmissionDecision { result: HealAdmissionResult, - displaced_task_id: Option, + displaced_request: Option, } impl HealAdmissionDecision { const fn new(result: HealAdmissionResult) -> Self { Self { result, - displaced_task_id: None, + displaced_request: None, } } - fn accepted_with_displacement(displaced_task_id: String) -> Self { + fn accepted_with_displacement(displaced_request: HealRequest) -> Self { Self { result: HealAdmissionResult::Accepted, - displaced_task_id: Some(displaced_task_id), + displaced_request: Some(displaced_request), } } + + fn displaced_task_id(&self) -> Option<&str> { + self.displaced_request.as_ref().map(|request| request.id.as_str()) + } } fn lock_mrf_repair_notice_targets( @@ -151,6 +156,55 @@ fn lock_mrf_repair_notice_targets( } } +fn lock_displaced_terminals( + registry: &StdMutex>>, +) -> StdMutexGuard<'_, HashMap>> { + match registry.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn record_displaced_terminal( + registry: &StdMutex>>, + request: &HealRequest, +) -> Arc { + let terminal = Arc::new(CompletedHealStatus { + heal_type: request.heal_type.clone(), + status: HealTaskStatus::Failed { + error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"), + }, + result_items_truncated: false, + completed_at: SystemTime::now(), + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + }); + let mut terminals = lock_displaced_terminals(registry); + prune_completed_heal_statuses(&mut terminals); + terminals.insert(request.id.clone(), Arc::clone(&terminal)); + terminal +} + +async fn remove_displaced_task_aliases( + aliases: &Arc>>, + terminals: &StdMutex>>, + task_id: &str, + terminal: &Arc, +) { + let mut aliases = aliases.lock().await; + let alias_ids = aliases + .iter() + .filter_map(|(alias_id, alias)| (alias.task_id == task_id).then_some(alias_id.clone())) + .collect::>(); + let mut displaced_terminals = lock_displaced_terminals(terminals); + prune_completed_heal_statuses(&mut displaced_terminals); + for alias_id in alias_ids { + displaced_terminals.insert(alias_id, Arc::clone(terminal)); + } + aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); +} + async fn remove_task_aliases_for_task(registry: &Arc>>, task_id: &str) { registry .lock() @@ -618,6 +672,14 @@ pub struct HealManager { /// are shared so the lookup helper can hand a completed entry to a /// caller without cloning the retained result window. completed_heals: Arc>>>, + /// Terminals for requests removed by priority displacement. An Accepted + /// task ID remains queryable for the same process lifetime and the normal + /// ten-minute status TTL; clients should treat `reason=displaced` as a + /// terminal result and submit a fresh request. This sidecar is synchronous + /// so admission can publish the terminal while the queue transition is + /// still under its lock, without awaiting another tokio lock. Queue state + /// is process-local, so this guarantee does not extend across restart. + displaced_terminals: Arc>>>, /// Client tokens merged into an existing task id. task_aliases: Arc>>, /// Heal tasks waiting for a retry backoff to expire. @@ -659,6 +721,7 @@ struct HealQueueContext<'a> { heal_queue: &'a Arc>, active_heals: &'a Arc>>>, completed_heals: &'a Arc>>>, + displaced_terminals: &'a Arc>>>, task_aliases: &'a Arc>>, retrying_heals: &'a Arc>>, mrf_repair_notice_targets: &'a Arc>>>, @@ -874,7 +937,7 @@ impl HealManager { result = "accepted_by_displacement", "Heal queue request accepted by displacement" }); - return HealAdmissionDecision::accepted_with_displacement(displaced.id); + return HealAdmissionDecision::accepted_with_displacement(displaced); } demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", { @@ -1105,6 +1168,7 @@ impl HealManager { active_heals: Arc::new(Mutex::new(HashMap::new())), heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())), completed_heals: Arc::new(Mutex::new(HashMap::new())), + displaced_terminals: Arc::new(StdMutex::new(HashMap::new())), task_aliases: Arc::new(Mutex::new(HashMap::new())), retrying_heals: Arc::new(Mutex::new(HashMap::new())), mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())), @@ -1209,6 +1273,10 @@ impl HealManager { active_heals.clear(); publish_active_heal_count(&active_heals); self.completed_heals.lock().await.clear(); + // Do not let the synchronous guard live across the following async lock. + { + lock_displaced_terminals(&self.displaced_terminals).clear(); + } self.task_aliases.lock().await.clear(); self.retrying_heals.lock().await.clear(); lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear(); @@ -1459,7 +1527,11 @@ impl HealManager { task_id = queued_id.to_owned(); } let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; - let displaced_task_id = admission_decision.displaced_task_id; + let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned); + let displaced_terminal = admission_decision + .displaced_request + .as_ref() + .map(|request| record_displaced_terminal(&self.displaced_terminals, request)); if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged) && let Some(target) = mrf_notice_target { @@ -1473,8 +1545,12 @@ impl HealManager { drop(queue); drop(active_heals); - if let Some(displaced_task_id) = displaced_task_id { - self.remove_aliases_for_task(&displaced_task_id).await; + if let (Some(displaced_task_id), Some(displaced_terminal)) = (displaced_task_id, displaced_terminal) { + // The queue has already removed the displaced request, so the + // synchronous terminal sidecar was published before aliases and + // MRF ownership are cleaned up. + remove_displaced_task_aliases(&self.task_aliases, &self.displaced_terminals, &displaced_task_id, &displaced_terminal) + .await; } if should_notify { @@ -1549,6 +1625,15 @@ impl HealManager { } } + if terminal_completed.is_none() { + let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals); + prune_completed_heal_statuses(&mut displaced_terminals); + terminal_completed = displaced_terminals + .get(canonical_task_id) + .filter(|terminal| matches_path(&terminal.heal_type)) + .cloned(); + } + match terminal_completed { Some(completed) => TaskStateLookup::Completed(completed), None => TaskStateLookup::NotFound, @@ -1669,9 +1754,19 @@ impl HealManager { let mut completed_heals = self.completed_heals.lock().await; prune_completed_heal_statuses(&mut completed_heals); - completed_heals + if completed_heals .values() .any(|completed| heal_type_matches_path(&completed.heal_type, heal_path)) + { + return true; + } + drop(completed_heals); + + let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals); + prune_completed_heal_statuses(&mut displaced_terminals); + displaced_terminals + .values() + .any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path)) } /// Get task progress diff --git a/crates/heal/src/heal/manager/auto_scan.rs b/crates/heal/src/heal/manager/auto_scan.rs index 10ac136a4..f7eb4482d 100644 --- a/crates/heal/src/heal/manager/auto_scan.rs +++ b/crates/heal/src/heal/manager/auto_scan.rs @@ -21,6 +21,7 @@ impl HealManager { let heal_queue = self.heal_queue.clone(); let active_heals = self.active_heals.clone(); let task_aliases = self.task_aliases.clone(); + let displaced_terminals = self.displaced_terminals.clone(); let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone(); let storage = self.storage.clone(); let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); @@ -481,6 +482,10 @@ impl HealManager { let admission = admission_decision.result; let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; + let displaced_terminal = admission_decision + .displaced_request + .as_ref() + .map(|request| record_displaced_terminal(&displaced_terminals, request)); if matches!(admission, HealAdmissionResult::Accepted) && let Some(anchor) = recovery_anchor { @@ -491,8 +496,16 @@ impl HealManager { } drop(queue); drop(config); - if let Some(displaced_task_id) = admission_decision.displaced_task_id { - remove_task_aliases_for_task(&task_aliases, &displaced_task_id).await; + if let (Some(displaced_task_id), Some(displaced_terminal)) = + (admission_decision.displaced_task_id().map(ToOwned::to_owned), displaced_terminal) + { + remove_displaced_task_aliases( + &task_aliases, + &displaced_terminals, + &displaced_task_id, + &displaced_terminal, + ) + .await; lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id); } if matches!(admission, HealAdmissionResult::Accepted) { diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index fbee9f212..c76a037ae 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -21,6 +21,7 @@ impl HealManager { let heal_queue = self.heal_queue.clone(); let active_heals = self.active_heals.clone(); let completed_heals = self.completed_heals.clone(); + let displaced_terminals = self.displaced_terminals.clone(); let task_aliases = self.task_aliases.clone(); let retrying_heals = self.retrying_heals.clone(); let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone(); @@ -53,6 +54,7 @@ impl HealManager { heal_queue: &heal_queue, active_heals: &active_heals, completed_heals: &completed_heals, + displaced_terminals: &displaced_terminals, task_aliases: &task_aliases, retrying_heals: &retrying_heals, mrf_repair_notice_targets: &mrf_repair_notice_targets, @@ -71,6 +73,7 @@ impl HealManager { heal_queue: &heal_queue, active_heals: &active_heals, completed_heals: &completed_heals, + displaced_terminals: &displaced_terminals, task_aliases: &task_aliases, retrying_heals: &retrying_heals, mrf_repair_notice_targets: &mrf_repair_notice_targets, @@ -98,6 +101,7 @@ impl HealManager { heal_queue, active_heals, completed_heals, + displaced_terminals, task_aliases, retrying_heals, mrf_repair_notice_targets, @@ -183,6 +187,7 @@ impl HealManager { let active_heals_clone = active_heals.clone(); let heal_queue_clone = heal_queue.clone(); let completed_heals_clone = completed_heals.clone(); + let displaced_terminals_clone = displaced_terminals.clone(); let task_aliases_clone = task_aliases.clone(); let retrying_heals_clone = retrying_heals.clone(); let mrf_repair_notice_targets_clone = mrf_repair_notice_targets.clone(); @@ -363,6 +368,7 @@ impl HealManager { let retry_heal_queue = heal_queue_clone.clone(); let retrying_heals_for_spawn = retrying_heals_clone.clone(); let retry_task_aliases = task_aliases_clone.clone(); + let retry_displaced_terminals = displaced_terminals_clone.clone(); let retry_mrf_repair_notice_targets = mrf_repair_notice_targets_clone.clone(); let retry_completed_heals = completed_heals_clone.clone(); let retry_notify = notify_clone.clone(); @@ -430,6 +436,14 @@ impl HealManager { let admission = admission_decision.result; let should_notify = matches!(admission, HealAdmissionResult::Accepted) && retry_config.event_driven_scheduler_enable; + // Publish the terminal synchronously while the + // queue transition is protected. The subsequent + // queue -> retrying handoff retains the lock order + // used by operations_snapshot. + let displaced_terminal = admission_decision + .displaced_request + .as_ref() + .map(|request| record_displaced_terminal(&retry_displaced_terminals, request)); match admission { HealAdmissionResult::Accepted => { // Transfer ownership while holding queue -> retrying, @@ -437,10 +451,18 @@ impl HealManager { #[cfg(test)] pause_retry_ownership_transition(&retry_request_id, true).await; retrying_heals_for_spawn.lock().await.remove(&retry_request_id); - let displaced_task_id = admission_decision.displaced_task_id; + let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned); drop(queue); - if let Some(displaced_task_id) = displaced_task_id { - remove_task_aliases_for_task(&retry_task_aliases, &displaced_task_id).await; + if let (Some(displaced_task_id), Some(displaced_terminal)) = + (displaced_task_id, displaced_terminal) + { + remove_displaced_task_aliases( + &retry_task_aliases, + &retry_displaced_terminals, + &displaced_task_id, + &displaced_terminal, + ) + .await; remove_mrf_repair_notice_targets( &retry_mrf_repair_notice_targets, &displaced_task_id, diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index ad50c4bf7..585b7a4e7 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -84,6 +84,7 @@ async fn process_manager_queue_once(manager: &HealManager) { heal_queue: &manager.heal_queue, active_heals: &manager.active_heals, completed_heals: &manager.completed_heals, + displaced_terminals: &manager.displaced_terminals, task_aliases: &manager.task_aliases, retrying_heals: &manager.retrying_heals, mrf_repair_notice_targets: &manager.mrf_repair_notice_targets, @@ -2778,7 +2779,10 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { HealAdmissionResult::Accepted ); assert_eq!(manager.get_queue_length().await, 1); - assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. }))); + assert!(matches!( + manager.get_task_status(&low_id).await, + Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced") + )); assert_eq!( manager .get_task_status(&high_id) @@ -2788,6 +2792,263 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { ); } +#[tokio::test] +async fn displaced_task_remains_queryable() { + let manager = HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + let mut displaced = HealRequest::new( + HealType::Bucket { + bucket: "displaced-bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + displaced.id = "displaced-task".to_string(); + let displaced_id = displaced.id.clone(); + manager + .submit_heal_request(displaced) + .await + .expect("displaced request should queue"); + + let successor = HealRequest::new( + HealType::Bucket { + bucket: "successor-bucket".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + manager + .submit_heal_request(successor) + .await + .expect("successor should displace low work"); + + let report = manager + .get_task_report(&displaced_id) + .await + .expect("displaced report should remain queryable"); + assert!(matches!(report.status, HealTaskStatus::Failed { ref error } if error.contains("reason=displaced"))); +} + +#[tokio::test] +async fn displaced_archive_failure_keeps_queryable_terminal() { + let manager = HealManager::new(Arc::new(MockStorage), None); + let mut request = HealRequest::new( + HealType::Bucket { + bucket: "archive-failure".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + request.id = "archive-failure-task".to_string(); + let request_id = request.id.clone(); + // The synchronous sidecar is the authoritative fallback when the normal + // completed-task archive has no entry (the failure window that must not + // turn an Accepted ID into NotFound). + record_displaced_terminal(&manager.displaced_terminals, &request); + assert!(manager.completed_heals.lock().await.is_empty()); + assert!(matches!( + manager.get_task_status(&request_id).await, + Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced") + )); +} + +#[tokio::test] +async fn scheduler_retry_displacement_keeps_evicted_task_queryable() { + let manager = Arc::new(HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + queue_size: 1, + event_driven_scheduler_enable: false, + ..HealConfig::default() + }), + )); + let mut retry_request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + retry_request.priority = HealPriority::High; + let retry_id = retry_request.id.clone(); + manager + .submit_heal_request(retry_request) + .await + .expect("retry request should queue"); + + // Process exactly one queue cycle so the retry task is spawned without a + // background scheduler consuming the filler request before the retry wakes. + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if manager.retrying_heals.lock().await.contains_key(&retry_id) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("retry request should enter backoff"); + + let filler = HealRequest::new( + HealType::Bucket { + bucket: "retry-displaced-filler".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + let filler_id = filler.id.clone(); + manager + .submit_heal_request(filler) + .await + .expect("filler request should occupy the queue"); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if matches!( + manager.get_task_status(&filler_id).await, + Ok(HealTaskStatus::Failed { ref error }) if error.contains("reason=displaced") + ) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("retry admission should displace the filler request"); + assert_eq!(manager.get_queue_length().await, 1); + assert_eq!( + manager.get_task_status(&retry_id).await.expect("retry should be queued"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn concurrent_displacers_produce_one_terminal_generation() { + let manager = Arc::new(HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + )); + let mut displaced = HealRequest::new( + HealType::Bucket { + bucket: "concurrent-displaced".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + displaced.id = "concurrent-displaced-task".to_string(); + let displaced_id = displaced.id.clone(); + manager + .submit_heal_request(displaced) + .await + .expect("initial request should queue"); + + let first = HealRequest::new( + HealType::Bucket { + bucket: "concurrent-successor-a".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + let second = HealRequest::new( + HealType::Bucket { + bucket: "concurrent-successor-b".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + let (first_result, second_result) = tokio::join!(manager.submit_heal_request(first), manager.submit_heal_request(second)); + let accepted = [&first_result, &second_result] + .into_iter() + .filter(|result| matches!(result, Ok(HealAdmissionResult::Accepted))) + .count(); + assert_eq!(accepted, 1, "exactly one concurrent displacer should win the full queue"); + assert!( + first_result.is_ok() && second_result.is_ok(), + "the losing request should receive a typed Full result" + ); + let terminals = lock_displaced_terminals(&manager.displaced_terminals); + assert_eq!(terminals.len(), 1); + assert!(terminals.contains_key(&displaced_id)); +} + +#[tokio::test] +async fn successor_chain_is_bounded_and_authorized() { + let manager = HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + let mut original = HealRequest::new( + HealType::Bucket { + bucket: "authorized-original".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + original.id = "authorized-original-task".to_string(); + let original_id = original.id.clone(); + manager.submit_heal_request(original).await.expect("original should queue"); + let mut duplicate = HealRequest::new( + HealType::Bucket { + bucket: "authorized-original".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + duplicate.id = "authorized-duplicate-task".to_string(); + let duplicate_id = duplicate.id.clone(); + manager + .submit_heal_request(duplicate) + .await + .expect("same-target duplicate should merge"); + let successor = HealRequest::new( + HealType::Bucket { + bucket: "authorized-successor".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + let successor_id = successor.id.clone(); + manager.submit_heal_request(successor).await.expect("successor should queue"); + assert!(manager.task_aliases.lock().await.is_empty()); + assert!(matches!(manager.get_task_status(&original_id).await, Ok(HealTaskStatus::Failed { .. }))); + assert!(matches!(manager.get_task_status(&duplicate_id).await, Ok(HealTaskStatus::Failed { .. }))); + assert_eq!( + manager + .get_task_status(&successor_id) + .await + .expect("successor should remain queued"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn displaced_terminal_expires_after_bounded_ttl() { + let manager = HealManager::new(Arc::new(MockStorage), None); + let mut request = HealRequest::new( + HealType::Bucket { + bucket: "expires".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + request.id = "expires-task".to_string(); + let request_id = request.id.clone(); + record_displaced_terminal(&manager.displaced_terminals, &request); + { + let mut terminals = lock_displaced_terminals(&manager.displaced_terminals); + let entry = + Arc::get_mut(terminals.get_mut(&request_id).expect("terminal should be retained")).expect("test owns terminal entry"); + entry.completed_at = SystemTime::now() - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_secs(1); + } + assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. }))); +} + #[tokio::test] async fn test_displacing_registered_mrf_task_drops_notice_ownership() { let storage: Arc = Arc::new(MockStorage); From 1a3be70d98fcd7170dd7ca4e39b59be0eb7fa198 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 22 Aug 2026 17:07:02 +0800 Subject: [PATCH 02/32] fix(ecstore): preserve remote delete error types (#6371) --- crates/ecstore/src/cluster/rpc/remote_disk.rs | 101 +++++++++++++++--- .../src/generated/proto_gen/node_service.rs | 4 + crates/protos/src/node.proto | 3 + rustfs/src/storage/rpc/node_service/disk.rs | 54 ++++++++-- 4 files changed, 134 insertions(+), 28 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index efb1b4d8d..32bf3ae57 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -50,10 +50,10 @@ use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_protos::proto_gen::node_service::{ BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest, - DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, - MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, - ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, - RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, + DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, + ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, + ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, + RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient, }; @@ -112,6 +112,28 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc"; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60); +fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec> { + if !response.item_errors.is_empty() { + if response.item_errors.len() != expected_len { + return vec![Some(Error::other("malformed delete_versions item errors")); expected_len]; + } + return response + .item_errors + .into_iter() + .map(|error| (error.code != 0).then(|| error.into())) + .collect(); + } + + if response.errors.len() != expected_len { + return vec![Some(Error::other("malformed delete_versions errors")); expected_len]; + } + response + .errors + .into_iter() + .map(|error| (!error.is_empty()).then(|| Error::other(error))) + .collect() +} + fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result { if !response.success { return Err(response.error.unwrap_or_default().into()); @@ -2406,8 +2428,6 @@ impl DiskAPI for RemoteDisk { return errors; } - // TODO(backlog): replace string errors with typed `StorageError` variants - let result = self .execute_with_timeout( || async { @@ -2439,17 +2459,7 @@ impl DiskAPI for RemoteDisk { } return errors; } - response - .errors - .iter() - .map(|error| { - if error.is_empty() { - None - } else { - Some(Error::other(error.to_string())) - } - }) - .collect() + decode_delete_versions_errors(response, versions.len()) } #[tracing::instrument(level = "trace", skip_all)] @@ -3760,6 +3770,63 @@ mod tests { static INIT: Once = Once::new(); + #[test] + fn delete_versions_response_preserves_typed_item_errors() { + let errors = decode_delete_versions_errors( + DeleteVersionsResponse { + success: true, + errors: vec!["file not found".to_string(), String::new()], + error: None, + item_errors: vec![ + rustfs_protos::proto_gen::node_service::Error { + code: DiskError::FileNotFound.to_u32(), + error_info: "file not found".to_string(), + }, + rustfs_protos::proto_gen::node_service::Error::default(), + ], + }, + 2, + ); + + assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None])); + } + + #[test] + fn delete_versions_response_accepts_legacy_string_errors() { + let errors = decode_delete_versions_errors( + DeleteVersionsResponse { + success: true, + errors: vec!["legacy error".to_string(), String::new()], + error: None, + item_errors: Vec::new(), + }, + 2, + ); + + assert_eq!(errors.len(), 2); + assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error")); + assert!(errors[1].is_none()); + } + + #[test] + fn delete_versions_response_rejects_misaligned_item_errors() { + let errors = decode_delete_versions_errors( + DeleteVersionsResponse { + success: true, + errors: vec!["file not found".to_string()], + error: None, + item_errors: vec![rustfs_protos::proto_gen::node_service::Error { + code: DiskError::FileNotFound.to_u32(), + error_info: "file not found".to_string(), + }], + }, + 2, + ); + + assert_eq!(errors.len(), 2); + assert!(errors.iter().all(Option::is_some)); + } + #[test] fn disk_mutation_digest_marks_rolling_compatibility() { let mut request = Request::new(()); diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 6fa01bf1e..3885be4a3 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -722,6 +722,10 @@ pub struct DeleteVersionsResponse { pub errors: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, + /// Senders dual-write the legacy strings and typed entries. Receivers prefer typed entries + /// when present and fall back to strings for peers that predate this field. Code zero means success. + #[prost(message, repeated, tag = "4")] + pub item_errors: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadMultipleRequest { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 290b93b03..70ca93565 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -493,6 +493,9 @@ message DeleteVersionsResponse { bool success = 1; repeated string errors = 2; optional Error error = 3; + // Senders dual-write the legacy strings and typed entries. Receivers prefer typed entries + // when present and fall back to strings for peers that predate this field. Code zero means success. + repeated Error item_errors = 4; } message ReadMultipleRequest { diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 5d9fbc969..de80a10f0 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -146,6 +146,29 @@ fn encode_file_info_msgpack(value: &FileInfo) -> std::result::Result, Di encode_msgpack_with_capacity(value, "FileInfo", FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT) } +fn encode_delete_versions_errors(disk_errors: Vec>) -> (Vec, Vec) { + let mut errors = Vec::with_capacity(disk_errors.len()); + let mut item_errors = Vec::with_capacity(disk_errors.len()); + for error in disk_errors { + match error { + Some(error) => { + let code = match &error { + DiskError::Io(source) if source.kind() == std::io::ErrorKind::NotFound => DiskError::FileNotFound.to_u32(), + _ => error.to_u32(), + }; + let error_info = error.to_string(); + errors.push(error_info.clone()); + item_errors.push(Error { code, error_info }); + } + None => { + errors.push(String::new()); + item_errors.push(Error::default()); + } + } + } + (errors, item_errors) +} + fn encode_msgpack_named(value: &T, value_name: &str) -> std::result::Result, DiskError> { let mut serializer = rmp_serde::Serializer::new(Vec::with_capacity(MSGPACK_ENCODE_CAPACITY_HINT)).with_struct_map(); value @@ -552,6 +575,7 @@ impl NodeService { success: false, errors: Vec::new(), error: Some(DiskError::other(format!("decode FileInfoVersions failed: {err}")).into()), + item_errors: Vec::new(), })); } }; @@ -563,30 +587,26 @@ impl NodeService { success: false, errors: Vec::new(), error: Some(DiskError::other(format!("decode DeleteOptions failed: {err}")).into()), + item_errors: Vec::new(), })); } }; - let errors = disk - .delete_versions(&request.volume, versions, opts) - .await - .into_iter() - .map(|error| match error { - Some(e) => e.to_string(), - None => "".to_string(), - }) - .collect(); + let (errors, item_errors) = + encode_delete_versions_errors(disk.delete_versions(&request.volume, versions, opts).await); Ok(Response::new(DeleteVersionsResponse { success: true, errors, error: None, + item_errors, })) } else { Ok(Response::new(DeleteVersionsResponse { success: false, errors: Vec::new(), error: Some(DiskError::other("cannot find disk".to_string()).into()), + item_errors: Vec::new(), })) } } @@ -1612,8 +1632,8 @@ impl NodeService { mod tests { use super::{ compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, - encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, - encode_read_multiple_response_payloads, encode_rename_data_response_payloads, + encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, + encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; use crate::storage::rpc::node_service::make_server; use crate::storage::storage_api::ReadMultipleResp; @@ -1632,6 +1652,18 @@ mod tests { count: u32, } + #[test] + fn delete_versions_response_dual_writes_typed_item_errors() { + let raw_not_found = super::DiskError::Io(std::io::Error::from(std::io::ErrorKind::NotFound)); + let (errors, item_errors) = encode_delete_versions_errors(vec![Some(raw_not_found), None]); + + assert!(errors[0].starts_with("io error ")); + assert!(errors[1].is_empty()); + assert_eq!(item_errors[0].code, super::DiskError::FileNotFound.to_u32()); + assert_eq!(item_errors[0].error_info, errors[0]); + assert_eq!(item_errors[1].code, 0); + } + #[tokio::test] #[serial] async fn handle_read_version_records_attribution_for_missing_disk() { From 04e1ea227afc40fad583acceb0ef7df2c5ceea5a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 22 Aug 2026 19:11:48 +0800 Subject: [PATCH 03/32] fix(scanner): isolate corrupt cycle state (#6354) * fix(scanner): isolate corrupt cycle state * fix(scanner): preserve newer state during recovery reset * fix(scanner): fence recovery reset state * fix(scanner): reject terminal recovery epochs * fix(scanner): reject trailing cycle state bytes * fix(scanner): reject terminal leadership epochs * fix(scanner): retain recovery wake notifications * fix(scanner): recover from oversized markers --- crates/scanner/src/data_usage_define.rs | 33 + .../src/data_usage_define/persistence.rs | 29 +- crates/scanner/src/lib.rs | 5 +- crates/scanner/src/scanner.rs | 127 +- crates/scanner/src/scanner/cycle_state.rs | 1123 ++++++++++++++++- crates/scanner/src/scanner/leadership.rs | 2 +- crates/scanner/src/scanner/tests.rs | 931 +++++++++++++- rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/scanner.rs | 97 +- rustfs/src/admin/route_policy.rs | 12 + rustfs/src/admin/route_registration_test.rs | 3 + 11 files changed, 2264 insertions(+), 99 deletions(-) diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 75c3ca06e..c6ecdd489 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -125,6 +125,34 @@ pub(crate) async fn read_config_with_revision( } } +/// Read only the object revision without materializing its body. +pub(crate) async fn read_config_revision(store: Arc, path: &str) -> StorageResult { + match store + .get_object_reader( + RUSTFS_META_BUCKET, + path, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader + .object_info + .etag + .filter(|etag| !etag.is_empty()) + .map(DataUsageCacheRevision::Etag) + .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))), + Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { + Ok(DataUsageCacheRevision::Missing) + } + Err(err) => Err(err), + } +} + #[derive(Clone, Debug)] pub(crate) struct DataUsageCacheRevisions { main: DataUsageCacheRevision, @@ -146,6 +174,11 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock = pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}")); +/// Durable companion object for a cycle-state object which cannot be decoded. +/// The primary object is deliberately never replaced or deleted by recovery. +pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock = + LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str())); + pub static BACKGROUND_HEAL_INFO_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json")); diff --git a/crates/scanner/src/data_usage_define/persistence.rs b/crates/scanner/src/data_usage_define/persistence.rs index 2ac453cf4..b0a28f504 100644 --- a/crates/scanner/src/data_usage_define/persistence.rs +++ b/crates/scanner/src/data_usage_define/persistence.rs @@ -74,7 +74,7 @@ impl DataUsageCache { let loaded = Self::load_cache(store.clone(), name).await?; let backup = match loaded.backup_revision { Some(revision) => Some(revision), - None => match Self::revision_for_path(store, &backup_path).await { + None => match read_config_revision(store, &backup_path).await { Ok(revision) => Some(revision), Err(err) => { counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1); @@ -336,33 +336,6 @@ impl DataUsageCache { } } - async fn revision_for_path(store: Arc, path: &str) -> StorageResult { - match store - .get_object_reader( - RUSTFS_META_BUCKET, - path, - None, - HeaderMap::new(), - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(reader) => reader - .object_info - .etag - .filter(|etag| !etag.is_empty()) - .map(DataUsageCacheRevision::Etag) - .ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))), - Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { - Ok(DataUsageCacheRevision::Missing) - } - Err(err) => Err(err), - } - } - pub(super) fn cache_save_timeout() -> Duration { crate::runtime_config::scanner_cache_save_timeout() } diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index a1e37f14e..ab01964a5 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -75,7 +75,10 @@ pub use remote_scanner::{ }; pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use rustfs_common::last_minute; -pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest}; +pub use scanner::{ + ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner, + reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest, +}; pub use scanner_io::{ ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state, diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 4db558a54..7a4a482cf 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock}; use crate::data_usage_define::{ BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, - DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision, + DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision, }; use crate::runtime_config::{ ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle, @@ -54,9 +54,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA use rustfs_data_usage::observed_data_usage_is_newer; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; -#[cfg(test)] -use tokio::sync::Notify; -use tokio::sync::mpsc; +use tokio::sync::{Notify, mpsc}; use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; @@ -104,6 +102,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; /// unavailable peer cannot drive a tight retry loop. const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5); const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60); +/// A transient backend outage remains self-healing after the short retry +/// budget is exhausted, but the probe is intentionally sparse until storage +/// recovers or an operator reset wakes the scanner. +const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60); +/// Permanent recovery states still get a sparse status probe so a reset that +/// races the wait registration cannot leave the scanner asleep forever. +const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60); const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1); #[cfg(not(test))] const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); @@ -125,6 +130,12 @@ type ScannerCycleStatePersistTestHook = (u64, Arc); static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock>> = LazyLock::new(|| StdMutex::new(None)); +static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock = LazyLock::new(Notify::new); + +pub(super) fn notify_scanner_cycle_recovery_wake() { + SCANNER_CYCLE_RECOVERY_WAKE.notify_one(); +} + #[cfg(test)] struct ScannerCycleStatePersistTestHookGuard; @@ -576,19 +587,21 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { tokio::time::sleep(sleep_time).await; } + let mut transient_backoff = ScannerRetryBackoff::default(); + let mut recovery_retry_count = 0_u32; loop { if ctx_clone.is_cancelled() { break; } - if let Err(e) = run_data_scanner_with_maintenance_state( + let run_result = run_data_scanner_with_maintenance_state( ctx_clone.clone(), storeapi_clone.clone(), startup_features, startup_maintenance_generation, ) - .await - { + .await; + if let Err(e) = &run_result { error!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, @@ -599,11 +612,52 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { "Scanner runtime iteration failed" ); } + let recovery_status = scanner_cycle_recovery_status(); + if recovery_status.retryable { + recovery_retry_count = recovery_retry_count.saturating_add(1); + let _ = record_scanner_cycle_recovery_retry(recovery_retry_count); + } else { + recovery_retry_count = 0; + } + + let recovery_status = scanner_cycle_recovery_status(); + if recovery_status.state == "paused" { + transient_backoff.record_retryable_cycle(false); + tokio::select! { + _ = ctx_clone.cancelled() => break, + _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {}, + _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {}, + } + recovery_retry_count = 0; + continue; + } + if !recovery_status.retryable + && matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending") + { + transient_backoff.record_retryable_cycle(false); + tokio::select! { + _ = ctx_clone.cancelled() => break, + _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {}, + _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {}, + } + continue; + } + + let retry_delay = if recovery_status.retryable || run_result.is_err() { + transient_backoff.record_retryable_cycle(true); + transient_backoff + .retry_interval(scanner_cycle_interval()) + .unwrap_or(SCANNER_RETRY_BASE_INTERVAL) + } else { + transient_backoff.record_retryable_cycle(false); + randomized_cycle_delay() + }; // Backoff before retrying after lock contention or scanner-level failures. // Keep this cancellation-aware so shutdown is not delayed by backoff sleep. tokio::select! { _ = ctx_clone.cancelled() => break, - _ = tokio::time::sleep(randomized_cycle_delay()) => {} + _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {}, + _ = tokio::time::sleep(retry_delay) => {} } } }); @@ -1606,40 +1660,22 @@ async fn run_data_scanner_with_maintenance_state( observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await; } - let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await { - Ok((buf, revision)) => (buf.unwrap_or_default(), revision), - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "revision_load_failed", - error = %err, - "Scanner cycle state revision load failed" - ); - global_metrics().set_cycle(None).await; - return Ok(()); - } - }; - let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) { - Ok(state) => state, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "cycle_decode_failed", - error = %err, - "Scanner stopped because persisted cycle state is invalid" - ); - global_metrics().set_cycle(None).await; - return Ok(()); - } - }; + let (mut cycle_info, mut leader_epoch, mut cycle_revision) = + match load_scanner_cycle_state_for_startup(storeapi.clone()).await { + ScannerCycleStateStartup::Ready { + cycle, + leader_epoch, + revision, + } => (cycle, leader_epoch, revision), + ScannerCycleStateStartup::Blocked => { + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleStateStartup::Transient(err) => { + global_metrics().set_cycle(None).await; + return Err(err); + } + }; let usage_floor = match persisted_usage_floor(storeapi.clone()).await { Ok(floor) => floor, Err(err) => { @@ -2219,7 +2255,12 @@ pub(crate) use activity::{ pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; #[cfg(test)] pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test; -pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence}; +pub use cycle_state::{ + ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status, +}; +pub(crate) use cycle_state::{ + current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup, +}; pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info}; pub use usage_store::store_data_usage_in_backend; diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 6f3af4b81..32c893fdf 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -13,6 +13,1067 @@ // limitations under the License. /// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence. use super::*; +use crate::ScannerGetObjectReader; +use crate::data_usage_define::DATA_USAGE_BLOOM_RECOVERY_PATH; +use crate::storage_api::owner::ObjectIO as _; +use tokio::io::AsyncReadExt as _; + +const SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION: u16 = 1; +const MAX_SCANNER_CYCLE_STATE_BYTES: u64 = 1024 * 1024; +pub(super) const MAX_SCANNER_CYCLE_RECOVERY_RETRIES: u32 = 5; +const METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED: &str = "rustfs_scanner_cycle_recovery_required"; +const METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT: &str = "rustfs_scanner_cycle_recovery_retry_count"; + +#[derive(Clone, Debug, Default, Serialize)] +pub struct ScannerCycleRecoveryStatus { + /// The immutable primary object whose revision is being guarded. + pub path: String, + /// The companion marker/quarantine object containing the recovery evidence. + pub quarantine_path: Option, + pub state: String, + pub classification: Option, + pub primary_revision: Option, + pub generation: Option, + pub leader_epoch: Option, + pub first_detected_at_unix_secs: Option, + pub last_attempt_at_unix_secs: Option, + pub retry_count: u64, + pub max_retries: u32, + /// Whether the scanner may retry this state automatically. + pub retryable: bool, + pub reason: Option, +} + +static SCANNER_CYCLE_RECOVERY_STATUS: LazyLock> = LazyLock::new(|| { + RwLock::new(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "healthy".to_string(), + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + ..Default::default() + }) +}); + +pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus { + SCANNER_CYCLE_RECOVERY_STATUS + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() +} + +fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) { + let recovery_required = if matches!(status.state.as_str(), "blocked" | "paused" | "recovery-required" | "cleanup-pending") { + 1.0 + } else { + 0.0 + }; + metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED).set(recovery_required); + metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT).set(status.retry_count as f64); + *SCANNER_CYCLE_RECOVERY_STATUS + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = status; +} + +pub(super) fn record_scanner_cycle_recovery_retry(attempt: u32) -> bool { + let mut status = scanner_cycle_recovery_status(); + status.retry_count = u64::from(attempt); + status.last_attempt_at_unix_secs = Some(unix_now_secs()); + if attempt >= MAX_SCANNER_CYCLE_RECOVERY_RETRIES { + status.state = "paused".to_string(); + status.retryable = false; + status.reason = Some("scanner cycle recovery retry budget reached; sparse backend probes continue".to_string()); + set_scanner_cycle_recovery_status(status); + false + } else { + status.retryable = true; + set_scanner_cycle_recovery_status(status); + true + } +} + +fn unix_now_secs() -> u64 { + u64::try_from(Utc::now().timestamp()).unwrap_or(0) +} + +fn recovery_status(state: &str, reason: Option<&str>, retryable: bool) -> ScannerCycleRecoveryStatus { + ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: state.to_string(), + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable, + last_attempt_at_unix_secs: Some(unix_now_secs()), + reason: reason.map(str::to_string), + ..Default::default() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScannerCycleRecoveryMarker { + pub schema_version: u16, + pub primary_revision: String, + pub generation: u64, + pub leader_epoch: u64, + pub classification: String, + pub first_detected_at_unix_secs: u64, + pub last_attempt_at_unix_secs: u64, + pub retry_count: u64, + pub reason: String, + pub path: String, + pub quarantine_path: String, + /// `blocked` means the marker guards the primary revision; `cleanup-pending` + /// means an operator reset is in progress and must remain fenced across a + /// restart, even if the primary object is subsequently rewritten. + #[serde(default = "default_recovery_marker_state")] + pub state: String, +} + +fn default_recovery_marker_state() -> String { + "blocked".to_string() +} + +#[derive(Debug, Deserialize)] +struct ScannerCycleRecoveryMarkerCompat { + schema_version: Option, + primary_revision: Option, + classification: Option, + first_detected_at_unix_secs: Option, + last_attempt_at_unix_secs: Option, + retry_count: Option, + reason: Option, + path: Option, + quarantine_path: Option, + state: Option, +} + +#[derive(Debug)] +pub(crate) enum ScannerCycleStateStartup { + Ready { + cycle: CurrentCycle, + leader_epoch: u64, + revision: DataUsageCacheRevision, + }, + Blocked, + Transient(ScannerError), +} + +#[derive(Debug, thiserror::Error)] +enum CycleRecoveryMarkerReadError { + #[error("cycle recovery marker backend read failed: {0}")] + Backend(#[source] EcstoreError), + #[error("invalid cycle recovery marker: {0}")] + Invalid(&'static str), + #[error("cycle recovery marker revision changed while publishing")] + Conflict, +} + +#[derive(Debug, thiserror::Error)] +enum CycleStateBodyReadError { + #[error("scanner cycle state exceeds the bounded object size")] + TooLarge, + #[error("scanner cycle state body read failed: {0}")] + Backend(#[source] EcstoreError), +} + +fn recovery_status_from_marker(marker: &ScannerCycleRecoveryMarker, state: &str) -> ScannerCycleRecoveryStatus { + ScannerCycleRecoveryStatus { + path: marker.path.clone(), + quarantine_path: Some(marker.quarantine_path.clone()), + state: state.to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(marker.primary_revision.clone()), + generation: Some(marker.generation), + leader_epoch: Some(marker.leader_epoch), + first_detected_at_unix_secs: Some(marker.first_detected_at_unix_secs), + last_attempt_at_unix_secs: Some(marker.last_attempt_at_unix_secs), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: false, + reason: Some(marker.reason.clone()), + } +} + +fn marker_matches_revision(marker: &ScannerCycleRecoveryMarker, revision: &DataUsageCacheRevision) -> bool { + matches!(revision, DataUsageCacheRevision::Etag(etag) if marker.primary_revision == *etag) +} + +fn validate_recovery_marker(marker: &ScannerCycleRecoveryMarker) -> Result<(), &'static str> { + if marker.schema_version != SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION { + return Err("cycle recovery marker schema is unsupported"); + } + if marker.primary_revision.is_empty() { + return Err("cycle recovery marker has no primary revision"); + } + if marker.path != *DATA_USAGE_BLOOM_NAME_PATH { + return Err("cycle recovery marker path does not match the scanner scope"); + } + if marker.quarantine_path != *DATA_USAGE_BLOOM_RECOVERY_PATH { + return Err("cycle recovery marker quarantine path does not match the scanner scope"); + } + if !matches!(marker.classification.as_str(), "corrupt" | "future_schema") { + return Err("cycle recovery marker classification is invalid"); + } + if !matches!(marker.state.as_str(), "blocked" | "cleanup-pending") { + return Err("cycle recovery marker state is invalid"); + } + Ok(()) +} + +/// Decode only the stable scope and revision fields needed by an authenticated +/// full-rescan reset. Startup keeps the strict decoder above so a newer marker +/// cannot be interpreted as a trusted cursor; reset deliberately rebuilds from +/// the persisted usage floor instead. +pub(super) fn decode_recovery_marker_for_reset( + data: &[u8], + marker_revision: &DataUsageCacheRevision, +) -> Result { + if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) { + return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string())); + } + let compat = serde_json::from_slice::(data).ok(); + let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version); + let primary_revision = compat + .as_ref() + .and_then(|marker| marker.primary_revision.clone()) + .filter(|revision| !revision.is_empty()) + .unwrap_or_default(); + let path = compat + .as_ref() + .and_then(|marker| marker.path.clone()) + .unwrap_or_else(|| DATA_USAGE_BLOOM_NAME_PATH.clone()); + let quarantine_path = compat + .as_ref() + .and_then(|marker| marker.quarantine_path.clone()) + .unwrap_or_else(|| DATA_USAGE_BLOOM_RECOVERY_PATH.clone()); + if path != *DATA_USAGE_BLOOM_NAME_PATH || quarantine_path != *DATA_USAGE_BLOOM_RECOVERY_PATH { + return Err(ScannerError::Other( + "cycle recovery marker path does not match the scanner scope".to_string(), + )); + } + let classification = match compat.as_ref().and_then(|marker| marker.classification.as_deref()) { + Some("corrupt") => "corrupt", + Some("future_schema") | None => "future_schema", + Some(_) => "future_schema", + }; + let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) { + Some("cleanup-pending") => "cleanup-pending", + _ => "blocked", + }; + let now = unix_now_secs(); + Ok(ScannerCycleRecoveryMarker { + schema_version: SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION, + primary_revision, + // Cursor and epoch values from an unknown marker are audit-only data; + // the reset path intentionally rebuilds both from the verified usage + // floor instead of carrying them across a version boundary. + generation: 0, + leader_epoch: 0, + classification: classification.to_string(), + first_detected_at_unix_secs: compat + .as_ref() + .and_then(|marker| marker.first_detected_at_unix_secs) + .unwrap_or(now), + last_attempt_at_unix_secs: compat + .as_ref() + .and_then(|marker| marker.last_attempt_at_unix_secs) + .unwrap_or(now), + retry_count: compat.as_ref().and_then(|marker| marker.retry_count).unwrap_or(0), + reason: compat + .as_ref() + .and_then(|marker| marker.reason.clone()) + .unwrap_or_else(|| "operator requested full scanner rescan".to_string()), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: state.to_string(), + }) +} + +async fn read_cycle_state_body(reader: &mut ScannerGetObjectReader) -> Result, CycleStateBodyReadError> { + let max_len = usize::try_from(MAX_SCANNER_CYCLE_STATE_BYTES).unwrap_or(usize::MAX); + let mut data = Vec::new(); + reader + .take(MAX_SCANNER_CYCLE_STATE_BYTES.saturating_add(1)) + .read_to_end(&mut data) + .await + .map_err(|err| CycleStateBodyReadError::Backend(EcstoreError::other(err)))?; + if data.len() > max_len { + return Err(CycleStateBodyReadError::TooLarge); + } + Ok(data) +} + +fn cycle_state_classification(buf: &[u8]) -> (&'static str, &'static str) { + if buf.len() >= 16 && &buf[8..12] == b"RSCY" && &buf[8..16] != SCANNER_CYCLE_STATE_MAGIC { + ("future_schema", "scanner cycle state schema is newer than this reader") + } else { + ("corrupt", "scanner cycle state failed validation") + } +} + +fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) { + let generation = buf + .get(..8) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_le_bytes) + .unwrap_or(0); + let leader_epoch = if buf.len() >= SCANNER_CYCLE_STATE_HEADER_LEN && &buf[8..16] == SCANNER_CYCLE_STATE_MAGIC { + u64::from_le_bytes(buf[16..24].try_into().unwrap_or([0; 8])) + } else { + 0 + }; + (generation, leader_epoch) +} + +async fn persist_cycle_recovery_marker( + storeapi: Arc, + primary_revision: &DataUsageCacheRevision, + generation: u64, + leader_epoch: u64, + classification: &'static str, + reason: &'static str, +) -> Result { + let now = unix_now_secs(); + let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { + Ok(result) => result, + Err(err) => return Err(err), + }; + let existing_marker = existing + .as_deref() + .and_then(|bytes| serde_json::from_slice::(bytes).ok()); + let primary_revision = match primary_revision { + DataUsageCacheRevision::Etag(etag) => etag.clone(), + DataUsageCacheRevision::Missing => { + return Err(CycleRecoveryMarkerReadError::Invalid("cycle state recovery requires a primary revision")); + } + }; + let marker = ScannerCycleRecoveryMarker { + schema_version: SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION, + primary_revision: primary_revision.clone(), + generation, + leader_epoch, + classification: classification.to_string(), + first_detected_at_unix_secs: existing_marker + .as_ref() + .filter(|marker| marker.primary_revision == primary_revision) + .map(|marker| marker.first_detected_at_unix_secs) + .unwrap_or(now), + last_attempt_at_unix_secs: now, + retry_count: existing_marker + .as_ref() + .filter(|marker| marker.primary_revision == primary_revision) + .map(|marker| marker.retry_count.saturating_add(1)) + .unwrap_or(0), + reason: reason.to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "blocked".to_string(), + }; + let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?; + let save_result = save_config_with_preconditions( + storeapi.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + bytes, + existing_revision.preconditions(), + ) + .await; + match save_result { + Ok(_) => Ok(marker), + Err(EcstoreError::PreconditionFailed) => Err(CycleRecoveryMarkerReadError::Conflict), + Err(err) => Err(CycleRecoveryMarkerReadError::Backend(err)), + } +} + +async fn read_cycle_recovery_marker_bytes( + storeapi: Arc, +) -> Result<(Option>, DataUsageCacheRevision), CycleRecoveryMarkerReadError> { + let mut reader = match storeapi + .get_object_reader( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + None, + http::HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err( + EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_) + | EcstoreError::ConfigNotFound, + ) => { + return Ok((None, DataUsageCacheRevision::Missing)); + } + Err(err) => return Err(CycleRecoveryMarkerReadError::Backend(err)), + }; + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag) + .ok_or(CycleRecoveryMarkerReadError::Invalid("marker has no revision"))?; + if reader.object_info.is_dir || reader.object_info.size < 0 || reader.object_info.size > 64 * 1024 { + return Err(CycleRecoveryMarkerReadError::Invalid("marker exceeds the bounded object size")); + } + let mut data = Vec::new(); + (&mut reader) + .take(64 * 1024 + 1) + .read_to_end(&mut data) + .await + .map_err(|err| CycleRecoveryMarkerReadError::Backend(EcstoreError::other(err)))?; + if data.len() > 64 * 1024 { + return Err(CycleRecoveryMarkerReadError::Invalid("marker exceeds the bounded object size")); + } + if data.is_empty() { + return Err(CycleRecoveryMarkerReadError::Invalid("marker is empty")); + } + Ok((Some(data), revision)) +} + +async fn read_cycle_recovery_marker_revision( + storeapi: Arc, +) -> Result { + let reader = match storeapi + .get_object_reader( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + None, + http::HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err( + EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_) + | EcstoreError::ConfigNotFound, + ) => return Ok(DataUsageCacheRevision::Missing), + Err(err) => return Err(CycleRecoveryMarkerReadError::Backend(err)), + }; + if reader.object_info.is_dir || reader.object_info.size < 0 { + return Err(CycleRecoveryMarkerReadError::Invalid("marker is not a regular object")); + } + reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag) + .ok_or(CycleRecoveryMarkerReadError::Invalid("marker has no revision")) +} + +async fn quarantine_invalid_cycle_state( + storeapi: Arc, + revision: &DataUsageCacheRevision, + buf: &[u8], +) -> ScannerCycleStateStartup { + let (classification, reason) = cycle_state_classification(buf); + let (generation, leader_epoch) = cycle_state_generation_and_epoch(buf); + quarantine_invalid_cycle_state_with_reason(storeapi, revision, generation, leader_epoch, classification, reason).await +} + +async fn quarantine_invalid_cycle_state_with_reason( + storeapi: Arc, + revision: &DataUsageCacheRevision, + generation: u64, + leader_epoch: u64, + classification: &'static str, + reason: &'static str, +) -> ScannerCycleStateStartup { + let now = unix_now_secs(); + let base_status = ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "recovery-required".to_string(), + classification: Some(classification.to_string()), + primary_revision: match revision { + DataUsageCacheRevision::Etag(etag) => Some(etag.clone()), + DataUsageCacheRevision::Missing => None, + }, + generation: Some(generation), + leader_epoch: Some(leader_epoch), + first_detected_at_unix_secs: Some(now), + last_attempt_at_unix_secs: Some(now), + retry_count: 0, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: true, + reason: Some(reason.to_string()), + }; + set_scanner_cycle_recovery_status(base_status); + match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await { + Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")), + Err(CycleRecoveryMarkerReadError::Backend(_)) => { + // Keep the poison object untouched and retry marker creation with the + // bounded startup backoff; recovery-required never becomes healthy. + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "failed to persist scanner cycle recovery marker".to_string(), + )); + } + Err(CycleRecoveryMarkerReadError::Conflict) => { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker revision changed while publishing"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker revision changed while publishing".to_string(), + )); + } + Err(CycleRecoveryMarkerReadError::Invalid(reason)) => { + set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); + return ScannerCycleStateStartup::Blocked; + } + } + ScannerCycleStateStartup::Blocked +} + +async fn mark_cycle_recovery_cleanup_pending( + storeapi: Arc, + mut marker: ScannerCycleRecoveryMarker, + marker_revision: &DataUsageCacheRevision, +) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { + marker.state = "cleanup-pending".to_string(); + marker.last_attempt_at_unix_secs = unix_now_secs(); + let bytes = serde_json::to_vec(&marker) + .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; + let info = save_config_with_preconditions( + storeapi.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + bytes, + marker_revision.preconditions(), + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; + let revision = info + .etag + .filter(|etag| !etag.is_empty()) + .map(DataUsageCacheRevision::Etag) + .ok_or_else(|| ScannerError::Other("cycle recovery marker save returned no revision".to_string()))?; + Ok((marker, revision)) +} + +pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc) -> ScannerCycleStateStartup { + let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { + Ok((None, _)) => None, + Ok((Some(data), marker_revision)) => match serde_json::from_slice::(&data) { + Ok(marker) => match validate_recovery_marker(&marker) { + Ok(()) => Some((marker, marker_revision)), + Err(reason) => { + set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); + return ScannerCycleStateStartup::Blocked; + } + }, + Err(_) => { + set_scanner_cycle_recovery_status(recovery_status( + "recovery-required", + Some("cycle recovery marker is invalid"), + false, + )); + return ScannerCycleStateStartup::Blocked; + } + }, + Err(CycleRecoveryMarkerReadError::Backend(err)) => { + let status = recovery_status("transient", Some("cycle recovery marker I/O is temporarily unavailable"), true); + set_scanner_cycle_recovery_status(status); + return ScannerCycleStateStartup::Transient(ScannerError::Other(format!( + "failed to read scanner cycle recovery marker: {err}" + ))); + } + Err(CycleRecoveryMarkerReadError::Invalid(reason)) => { + set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); + return ScannerCycleStateStartup::Blocked; + } + Err(CycleRecoveryMarkerReadError::Conflict) => { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker revision changed while being inspected"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker revision changed while being inspected".to_string(), + )); + } + }; + + let mut reader = match storeapi + .get_object_reader( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + None, + http::HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err( + EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_) + | EcstoreError::ConfigNotFound, + ) => { + if let Some((marker, _)) = marker { + let state = if marker.state == "cleanup-pending" { + "cleanup-pending" + } else { + "recovery-required" + }; + set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, state)); + return ScannerCycleStateStartup::Blocked; + } + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + return ScannerCycleStateStartup::Ready { + cycle: CurrentCycle::default(), + leader_epoch: 0, + revision: DataUsageCacheRevision::Missing, + }; + } + Err(err) => { + set_scanner_cycle_recovery_status(recovery_status("transient", Some("cycle state could not be inspected"), true)); + return ScannerCycleStateStartup::Transient(ScannerError::Other(format!( + "failed to inspect scanner cycle state: {err}" + ))); + } + }; + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag); + let Some(revision) = revision else { + set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some("cycle state has no revision"), false)); + return ScannerCycleStateStartup::Blocked; + }; + let max_size = i64::try_from(MAX_SCANNER_CYCLE_STATE_BYTES).unwrap_or(i64::MAX); + if reader.object_info.is_dir || reader.object_info.size < 0 || reader.object_info.size > max_size { + return quarantine_invalid_cycle_state_with_reason( + storeapi, + &revision, + 0, + 0, + "corrupt", + "scanner cycle state object is oversized or not a regular object", + ) + .await; + } + if let Some((marker, _)) = marker + .as_ref() + .filter(|(marker, _)| marker.state == "cleanup-pending" || marker_matches_revision(marker, &revision)) + { + let state = if marker.state == "cleanup-pending" { + "cleanup-pending" + } else { + "blocked" + }; + set_scanner_cycle_recovery_status(recovery_status_from_marker(marker, state)); + return ScannerCycleStateStartup::Blocked; + } + let data = match read_cycle_state_body(&mut reader).await { + Ok(data) => data, + Err(CycleStateBodyReadError::TooLarge) => { + return quarantine_invalid_cycle_state_with_reason( + storeapi, + &revision, + 0, + 0, + "corrupt", + "scanner cycle state exceeds the bounded object size", + ) + .await; + } + Err(CycleStateBodyReadError::Backend(err)) => { + set_scanner_cycle_recovery_status(recovery_status("transient", Some("cycle state read failed"), true)); + return ScannerCycleStateStartup::Transient(ScannerError::Other(format!( + "failed to read scanner cycle state: {err}" + ))); + } + }; + if data.is_empty() { + return quarantine_invalid_cycle_state_with_reason( + storeapi, + &revision, + 0, + 0, + "corrupt", + "scanner cycle state object is empty", + ) + .await; + } + match decode_scanner_cycle_state_for_startup(&data) { + Ok((cycle, leader_epoch)) => { + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + ScannerCycleStateStartup::Ready { + cycle, + leader_epoch, + revision, + } + } + Err(_) => quarantine_invalid_cycle_state(storeapi, &revision, &data).await, + } +} + +/// Reset a blocked cycle state after an operator has explicitly requested a full +/// usage rebuild. The primary object is changed first with its observed ETag; +/// the recovery marker is removed only when its own ETag still matches. +pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc) -> Result<(), ScannerError> { + let lock = storeapi + .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock") + .await + .map_err(|err| ScannerError::Other(format!("failed to acquire scanner leader lock: {err}")))?; + let guard = lock + .get_write_lock_quiet(Duration::from_secs(5)) + .await + .map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?; + + if guard.is_lock_lost() { + return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); + } + + let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { + Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false), + Err(CycleRecoveryMarkerReadError::Invalid(_)) => { + let marker_revision = read_cycle_recovery_marker_revision(storeapi.clone()) + .await + .map_err(|err| ScannerError::Other(format!("failed to read cycle recovery marker: {err}")))?; + (Some(Vec::new()), marker_revision, true) + } + Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))), + }; + let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?; + let (marker, force_full_rescan) = match serde_json::from_slice::(&marker_data) { + Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false), + _ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true), + }; + let force_full_rescan = force_full_rescan || marker_body_invalid; + + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost while reading recovery state".to_string(), + )); + } + + let (mut primary_reader, primary_revision) = match storeapi + .get_object_reader( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + None, + http::HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => { + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .ok_or_else(|| ScannerError::Other("scanner cycle state has no revision".to_string()))?; + (Some(reader), DataUsageCacheRevision::Etag(revision)) + } + Err( + EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_) + | EcstoreError::ConfigNotFound, + ) => (None, DataUsageCacheRevision::Missing), + Err(err) => return Err(ScannerError::Other(format!("failed to inspect scanner cycle state: {err}"))), + }; + let marker_cleanup_pending = marker.state == "cleanup-pending"; + let marker_matches_primary = marker_matches_revision(&marker, &primary_revision); + if (marker_cleanup_pending || !marker_matches_primary) + && let Some(mut reader) = primary_reader.take() + { + // A newer, independently fenced primary is authoritative. A + // full-rescan reset must not overwrite that progress; it only + // removes the stale recovery marker after validating and re-fencing + // the state. + let max_size = i64::try_from(MAX_SCANNER_CYCLE_STATE_BYTES).unwrap_or(i64::MAX); + if reader.object_info.is_dir || reader.object_info.size < 0 { + return Err(ScannerError::Other("scanner cycle state changed since recovery was recorded".to_string())); + } + let primary_is_oversized = reader.object_info.size > max_size; + let primary_state = if primary_is_oversized { + None + } else { + match read_cycle_state_body(&mut reader).await { + Ok(data) if data.is_empty() => None, + Ok(data) => decode_scanner_cycle_state_for_startup(&data).ok(), + Err(CycleStateBodyReadError::TooLarge) if force_full_rescan || marker_cleanup_pending => None, + Err(err) => { + return Err(ScannerError::Other(format!( + "scanner cycle state changed since recovery was recorded: {err}" + ))); + } + } + }; + if let Some((primary_cycle, primary_epoch)) = primary_state { + let (cleanup_marker, cleanup_marker_revision) = + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?; + set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); + let usage_floor = persisted_usage_floor(storeapi.clone()).await?; + let fence_epoch = primary_epoch + .max(usage_floor.leader_epoch) + .checked_add(1) + .filter(|epoch| *epoch < u64::MAX) + .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?; + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost before preserving newer cycle state".to_string(), + )); + } + let preserved_data = encode_scanner_cycle_state(&primary_cycle, fence_epoch) + .map_err(|err| ScannerError::Other(format!("failed to encode preserved scanner cycle state: {err}")))?; + if u64::try_from(preserved_data.len()).unwrap_or(u64::MAX) > MAX_SCANNER_CYCLE_STATE_BYTES { + return Err(ScannerError::Other( + "preserved scanner cycle state exceeds the bounded object size".to_string(), + )); + } + let preserved_info = save_config_with_preconditions( + storeapi.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + preserved_data, + primary_revision.preconditions(), + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?; + let preserved_revision = preserved_info + .etag + .filter(|etag| !etag.is_empty()) + .map(DataUsageCacheRevision::Etag) + .ok_or_else(|| ScannerError::Other("preserved scanner cycle state has no revision".to_string()))?; + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost after fencing newer cycle state".to_string(), + )); + } + fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch) + .await + .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost after fencing newer cycle state".to_string(), + )); + } + let current_revision = read_config_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify preserved scanner cycle state: {err}")))?; + if current_revision != preserved_revision { + return Err(ScannerError::Other( + "scanner cycle state changed before recovery marker cleanup".to_string(), + )); + } + storeapi + .delete_config_object( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(cleanup_marker_revision.preconditions()), + ..Default::default() + }, + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?; + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + super::notify_scanner_cycle_recovery_wake(); + return Ok(()); + } else if !force_full_rescan && !marker_cleanup_pending { + // An invalid compatibility marker cannot fence a corrupt primary + // by revision, so rebuild it from the verified usage floor below. + // A strict marker keeps the existing fail-closed behavior for an + // unexpected stale-primary mutation. + return Err(ScannerError::Other("scanner cycle state changed since recovery was recorded".to_string())); + } + } + + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost before rebuilding cycle state".to_string(), + )); + } + + let floor = persisted_usage_floor(storeapi.clone()).await?; + // A full rescan must not trust a cursor recovered from a corrupt, future, + // or mixed-version marker. The durable usage floor is the only verified + // starting point; marker generation/epoch fields remain audit evidence. + let next = floor.next_cycle; + if next == u64::MAX { + return Err(ScannerError::Other("scanner cycle counter is exhausted".to_string())); + } + let leader_epoch = floor + .leader_epoch + .checked_add(1) + .filter(|epoch| *epoch < u64::MAX) + .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?; + let cycle = CurrentCycle { + next, + ..Default::default() + }; + let data = encode_scanner_cycle_state(&cycle, leader_epoch) + .map_err(|err| ScannerError::Other(format!("failed to encode rebuilt scanner cycle state: {err}")))?; + // Persist the cleanup-pending phase before rewriting the primary. If the + // process dies after the rewrite, startup still sees a durable fence and + // cannot mistake the partially completed reset for a healthy state. + let (marker, marker_revision) = if marker.state == "cleanup-pending" { + (marker, marker_revision) + } else { + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await? + }; + let rebuilt_info = save_config_with_preconditions( + storeapi.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + data, + primary_revision.preconditions(), + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?; + let rebuilt_revision = rebuilt_info + .etag + .filter(|etag| !etag.is_empty()) + .ok_or_else(|| ScannerError::Other("rebuilt scanner cycle state has no revision".to_string()))?; + if guard.is_lock_lost() { + return Err(ScannerError::Other( + "scanner leader lock was lost after rebuilding cycle state".to_string(), + )); + } + if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(rebuilt_revision.clone()), + generation: Some(next), + leader_epoch: Some(leader_epoch), + first_detected_at_unix_secs: Some(marker.first_detected_at_unix_secs), + last_attempt_at_unix_secs: Some(unix_now_secs()), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: false, + reason: Some("cycle state rebuilt but usage epoch fencing failed".to_string()), + }); + return Err(err); + } + + let current_revision = match read_config_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify rebuilt scanner cycle state: {err}")))? + { + DataUsageCacheRevision::Etag(etag) => etag, + DataUsageCacheRevision::Missing => { + return Err(ScannerError::Other("rebuilt scanner cycle state lost its revision".to_string())); + } + }; + if current_revision != rebuilt_revision { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(current_revision), + generation: Some(next), + leader_epoch: Some(leader_epoch), + first_detected_at_unix_secs: Some(marker.first_detected_at_unix_secs), + last_attempt_at_unix_secs: Some(unix_now_secs()), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: false, + reason: Some("rebuilt scanner cycle state changed before marker cleanup".to_string()), + }); + return Err(ScannerError::Other( + "rebuilt scanner cycle state changed before recovery marker cleanup".to_string(), + )); + } + + if guard.is_lock_lost() { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(rebuilt_revision.clone()), + generation: Some(next), + leader_epoch: Some(leader_epoch), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: false, + reason: Some("cycle state rebuilt but recovery marker was not cleared".to_string()), + ..Default::default() + }); + return Err(ScannerError::Other( + "scanner leader lock was lost before clearing recovery marker".to_string(), + )); + } + + if let Err(err) = storeapi + .delete_config_object( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(marker_revision.preconditions()), + ..Default::default() + }, + ) + .await + { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(rebuilt_revision.clone()), + generation: Some(next), + leader_epoch: Some(leader_epoch), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: false, + reason: Some("cycle state rebuilt but recovery marker cleanup failed".to_string()), + ..Default::default() + }); + return Err(ScannerError::Other(format!("failed to clear cycle recovery marker: {err}"))); + } + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "healthy".to_string(), + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + ..Default::default() + }); + super::notify_scanner_cycle_recovery_wake(); + Ok(()) +} #[derive(Debug, thiserror::Error)] pub(super) enum ScannerCycleStateError { @@ -86,7 +1147,11 @@ pub(super) fn decode_scanner_cycle_state(buf: &[u8]) -> Result<(CurrentCycle, u6 (0, &buf[8..]) }; - let cycle_info = rmp_serde::from_slice::(payload)?; + let mut deserializer = rmp_serde::Deserializer::new(std::io::Cursor::new(payload)); + let cycle_info = CurrentCycle::deserialize(&mut deserializer)?; + if deserializer.position() != u64::try_from(payload.len()).unwrap_or(u64::MAX) { + return Err(ScannerCycleStateError::InvalidData("scanner cycle state has trailing bytes")); + } if cycle_info.next != persisted_next { return Err(ScannerCycleStateError::InvalidData("scanner cycle counter disagrees with encoded state")); } @@ -146,7 +1211,7 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), pub(super) async fn persisted_usage_floor(storeapi: Arc) -> Result { let mut floor = PersistedUsageFloor::default(); - let update_floor = |floor: &mut PersistedUsageFloor, usage: DataUsageInfo, path: &str| -> Result<(), ScannerError> { + let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> { floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); if let Some(completed_cycle) = usage.scanner_cycle { let next_cycle = completed_cycle @@ -159,25 +1224,45 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - }; for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] { let backup_path = format!("{primary_path}.bkp"); - let mut pair_found = false; - for path in [primary_path, backup_path.as_str()] { - let data = match read_config(storeapi.clone(), path).await { - Ok(data) => { - pair_found = true; - data + let primary_epoch = match read_config(storeapi.clone(), primary_path).await { + Ok(data) => { + let usage = serde_json::from_slice::(&data).map_err(|err| { + ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) + })?; + let epoch = usage.scanner_epoch.unwrap_or_default(); + update_floor(&mut floor, &usage, primary_path)?; + Some(epoch) + } + Err(EcstoreError::ConfigNotFound) => None, + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read scanner usage epoch floor from {primary_path}: {err}" + ))); + } + }; + let mut any_found = primary_epoch.is_some(); + match read_config(storeapi.clone(), &backup_path).await { + Ok(data) => { + any_found = true; + let usage = serde_json::from_slice::(&data).map_err(|err| { + ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) + })?; + let backup_epoch = usage.scanner_epoch.unwrap_or_default(); + // A backup write from an older leader may complete after the + // primary epoch has been fenced. It must not advance the startup + // floor unless its epoch is at least as new as the primary. + if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch) { + update_floor(&mut floor, &usage, &backup_path)?; } - Err(EcstoreError::ConfigNotFound) => continue, - Err(err) => { - return Err(ScannerError::Other(format!( - "failed to read scanner usage epoch floor from {path}: {err}" - ))); - } - }; - let usage = serde_json::from_slice::(&data) - .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage floor from {path}: {err}")))?; - update_floor(&mut floor, usage, path)?; + } + Err(EcstoreError::ConfigNotFound) => {} + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read scanner usage epoch floor from {backup_path}: {err}" + ))); + } } - if pair_found { + if any_found { break; } } diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index 0ac948549..ab22f56d9 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -196,7 +196,7 @@ pub(super) async fn claim_scanner_leadership( if ctx.is_cancelled() { return false; } - let Some(claimed_epoch) = persisted_epoch.checked_add(1) else { + let Some(claimed_epoch) = persisted_epoch.checked_add(1).filter(|epoch| *epoch < u64::MAX) else { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index c40f91849..5430e9d29 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -15,11 +15,12 @@ use super::*; use crate::EcstoreResult; use crate::{ - Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader, - ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, - init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, + DATA_USAGE_BLOOM_RECOVERY_PATH, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, + ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, + ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, + init_local_disks_with_instance_ctx, }; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::task::Poll; use temp_env::{with_var, with_var_unset}; @@ -117,6 +118,15 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { assert!(cycle_ctx.is_cancelled()); } +#[tokio::test] +async fn scanner_cycle_recovery_wake_survives_wait_registration_race() { + notify_scanner_cycle_recovery_wake(); + + tokio::time::timeout(Duration::from_secs(1), SCANNER_CYCLE_RECOVERY_WAKE.notified()) + .await + .expect("recovery wake should retain a permit until the waiter registers"); +} + struct ScannerDefaultSpeedGuard; impl ScannerDefaultSpeedGuard { @@ -151,6 +161,7 @@ impl Drop for ScannerDefaultCycleGuard { struct MemoryConfigStore { objects: Mutex>>, revisions: Mutex>, + non_regular_objects: Mutex>, fail_put_number: Mutex>, object_not_found_put_number: Mutex>, error_after_commit_put_number: Mutex>, @@ -191,12 +202,16 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore { .get(&key) .cloned() .ok_or(EcstoreError::FileNotFound)?; - let revision = *self.revisions.lock().await.entry(key).or_insert(1); + let data_len = i64::try_from(data.len()).expect("memory test object length should fit in i64"); + let revision = *self.revisions.lock().await.entry(key.clone()).or_insert(1); + let is_dir = self.non_regular_objects.lock().await.contains(&key); Ok(GetObjectReader { stream: Box::new(Cursor::new(data)), object_info: ObjectInfo { etag: Some(format!("memory-{revision}")), + size: data_len, + is_dir, ..Default::default() }, buffered_body: None, @@ -797,6 +812,10 @@ fn scanner_cycle_state_decodes_legacy_and_fenced_formats() { let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode"); assert_eq!(fenced_cycle.next, 13); assert_eq!(fenced_epoch, 7); + + let mut trailing = fenced; + trailing.push(0); + assert!(decode_scanner_cycle_state(&trailing).is_err()); } #[test] @@ -823,6 +842,840 @@ fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() { assert!(encode_scanner_cycle_state(&exhausted, 7).is_err()); } +#[tokio::test] +async fn corrupt_cycle_state_is_quarantined_once() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.objects.lock().await.insert(state_key.clone(), vec![1]); + store.revisions.lock().await.insert(state_key.clone(), 7); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store.clone()).await, + ScannerCycleStateStartup::Blocked + )); + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + let marker_data = store + .objects + .lock() + .await + .get(&marker_key) + .cloned() + .expect("corrupt state must leave a durable recovery marker"); + let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should be valid JSON"); + assert_eq!(marker.primary_revision, "memory-7"); + assert_eq!(marker.path, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + assert_eq!(marker.quarantine_path, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + assert_eq!(marker.classification, "corrupt"); + + // A second startup sees the matching marker before consuming the poison body. + assert!(matches!( + load_scanner_cycle_state_for_startup(store.clone()).await, + ScannerCycleStateStartup::Blocked + )); + + // Replacing the primary object advances its revision; the stale marker must + // not quarantine the newer, valid state. + let cycle = CurrentCycle { + next: 9, + ..Default::default() + }; + let encoded = encode_scanner_cycle_state(&cycle, 3).expect("valid state should encode"); + store.objects.lock().await.insert(state_key.clone(), encoded); + store.revisions.lock().await.insert(state_key, 8); + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Ready { + cycle: CurrentCycle { next: 9, .. }, + leader_epoch: 3, + .. + } + )); +} + +#[tokio::test] +async fn empty_cycle_state_object_is_quarantined_as_corrupt() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.objects.lock().await.insert(state_key.clone(), Vec::new()); + store.revisions.lock().await.insert(state_key, 6); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Blocked + )); + assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt")); + assert!( + scanner_cycle_recovery_status() + .reason + .as_deref() + .is_some_and(|reason| reason.contains("empty")) + ); +} + +#[tokio::test] +async fn future_cycle_state_schema_is_recovery_required() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let mut future = 17_u64.to_le_bytes().to_vec(); + future.extend_from_slice(b"RSCYC999"); + future.extend_from_slice(&4_u64.to_le_bytes()); + future.extend_from_slice(&[0x90]); + store.objects.lock().await.insert(state_key.clone(), future); + store.revisions.lock().await.insert(state_key, 13); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Blocked + )); + assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("future_schema")); +} + +#[tokio::test] +async fn concurrent_leaders_cannot_quarantine_newer_cycle_state() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.objects.lock().await.insert(state_key.clone(), vec![1]); + store.revisions.lock().await.insert(state_key, 4); + + let (first, second) = tokio::join!( + load_scanner_cycle_state_for_startup(store.clone()), + load_scanner_cycle_state_for_startup(store.clone()), + ); + assert!(matches!(first, ScannerCycleStateStartup::Blocked)); + assert!(matches!(second, ScannerCycleStateStartup::Blocked)); + + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + let marker_data = store + .objects + .lock() + .await + .get(&marker_key) + .cloned() + .expect("one contender must publish the recovery marker"); + let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&marker_data).expect("marker should decode"); + assert_eq!(marker.primary_revision, "memory-4"); +} + +#[tokio::test] +async fn cleanup_pending_marker_blocks_a_rewritten_primary_after_restart() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + let encoded = encode_scanner_cycle_state( + &CurrentCycle { + next: 12, + ..Default::default() + }, + 8, + ) + .expect("valid state should encode"); + store.objects.lock().await.insert(state_key.clone(), encoded); + store.revisions.lock().await.insert(state_key, 22); + let marker = ScannerCycleRecoveryMarker { + schema_version: 1, + primary_revision: "memory-21".to_string(), + generation: 11, + leader_epoch: 7, + classification: "corrupt".to_string(), + first_detected_at_unix_secs: 1, + last_attempt_at_unix_secs: 2, + retry_count: 1, + reason: "reset in progress".to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "cleanup-pending".to_string(), + }; + store + .objects + .lock() + .await + .insert(marker_key.clone(), serde_json::to_vec(&marker).expect("marker should encode")); + store.revisions.lock().await.insert(marker_key, 3); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Blocked + )); + assert_eq!(scanner_cycle_recovery_status().state, "cleanup-pending"); +} + +#[test] +fn full_rescan_reset_accepts_unknown_marker_fields_without_trusting_cursor() { + let marker = br#"{ + "schema_version": 99, + "primary_revision": "memory-7", + "generation": 9000, + "leader_epoch": 9000, + "classification": "new-future-classification", + "first_detected_at_unix_secs": 1, + "last_attempt_at_unix_secs": 2, + "retry_count": 9, + "reason": "future marker", + "path": "buckets/.bloomcycle.bin", + "quarantine_path": "buckets/.bloomcycle.bin.recovery-required.json", + "future_field": {"cursor": "untrusted"} + }"#; + let decoded = + super::cycle_state::decode_recovery_marker_for_reset(marker, &DataUsageCacheRevision::Etag("memory-3".to_string())) + .expect("full-rescan compatibility decoder should accept additive fields"); + assert_eq!(decoded.primary_revision, "memory-7"); + assert_eq!(decoded.classification, "future_schema"); + assert_eq!(decoded.generation, 0); + assert_eq!(decoded.leader_epoch, 0); + assert_eq!(decoded.state, "blocked"); + + let malformed = + super::cycle_state::decode_recovery_marker_for_reset(b"{not-json", &DataUsageCacheRevision::Etag("memory-4".to_string())) + .expect("a full-rescan reset must recover even when the marker is malformed"); + assert!(malformed.primary_revision.is_empty()); + assert_eq!(malformed.classification, "future_schema"); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_after_malformed_marker_without_trusting_cursor() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) + .await + .expect("corrupt cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec()) + .await + .expect("malformed marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("full-rescan reset should recover malformed marker"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(cycle.next, 0, "reset must use the verified usage floor, not marker cursor"); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_ignores_epoch_from_malformed_future_primary() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let mut future_primary = vec![0; 24]; + future_primary[8..16].copy_from_slice(b"RSCY9999"); + future_primary[16..24].copy_from_slice(&u64::MAX.to_le_bytes()); + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), future_primary) + .await + .expect("future cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), br#"{not-json"#.to_vec()) + .await + .expect("malformed marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("full-rescan reset should recover malformed future state"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(leader_epoch, 1, "invalid persisted bytes must not raise the recovery epoch"); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn ecstore_exact_recovery_marker_delete_honors_etag() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v1".to_vec()) + .await + .expect("initial recovery marker should be persisted"); + let (_, stale_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("initial marker revision should load"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"marker-v2".to_vec()) + .await + .expect("replacement recovery marker should be persisted"); + + let delete_result = store + .delete_config_object( + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ObjectOptions { + http_preconditions: Some(stale_revision.preconditions()), + ..Default::default() + }, + ) + .await; + assert!(matches!(delete_result, Err(EcstoreError::PreconditionFailed))); + assert_eq!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("replacement marker should remain durable"), + b"marker-v2" + ); +} + +#[tokio::test] +async fn full_rescan_reset_rejects_corrupt_primary_under_stale_blocked_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let corrupt_primary = vec![0xff, 0x00, 0x01]; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), corrupt_primary.clone()) + .await + .expect("corrupt cycle state should be persisted"); + let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary revision should load"); + let marker = ScannerCycleRecoveryMarker { + schema_version: 1, + primary_revision: "memory-stale".to_string(), + generation: 1, + leader_epoch: 1, + classification: "corrupt".to_string(), + first_detected_at_unix_secs: 1, + last_attempt_at_unix_secs: 2, + retry_count: 1, + reason: "blocked primary changed".to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "blocked".to_string(), + }; + let marker_data = serde_json::to_vec(&marker).expect("blocked marker should encode"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), marker_data.clone()) + .await + .expect("blocked marker should be persisted"); + + assert!( + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .is_err(), + "a strict marker must fail closed when its primary revision changed" + ); + assert_eq!( + read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary should remain readable"), + corrupt_primary + ); + assert_eq!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("blocked marker should remain durable"), + marker_data + ); + assert!(!matches!(primary_revision, DataUsageCacheRevision::Missing)); +} + +#[tokio::test] +async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let primary = CurrentCycle { + next: 42, + ..Default::default() + }; + let old_primary_data = encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode"); + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), old_primary_data.clone()) + .await + .expect("valid cycle state should be persisted"); + let (_, old_primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary state revision should load"); + let old_usage = DataUsageInfo { + scanner_epoch: Some(7), + scanner_cycle: Some(41), + ..Default::default() + }; + let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone()) + .await + .expect("usage snapshot should be persisted"); + let (_, old_usage_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("usage snapshot revision should load"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("reset should clear a stale malformed marker"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("valid primary should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("primary cycle state should decode"); + assert_eq!(cycle.next, 42, "reset must not regress an independently fenced primary"); + assert_eq!(leader_epoch, 8, "reset must advance the preserved primary epoch"); + let stale_primary_save = save_config_with_preconditions( + store.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + old_primary_data, + old_primary_revision.preconditions(), + ) + .await; + assert!(matches!(stale_primary_save, Err(EcstoreError::PreconditionFailed))); + let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("usage epoch fence should remain durable"); + assert_eq!( + serde_json::from_slice::(&usage) + .expect("fenced usage should decode") + .scanner_epoch, + Some(8) + ); + let stale_save = save_config_with_preconditions( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + old_usage_data, + old_usage_revision.preconditions(), + ) + .await; + assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed))); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let completed_at = Utc::now(); + let primary = CurrentCycle { + current: 3, + next: 42, + cycle_completed: vec![completed_at], + started: completed_at, + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + encode_scanner_cycle_state(&primary, 7).expect("valid cycle state should encode"), + ) + .await + .expect("valid cycle state should be persisted"); + let usage = DataUsageInfo { + scanner_epoch: Some(7), + scanner_cycle: Some(41), + ..Default::default() + }; + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&usage).expect("usage snapshot should encode"), + ) + .await + .expect("usage snapshot should be persisted"); + let marker = ScannerCycleRecoveryMarker { + schema_version: 1, + primary_revision: "memory-old".to_string(), + generation: 41, + leader_epoch: 7, + classification: "corrupt".to_string(), + first_detected_at_unix_secs: 1, + last_attempt_at_unix_secs: 2, + retry_count: 1, + reason: "reset in progress".to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "cleanup-pending".to_string(), + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + serde_json::to_vec(&marker).expect("marker should encode"), + ) + .await + .expect("cleanup marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("reset should resume a cleanup-pending preserved primary"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("preserved cycle state should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("cycle state should decode"); + assert_eq!(cycle.current, 3, "cleanup retry must preserve the in-progress cursor"); + assert_eq!(cycle.next, 42); + assert_eq!(cycle.cycle_completed, vec![completed_at]); + assert_eq!(cycle.started, completed_at); + assert_eq!(leader_epoch, 8); + let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("usage epoch fence should remain durable"); + assert_eq!( + serde_json::from_slice::(&usage) + .expect("usage should decode") + .scanner_epoch, + Some(8) + ); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_oversized_regular_primary_with_malformed_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1]) + .await + .expect("oversized cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("explicit full-rescan reset should replace an oversized regular primary"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(cycle.next, 0); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_oversized_primary_after_cleanup_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1]) + .await + .expect("oversized cycle state should be persisted"); + let (_, primary_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary revision should load"); + let marker = ScannerCycleRecoveryMarker { + schema_version: 1, + primary_revision: match primary_revision { + DataUsageCacheRevision::Etag(etag) => etag, + DataUsageCacheRevision::Missing => panic!("primary revision should be present"), + }, + generation: 1, + leader_epoch: 1, + classification: "corrupt".to_string(), + first_detected_at_unix_secs: 1, + last_attempt_at_unix_secs: 2, + retry_count: 1, + reason: "reset in progress".to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "cleanup-pending".to_string(), + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + serde_json::to_vec(&marker).expect("cleanup marker should encode"), + ) + .await + .expect("cleanup marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("cleanup retry should rebuild an oversized primary"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(cycle.next, 0); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_with_oversized_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) + .await + .expect("corrupt cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), vec![b'x'; 64 * 1024 + 1]) + .await + .expect("oversized recovery marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("full-rescan reset should recover an oversized marker"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_with_empty_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) + .await + .expect("corrupt cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), Vec::new()) + .await + .expect("empty recovery marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("full-rescan reset should recover an empty marker"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (_, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_keeps_cleanup_marker_when_preserved_epoch_is_exhausted() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let primary = CurrentCycle { + next: 42, + ..Default::default() + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + encode_scanner_cycle_state(&primary, u64::MAX).expect("valid cycle state should encode"), + ) + .await + .expect("valid cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + assert!( + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .is_err() + ); + + let marker = read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("cleanup marker should remain durable"); + assert_eq!( + serde_json::from_slice::(&marker) + .expect("cleanup marker should decode") + .state, + "cleanup-pending" + ); + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Blocked + )); +} + +#[tokio::test] +async fn full_rescan_reset_rejects_preserved_epoch_that_would_be_terminal() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let primary = CurrentCycle { + next: 42, + ..Default::default() + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + encode_scanner_cycle_state(&primary, u64::MAX - 1).expect("valid cycle state should encode"), + ) + .await + .expect("valid cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + assert!( + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .is_err(), + "reset must not persist the terminal leader epoch" + ); + + let marker = read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("cleanup marker should remain durable"); + assert_eq!( + serde_json::from_slice::(&marker) + .expect("cleanup marker should decode") + .state, + "cleanup-pending" + ); +} + +#[tokio::test] +async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) + .await + .expect("corrupt cycle state should be persisted"); + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&DataUsageInfo { + scanner_epoch: Some(u64::MAX - 1), + ..Default::default() + }) + .expect("usage floor should encode"), + ) + .await + .expect("usage floor should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + assert!( + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .is_err(), + "reset must not persist the terminal leader epoch" + ); + assert_eq!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("recovery marker should remain durable"), + b"{not-json" + ); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_empty_primary_with_malformed_marker() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), Vec::new()) + .await + .expect("empty cycle state should be persisted"); + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{not-json".to_vec()) + .await + .expect("malformed marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("explicit full-rescan reset should replace an empty primary"); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt cycle state should remain durable"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(cycle.next, 0); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn full_rescan_reset_rebuilds_when_primary_cycle_state_is_missing() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let marker = ScannerCycleRecoveryMarker { + schema_version: 1, + primary_revision: "memory-missing".to_string(), + generation: u64::MAX, + leader_epoch: u64::MAX, + classification: "corrupt".to_string(), + first_detected_at_unix_secs: 1, + last_attempt_at_unix_secs: 2, + retry_count: 0, + reason: "missing primary".to_string(), + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(), + state: "blocked".to_string(), + }; + save_config( + store.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + serde_json::to_vec(&marker).expect("marker should encode"), + ) + .await + .expect("marker should be persisted"); + + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("full-rescan reset should recreate missing primary"); + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("missing primary should be rebuilt"); + let (cycle, leader_epoch) = decode_scanner_cycle_state(&state).expect("rebuilt cycle state should decode"); + assert_eq!(cycle.next, 0); + assert_eq!(leader_epoch, 1); + assert!(matches!( + read_config(store, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn corrupt_cycle_state_rename_or_marker_failure_stays_recovery_required() { + let store = Arc::new(MemoryConfigStore::default()); + let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + store.objects.lock().await.insert(state_key.clone(), vec![1]); + store.revisions.lock().await.insert(state_key, 9); + store.fail_put_number.lock().await.insert(marker_key, 1); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store.clone()).await, + ScannerCycleStateStartup::Transient(_) + )); + let status = scanner_cycle_recovery_status(); + assert_eq!(status.state, "recovery-required"); + assert!(status.retryable); + assert!( + store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str())) + ); +} + +#[tokio::test] +async fn oversized_or_symlinked_cycle_state_is_rejected() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.objects.lock().await.insert(key.clone(), vec![0; 1024 * 1024 + 1]); + store.revisions.lock().await.insert(key.clone(), 11); + + assert!(matches!( + load_scanner_cycle_state_for_startup(store.clone()).await, + ScannerCycleStateStartup::Blocked + )); + assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("corrupt")); + assert!( + scanner_cycle_recovery_status() + .reason + .as_deref() + .is_some_and(|reason| reason.contains("oversized")) + ); + + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()); + store.objects.lock().await.remove(&marker_key); + store.objects.lock().await.insert(key.clone(), vec![1]); + store.revisions.lock().await.insert(key.clone(), 12); + store.non_regular_objects.lock().await.insert(key); + // The object contract exposes a non-regular object as `is_dir`; local + // backends reject symlink/reparse entries before they become an object. + assert!(matches!( + load_scanner_cycle_state_for_startup(store).await, + ScannerCycleStateStartup::Blocked + )); +} + #[tokio::test] async fn scanner_startup_uses_primary_and_backup_usage_floor() { let store = Arc::new(MemoryConfigStore::default()); @@ -855,6 +1708,31 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() { assert_eq!(epoch, 11); } +#[tokio::test] +async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() { + let store = Arc::new(MemoryConfigStore::default()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] { + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, path), + serde_json::to_vec(&DataUsageInfo { + scanner_epoch: Some(epoch), + scanner_cycle: Some(cycle), + ..Default::default() + }) + .expect("usage snapshot should encode"), + ); + } + + assert_eq!( + persisted_usage_floor(store).await.expect("usage floor should load"), + PersistedUsageFloor { + next_cycle: 101, + leader_epoch: 8, + } + ); +} + #[test] fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() { let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1); @@ -987,6 +1865,15 @@ async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() assert!(persisted_usage_floor(store.clone()).await.is_err()); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + br#"{}"#.to_vec(), + ); + assert!( + persisted_usage_floor(store.clone()).await.is_err(), + "a structurally incomplete usage snapshot must not be treated as an empty floor" + ); + store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), serde_json::to_vec(&DataUsageInfo { @@ -1244,6 +2131,22 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf assert_eq!(store.put_counts.lock().await.get(&key), Some(&3)); } +#[tokio::test] +async fn test_leadership_claim_rejects_terminal_epoch() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = u64::MAX - 1; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); + assert_eq!(persisted_epoch, u64::MAX - 1); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); +} + #[tokio::test] async fn test_leadership_claim_confirms_commit_after_returned_error() { let store = Arc::new(MemoryConfigStore::default()); @@ -2975,6 +3878,24 @@ fn superseded_retry_backoff_grows_from_the_default_cycle() { } } +#[tokio::test(start_paused = true)] +async fn corrupt_cycle_state_backoff_uses_virtual_clock() { + let mut backoff = ScannerRetryBackoff::default(); + backoff.record_retryable_cycle(true); + let first_delay = backoff + .retry_interval(Duration::from_secs(60)) + .expect("the first recovery retry should be scheduled"); + assert_eq!(first_delay, Duration::from_secs(5)); + + let deadline = Instant::now() + first_delay; + assert!(Instant::now() < deadline); + tokio::time::advance(first_delay).await; + assert!(Instant::now() >= deadline); + + backoff.record_retryable_cycle(true); + assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(10))); +} + #[test] fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() { let runtime_config = ScannerRuntimeConfig { diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index f0a32f402..6822ccaa4 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -126,6 +126,7 @@ mod tests { let _list_remote_target_handler = replication::ListRemoteTargetHandler {}; let _remove_remote_target_handler = replication::RemoveRemoteTargetHandler {}; let _scanner_status_handler = scanner::ScannerStatusHandler {}; + let _scanner_cycle_state_reset_handler = scanner::ScannerCycleStateResetHandler {}; let _ilm_expiry_status_handler = scanner::IlmExpiryStatusHandler {}; let _manual_transition_handler = ilm_transition::ManualTransitionRunHandler {}; let _manual_transition_status_handler = ilm_transition::ManualTransitionJobStatusHandler {}; diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index ad500004d..fa8df2a69 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -13,8 +13,11 @@ // limitations under the License. use crate::admin::auth::authorize_admin_request; +use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::runtime_sources::current_scanner_metrics_report; +use crate::admin::runtime_sources::{ + app_context_from_req, current_object_store_handle_for_context, current_scanner_metrics_report, +}; use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env}; use crate::server::ADMIN_PREFIX; use chrono::Utc; @@ -22,11 +25,13 @@ use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; use rustfs_common::metrics::{ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport}; +use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use tokio_util::sync::CancellationToken; const JSON_CONTENT_TYPE: &str = "application/json"; @@ -38,6 +43,13 @@ struct ScannerStatusResponse { metrics: ScannerMetricsReport, cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, + cycle_recovery: rustfs_scanner::ScannerCycleRecoveryStatus, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ScannerCycleResetRequest { + mode: String, } #[derive(Debug, Serialize)] @@ -117,6 +129,7 @@ fn scanner_status_response( metrics, cycle_schedule, runtime_config, + cycle_recovery: rustfs_scanner::scanner::scanner_cycle_recovery_status(), } } @@ -144,6 +157,11 @@ pub fn register_scanner_route(r: &mut S3Router) -> std::io::Resu format!("{ADMIN_PREFIX}/v3/scanner/status").as_str(), AdminOperation(&ScannerStatusHandler {}), )?; + r.insert( + Method::POST, + format!("{ADMIN_PREFIX}/v3/scanner/cycle-state/reset").as_str(), + AdminOperation(&ScannerCycleStateResetHandler {}), + )?; r.insert( Method::GET, format!("{ADMIN_PREFIX}/v3/ilm/expiry/status").as_str(), @@ -163,6 +181,13 @@ async fn validate_scanner_status_request(req: &S3Request) -> S3Result) -> S3Result { + if req.credentials.is_none() { + return Err(s3_error!(InvalidRequest, "missing credentials")); + } + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)]).await +} + fn json_response(body: Vec) -> S3Result> { let mut headers = HeaderMap::new(); let content_type = HeaderValue::from_str(JSON_CONTENT_TYPE) @@ -192,6 +217,37 @@ impl Operation for ScannerStatusHandler { pub struct IlmExpiryStatusHandler {} +pub struct ScannerCycleStateResetHandler {} + +#[async_trait::async_trait] +impl Operation for ScannerCycleStateResetHandler { + async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let _cred = validate_scanner_reset_request(&req).await?; + let body = req + .input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?; + let reset = serde_json::from_slice::(&body) + .map_err(|err| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid reset request body: {err}")))?; + if reset.mode != "full-rescan" { + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "reset mode must be full-rescan")); + } + let context = app_context_from_req(&req) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?; + let store = current_object_store_handle_for_context(Some(context.as_ref())) + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "storage layer not initialized"))?; + supervise_admin_mutation("scanner cycle state reset", async move { + rustfs_scanner::scanner::reset_scanner_cycle_recovery(CancellationToken::new(), store) + .await + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?; + Ok::<_, S3Error>(()) + }) + .await?; + json_response(br#"{"status":"reset","mode":"full-rescan"}"#.to_vec()) + } +} + #[async_trait::async_trait] impl Operation for IlmExpiryStatusHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -237,6 +293,38 @@ mod tests { assert_eq!(err.message(), Some("missing credentials")); } + #[tokio::test] + async fn scanner_reset_gate_rejects_missing_credentials() { + let req = S3Request { + input: Body::from(String::new()), + method: Method::POST, + uri: http::Uri::from_static("/rustfs/admin/v3/scanner/cycle-state/reset"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = validate_scanner_reset_request(&req) + .await + .expect_err("a reset request without credentials must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("missing credentials")); + } + + #[test] + fn admin_reset_requires_full_rescan_or_verified_cursor() { + let full_rescan: ScannerCycleResetRequest = + serde_json::from_str(r#"{"mode":"full-rescan"}"#).expect("full rescan must be accepted"); + assert_eq!(full_rescan.mode, "full-rescan"); + let cursor: ScannerCycleResetRequest = + serde_json::from_str(r#"{"mode":"cursor"}"#).expect("mode validation belongs to the handler"); + assert_ne!(cursor.mode, "full-rescan"); + assert!(serde_json::from_str::(r#"{"mode":"full-rescan","cursor":"untrusted"}"#).is_err()); + } + #[test] fn scanner_disabled_reason_reports_startup_env_key() { assert_eq!(scanner_disabled_reason(true), None); @@ -304,6 +392,11 @@ mod tests { assert_eq!(encoded["cycle_schedule"]["effective_interval_seconds"], 0); assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_enabled"], false); assert_eq!(encoded["cycle_schedule"]["clean_idle_backoff_multiplier"], 1); + assert_eq!(encoded["cycle_recovery"]["state"], "healthy"); + assert_eq!( + encoded["cycle_recovery"]["quarantine_path"], + rustfs_scanner::DATA_USAGE_BLOOM_RECOVERY_PATH.as_str() + ); } #[test] diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index ccb013211..e423c0b0e 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -428,6 +428,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/scanner/cycle-state/reset", + CONFIG_UPDATE, + RouteRiskLevel::High, + ), admin( HttpMethod::Get, "/rustfs/admin/v3/ilm/expiry/status", @@ -2020,6 +2026,12 @@ mod tests { assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/expiry/status", SET_TIER); } + #[test] + fn route_policy_requires_config_update_for_scanner_cycle_reset() { + assert_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", CONFIG_UPDATE); + assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/scanner/cycle-state/reset", SERVER_INFO); + } + #[test] fn route_policy_uses_tier_actions_for_transition_routes() { assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER); diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index e48e09c94..8dcb5d901 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -243,6 +243,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::GET, "/v3/config"), admin_route(Method::PUT, "/v3/config"), admin_route(Method::GET, "/v3/scanner/status"), + admin_route(Method::POST, "/v3/scanner/cycle-state/reset"), admin_route(Method::GET, "/v3/audit/target/list"), admin_route_sample( Method::PUT, @@ -879,6 +880,7 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::GET, &admin_path("/v3/config")); assert_route(&router, Method::PUT, &admin_path("/v3/config")); assert_route(&router, Method::GET, &admin_path("/v3/scanner/status")); + assert_route(&router, Method::POST, &admin_path("/v3/scanner/cycle-state/reset")); assert_route(&router, Method::GET, &admin_path("/v3/ilm/expiry/status")); assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/run")); assert_route( @@ -1367,6 +1369,7 @@ fn test_admin_alias_paths_match_existing_admin_routes() { (Method::GET, compat_admin_alias_path("/v3/config")), (Method::PUT, compat_admin_alias_path("/v3/config")), (Method::GET, compat_admin_alias_path("/v3/scanner/status")), + (Method::POST, compat_admin_alias_path("/v3/scanner/cycle-state/reset")), (Method::GET, compat_admin_alias_path("/v3/ilm/expiry/status")), ] { assert!( From f9d45e41e159dd42de70137ad4ccf37abd525cbb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 22 Aug 2026 19:39:01 +0800 Subject: [PATCH 04/32] fix(quota): account compressed deletes by committed size (#6365) --- crates/ecstore/src/api/mod.rs | 4 +- .../bucket/replication/replication_pool.rs | 2 +- .../replication/replication_resyncer.rs | 2 +- .../replication_target_boundary.rs | 16 +- crates/ecstore/src/core/sets.rs | 123 ++++---- crates/ecstore/src/data_usage/mod.rs | 116 +++++++- crates/ecstore/src/object_api/types.rs | 38 ++- crates/ecstore/src/set_disk/mod.rs | 2 +- crates/ecstore/src/set_disk/ops/object.rs | 163 ++++++++++- .../ecstore/src/storage_api_contracts/mod.rs | 4 +- crates/ecstore/src/store/object.rs | 65 ++++- crates/storage-api/src/lib.rs | 1 + crates/storage-api/src/object.rs | 24 ++ rustfs/src/app/bucket_usecase.rs | 2 +- rustfs/src/app/object_usecase.rs | 271 +++++++++++++++--- rustfs/src/app/storage_api.rs | 10 +- rustfs/src/storage/s3_api/bucket.rs | 44 ++- rustfs/src/storage/storage_api.rs | 2 + 18 files changed, 754 insertions(+), 135 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 17fffac3d..456d38ac7 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -317,8 +317,6 @@ pub mod config { } pub mod data_usage { - #[cfg(feature = "test-util")] - pub use crate::data_usage::seed_bucket_usage_memory_for_test; pub use crate::data_usage::{ DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, @@ -330,6 +328,8 @@ pub mod data_usage { remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend, store_data_usage_in_backend, }; + #[cfg(feature = "test-util")] + pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test}; } pub mod disk { diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index efe517b99..61dbf2f1e 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info( .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); let mut rstate = oi.replication_state(); rstate.replicate_decision_str = dsc.to_string(); - let asz = oi.get_actual_size().unwrap_or_default(); + let asz = oi.get_actual_size_or_physical(); let ssec = replication_object_is_ssec_encrypted(&oi.user_defined); let checksum = if ssec { oi.checksum.clone() } else { None }; diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index d921ea411..e670a7d6f 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC }; let mut replication_state = oi.replication_state(); replication_state.replicate_decision_str = dsc.to_string(); - let actual_size = oi.get_actual_size().unwrap_or_default(); + let actual_size = oi.get_actual_size_or_physical(); Ok(ReplicateObjectInfo { name: oi.name.clone(), diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 455a37ed0..4fe89967d 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)), version_id: object_info.version_id.map(|version_id| version_id.to_string()), etag: object_info.etag.as_deref(), - actual_size: object_info.get_actual_size().unwrap_or_default(), + actual_size: object_info.get_actual_size_or_physical(), delete_marker: object_info.delete_marker, content_type: object_info.content_type.as_deref(), content_encoding: object_info.content_encoding.as_deref(), @@ -542,6 +542,20 @@ mod tests { assert!(replication_target_head_is_newer_null_version(&source, &target)); } + #[test] + fn replication_source_uses_physical_size_for_unknown_compressed_object() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + let source = ObjectInfo { + size: 128, + actual_size: -1, + user_defined: Arc::new(metadata), + ..Default::default() + }; + + assert_eq!(replication_source_object(&source).actual_size, 128); + } + #[test] fn replication_target_head_content_matches_compare_etag_only() { let source = ObjectInfo { diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 3acf1a705..d9b354a08 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -21,7 +21,7 @@ use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}, - object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, + object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, range::HTTPRangeSpec, }; use crate::{ @@ -414,6 +414,66 @@ fn apply_delete_objects_results( } } +fn apply_delete_accounting_results( + accounting: &mut [Option], + set_objects: &[DelObj], + set_accounting: &[Option], +) { + for (obj, value) in set_objects.iter().zip(set_accounting.iter()) { + accounting[obj.orig_idx] = value.clone(); + } +} + +impl Sets { + pub(crate) async fn delete_objects_with_accounting( + &self, + bucket: &str, + objects: Vec, + opts: ObjectOptions, + ) -> (Vec, Vec>, Vec>) { + let mut del_objects = vec![DeletedObject::default(); objects.len()]; + let mut del_errs = vec![None; objects.len()]; + let mut accounting = vec![None; objects.len()]; + let mut set_obj_map = HashMap::new(); + + for (i, obj) in objects.iter().enumerate() { + let idx = self.get_hashed_set_index(obj.object_name.as_str()); + set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj { + orig_idx: i, + obj: obj.clone(), + }); + } + + let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1); + let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent)); + let mut futures = FuturesUnordered::new(); + let bucket = bucket.to_owned(); + + for (set_index, set_objects) in set_obj_map { + let disks = self.get_disks(set_index); + let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::>(); + let bucket = bucket.clone(); + let opts = opts.clone(); + let semaphore = semaphore.clone(); + futures.push(async move { + let _permit = semaphore + .acquire_owned() + .await + .expect("delete_objects semaphore should remain open"); + let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await; + (set_objects, deleted, errors, accounting) + }); + } + + while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await { + apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors); + apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting); + } + + (del_objects, del_errs, accounting) + } +} + #[async_trait::async_trait] impl crate::storage_api_contracts::object::ObjectIO for Sets { type Error = Error; @@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets { objects: Vec, opts: ObjectOptions, ) -> (Vec, Vec>) { - // Default return value - let mut del_objects = vec![DeletedObject::default(); objects.len()]; - - let mut del_errs = Vec::with_capacity(objects.len()); - for _ in 0..objects.len() { - del_errs.push(None) - } - - let mut set_obj_map = HashMap::new(); - - // hash key - for (i, obj) in objects.iter().enumerate() { - let idx = self.get_hashed_set_index(obj.object_name.as_str()); - - if !set_obj_map.contains_key(&idx) { - set_obj_map.insert( - idx, - vec![DelObj { - // set_idx: idx, - orig_idx: i, - obj: obj.clone(), - }], - ); - } else if let Some(val) = set_obj_map.get_mut(&idx) { - val.push(DelObj { - // set_idx: idx, - orig_idx: i, - obj: obj.clone(), - }); - } - } - - let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1); - let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent)); - let mut futures = FuturesUnordered::new(); - let bucket = bucket.to_string(); - - for (k, v) in set_obj_map { - let disks = self.get_disks(k); - let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); - let bucket = bucket.clone(); - let opts = opts.clone(); - let semaphore = semaphore.clone(); - - futures.push(async move { - let _permit = semaphore - .acquire_owned() - .await - .expect("delete_objects semaphore should remain open"); - let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await; - (v, dobjects, errs) - }); - } - - while let Some((v, dobjects, errs)) = futures.next().await { - apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs); - } - - (del_objects, del_errs) + let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await; + (deleted, errors) } #[tracing::instrument(skip(self))] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 917edc649..3bf9bf500 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator { } pub fn quota_object_size(object: &ObjectInfo) -> Result { - let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?; + // A compressed object may carry -1 while the transformed size is unknown + // (legacy streaming sentinel). In that case the persisted physical size + // is still a valid accounting floor; every other negative value is corrupt. + // An explicit negative `actual-size` metadata value is corrupt, however: + // the sentinel is only valid in the in-memory/object-part field written by + // the legacy streaming path, not as a persisted declared size. + let compressed = object.is_compressed(); + if object.actual_size < -1 || (object.actual_size == -1 && !compressed) { + return Err(Error::PartMissingOrCorrupt); + } + if object + .parts + .iter() + .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed)) + { + return Err(Error::PartMissingOrCorrupt); + } + let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE) + .filter(|value| !value.is_empty()); + if declared_actual_size + .as_deref() + .and_then(|value| value.parse::().ok()) + .is_some_and(|size| size < 0) + { + return Err(Error::PartMissingOrCorrupt); + } + let logical_size = match object.get_actual_size().map_err(Error::other)? { + size if size == -1 && compressed && declared_actual_size.is_none() => None, + size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?), + _ => return Err(Error::PartMissingOrCorrupt), + }; let persisted_part_size = if object.parts.is_empty() { u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)? } else { @@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result { // Compressed streaming objects persist -1 when the transformed // part size is unknown. The physical part size remains a valid // quota floor; reject only non-negative values that overflow. - let actual_size = if part.actual_size < 0 { - if object.is_compressed() { - 0 - } else { - return Err(Error::PartMissingOrCorrupt); - } + let actual_size = if part.actual_size == -1 { + 0 } else { u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)? }; @@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result { total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt) })? }; - Ok(logical_size.max(persisted_part_size)) + Ok(logical_size.unwrap_or(0).max(persisted_part_size)) } type UsageVersionPage = StorageListObjectVersionsInfo; @@ -3320,6 +3346,80 @@ mod tests { ); } + #[test] + fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let object = ObjectInfo { + size: 400, + actual_size: -1, + user_defined: Arc::new(metadata), + ..Default::default() + }; + + assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400); + } + + #[test] + fn quota_object_size_rejects_compressed_part_sum_overflow() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let object = ObjectInfo { + size: 1, + user_defined: Arc::new(metadata), + parts: Arc::new(vec![ + rustfs_filemeta::ObjectPartInfo { + actual_size: i64::MAX, + ..Default::default() + }, + rustfs_filemeta::ObjectPartInfo { + actual_size: 1, + ..Default::default() + }, + ]), + ..Default::default() + }; + + assert!(matches!(quota_object_size(&object), Err(Error::Io(_)))); + } + + #[test] + fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let corrupt_object = ObjectInfo { + size: 400, + actual_size: -2, + user_defined: Arc::new(metadata.clone()), + ..Default::default() + }; + assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt))); + + let corrupt_part = ObjectInfo { + size: 400, + user_defined: Arc::new(metadata), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 400, + actual_size: -2, + ..Default::default() + }]), + ..Default::default() + }; + assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt))); + } + #[tokio::test] #[serial] async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() { diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 1bbff7a7f..0194667e9 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -689,6 +689,9 @@ impl ObjectInfo { } pub fn get_actual_size(&self) -> std::io::Result { + if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) { + return Err(std::io::Error::other("invalid negative actual size")); + } if self.actual_size > 0 { return Ok(self.actual_size); } @@ -700,10 +703,25 @@ impl ObjectInfo { let size = size_str.parse::().map_err(|e| std::io::Error::other(e.to_string()))?; return Ok(size); } - let mut actual_size = 0; - self.parts.iter().for_each(|part| { - actual_size += part.actual_size; - }); + if self.actual_size == -1 && self.parts.is_empty() { + return Ok(-1); + } + let mut actual_size = 0_i64; + let mut unknown = false; + for part in self.parts.iter() { + match part.actual_size { + -1 => unknown = true, + size if size >= 0 => { + actual_size = actual_size + .checked_add(size) + .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?; + } + _ => return Err(std::io::Error::other("invalid negative compressed part size")), + } + } + if unknown { + return Ok(-1); + } if actual_size == 0 && actual_size != self.size { return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size))); } @@ -718,6 +736,18 @@ impl ObjectInfo { Ok(self.size) } + /// Returns a non-negative size for client and replication boundaries. + /// + /// Compressed legacy metadata can retain the internal `-1` unknown-size + /// sentinel. Those boundaries cannot emit a negative length, so they use + /// the persisted physical size while quota accounting keeps the sentinel + /// distinction in [`crate::data_usage::quota_object_size`]. + pub fn get_actual_size_or_physical(&self) -> i64 { + self.get_actual_size() + .map(|size| if size >= 0 { size } else { self.size.max(0) }) + .unwrap_or_else(|_| self.size.max(0)) + } + pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo { let mut version_id = fi.version_id; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index cbf050a4b..e64e1704f 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -97,7 +97,7 @@ use crate::storage_api_contracts::{ CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo, }, namespace::NamespaceLocking as _, - object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, + object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, range::HTTPRangeSpec, }; use crate::store::utils::is_reserved_or_invalid_bucket; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index cb8f70766..faacca3a8 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -45,6 +45,7 @@ use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta, }; +use crate::data_usage::quota_object_size; use crate::diagnostics::get::GetObjectFailureReason; use crate::disk::{DataDirDeleteStatus, OldCurrentSize}; use crate::error::is_err_invalid_upload_id; @@ -5655,7 +5656,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { objects: Vec, opts: ObjectOptions, ) -> (Vec, Vec>) { + let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await; + (deleted, errors) + } + + async fn delete_objects_with_accounting( + &self, + bucket: &str, + objects: Vec, + opts: ObjectOptions, + ) -> (Vec, Vec>, Vec>) { let mut del_objects = vec![DeletedObject::default(); objects.len()]; + let mut accounting = vec![None; objects.len()]; let delete_config_snapshot = opts .delete_replication_config_snapshot .clone() @@ -5745,7 +5757,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { *item = Some(Error::other(message.clone())); } } - return (del_objects, del_errs); + return (del_objects, del_errs, accounting); } }, } @@ -5792,6 +5804,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let source_missing = gerr .as_ref() .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)); + // Resolve accounting from the generation selected under this + // object's write lock. A request-layer pre-stat is only an + // optimization and cannot identify a concurrent overwrite. + let (accounting_size, accounting_version_id, removed_current_object) = if source_missing + || dobj.synthetic_version_id + || set_disk_delete_creates_delete_marker(&check_opts) + || goi.delete_marker + { + (None, None, false) + } else { + ( + quota_object_size(&goi).ok(), + goi.version_id.filter(|version_id| !version_id.is_nil()), + (dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id, + ) + }; // Normalize both sides before comparing. `goi.version_id` is the // client-facing identity, where `from_file_info` synthesizes // `Some(Uuid::nil())` for a null version on a versioned or @@ -5920,7 +5948,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { }, replication_state: vr.replication_state_internal.clone(), ..Default::default() - } + }; + accounting[i] = Some(DeleteAccounting { + size: accounting_size, + version_id: accounting_version_id, + removed_current_object, + }); } // Only add to vers_map if we hold the lock @@ -5966,7 +5999,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { }); } } - return (del_objects, del_errs); + return (del_objects, del_errs, accounting); } let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len()); @@ -6204,7 +6237,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - (del_objects, del_errs) + // An accounting identity is actionable only when the delete result is + // successful. Never let a failed commit (including a partial quorum + // failure) reach the request-layer fast delta path. + for (index, err) in del_errs.iter().enumerate() { + if err.is_some() { + accounting[index] = None; + } + } + + (del_objects, del_errs, accounting) } #[tracing::instrument(skip(self))] @@ -6533,6 +6575,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended); obj_info.size = goi.size; + // Keep the committed source metadata on the internal delete result so + // the request layer can derive canonical accounting for this exact + // generation. Delete responses do not expose these fields. + obj_info.actual_size = goi.actual_size; + obj_info.user_defined = Arc::clone(&goi.user_defined); + obj_info.parts = Arc::clone(&goi.parts); obj_info.user_tags = Arc::clone(&goi.user_tags); self.invalidate_get_object_metadata_cache(bucket, object).await; Ok(obj_info) @@ -7824,6 +7872,113 @@ mod replication_quota_safety_tests { assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1); } + #[tokio::test] + async fn delete_returns_canonical_compressed_accounting_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "compressed-delete-accounting"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut user_defined = HashMap::new(); + insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string()); + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false) + .expect("compressed fixture reader should be valid"), + ); + set_disks + .put_object( + bucket, + "object", + &mut reader, + &ObjectOptions { + user_defined, + ..Default::default() + }, + ) + .await + .expect("compressed object should be written"); + + let (deleted, errors, accounting) = set_disks + .delete_objects_with_accounting( + bucket, + vec![ObjectToDelete { + object_name: "object".to_string(), + ..Default::default() + }], + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; + + assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]); + assert!(deleted[0].found, "the committed object must be reported as found"); + assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000)); + assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none())); + assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object)); + } + + #[tokio::test] + async fn suspended_delete_marker_does_not_return_body_accounting() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "suspended-delete-accounting"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut user_defined = HashMap::new(); + insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string()); + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false) + .expect("compressed fixture reader should be valid"), + ); + let suspended_opts = ObjectOptions { + version_suspended: true, + delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)), + ..Default::default() + }, + None, + ))), + user_defined, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), + ..Default::default() + }; + set_disks + .put_object(bucket, "object", &mut reader, &suspended_opts) + .await + .expect("compressed object should be written"); + + let (deleted, errors, accounting) = set_disks + .delete_objects_with_accounting( + bucket, + vec![ObjectToDelete { + object_name: "object".to_string(), + ..Default::default() + }], + suspended_opts, + ) + .await; + assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]); + assert!(deleted[0].delete_marker); + assert!(accounting[0].is_none(), "a delete marker must not carry body accounting"); + } + #[tokio::test] async fn direct_put_cannot_persist_a_tiny_logical_size() { let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/storage_api_contracts/mod.rs b/crates/ecstore/src/storage_api_contracts/mod.rs index 11e7b800f..78a3f9c2d 100644 --- a/crates/ecstore/src/storage_api_contracts/mod.rs +++ b/crates/ecstore/src/storage_api_contracts/mod.rs @@ -62,8 +62,8 @@ pub(crate) mod object { use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::storage_api_contracts::range::HTTPRangeSpec; pub(crate) use rustfs_storage_api::{ - DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations, - ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete, + DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, + ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete, }; pub(crate) trait EcstoreObjectIO: diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 312a25662..e5f2f0465 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -41,7 +41,7 @@ use crate::set_disk::{ }; use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, - object::{ObjectIO as _, ObjectOperations as _}, + object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _}, }; use parking_lot::Mutex as ParkingMutex; use rustfs_io_metrics::{ @@ -1216,6 +1216,14 @@ fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Ve (del_objects, del_errs) } +fn return_batch_delete_lock_error_with_accounting( + objects: &[ObjectToDelete], + err: Error, +) -> (Vec, Vec>, Vec>) { + let (deleted, errors) = return_batch_delete_lock_error(objects, err); + (deleted, errors, vec![None; objects.len()]) +} + fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> { let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect(); object_names.sort_unstable(); @@ -2312,6 +2320,22 @@ impl ECStore { result } + pub async fn delete_objects_with_tier_delete_journal_and_accounting( + self: &Arc, + bucket: &str, + objects: Vec, + opts: ObjectOptions, + ) -> (Vec, Vec>, Vec>) { + let result = self + .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self))) + .await; + let success_count = result.1.iter().filter(|err| err.is_none()).count(); + if success_count > 0 { + list_objects::observe_list_objects_mutations(self, bucket, success_count).await; + } + result + } + #[instrument(skip(self))] pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { self.handle_delete_object_with_journal(bucket, object, opts, None).await @@ -2689,6 +2713,19 @@ impl ECStore { opts: ObjectOptions, tier_journal_api: Option>, ) -> (Vec, Vec>) { + let (deleted, errors, _) = self + .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api) + .await; + (deleted, errors) + } + + pub(super) async fn handle_delete_objects_with_journal_and_accounting( + &self, + bucket: &str, + objects: Vec, + opts: ObjectOptions, + tier_journal_api: Option>, + ) -> (Vec, Vec>, Vec>) { // encode object name let objects: Vec = objects .iter() @@ -2701,6 +2738,7 @@ impl ECStore { // Default return value let mut del_objects = vec![DeletedObject::default(); objects.len()]; + let mut accounting = vec![None; objects.len()]; let mut del_errs = Vec::with_capacity(objects.len()); for _ in 0..objects.len() { @@ -2714,7 +2752,7 @@ impl ECStore { } else { match self.acquire_bucket_lifecycle_read_lock(bucket).await { Ok(guard) => Some(guard), - Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), } }; if let Some(guard) = _bucket_lifecycle_guard.as_ref() { @@ -2726,21 +2764,21 @@ impl ECStore { Err(err) => { let message = err.to_string(); let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect(); - return (del_objects, errors); + return (del_objects, errors, accounting); } } } if !is_meta_bucketname(bucket) && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await { - return return_batch_delete_lock_error(objects.as_slice(), err); + return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err); } let _object_lock_metadata_guard = if is_meta_bucketname(bucket) { None } else { Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await { Ok(guard) => guard, - Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), }) }; if let Some(guard) = _object_lock_metadata_guard.as_ref() { @@ -2750,7 +2788,7 @@ impl ECStore { let (state, incarnation_id, config_revision) = match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await { Ok(snapshot) => snapshot, - Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), }; opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket( self.id, @@ -2766,7 +2804,10 @@ impl ECStore { if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id) && expected != current { - return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string())); + return return_batch_delete_lock_error_with_accounting( + objects.as_slice(), + StorageError::BucketNotFound(bucket.to_string()), + ); } #[cfg(test)] if current_bucket_incarnation_id.is_some() { @@ -2774,7 +2815,7 @@ impl ECStore { } let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { Ok(guards) => guards, - Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), }; let mut futures = Vec::with_capacity(self.pools.len()); @@ -2783,22 +2824,24 @@ impl ECStore { if self.is_pool_rebalancing(pool.pool_idx).await { continue; } - futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone())); + futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone())); } let results = join_all(futures).await; for idx in 0..del_objects.len() { - for (dels, errs) in results.iter() { + for (dels, errs, pool_accounting) in results.iter() { if errs[idx].is_none() && dels[idx].found { del_errs[idx] = None; del_objects[idx] = dels[idx].clone(); + accounting[idx] = pool_accounting[idx].clone(); break; } if del_errs[idx].is_none() { del_errs[idx] = errs[idx].clone(); del_objects[idx] = dels[idx].clone(); + accounting[idx] = pool_accounting[idx].clone(); } } } @@ -2807,7 +2850,7 @@ impl ECStore { v.object_name = decode_dir_object(&v.object_name); }); - (del_objects, del_errs) + (del_objects, del_errs, accounting) // let mut futures = Vec::with_capacity(objects.len()); diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index e011327f9..1114349e1 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -76,6 +76,7 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus}; pub use error::{StorageErrorCode, StorageResult}; pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; +pub use object::DeleteAccounting; pub use object::ObjectLockDeleteOptions; pub use object::{DeletedObject, ObjectToDelete}; pub use object::{ExpirationOptions, TransitionedObject}; diff --git a/crates/storage-api/src/object.rs b/crates/storage-api/src/object.rs index 9f0957bde..7959354dc 100644 --- a/crates/storage-api/src/object.rs +++ b/crates/storage-api/src/object.rs @@ -218,6 +218,17 @@ pub struct DeletedObject { pub force_delete_generation: Option, } +/// Accounting identity returned by the internal commit-time delete path. +/// +/// This is carried separately from [`DeletedObject`] so adding quota details +/// does not change the source shape of the public S3 delete result contract. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct DeleteAccounting { + pub size: Option, + pub version_id: Option, + pub removed_current_object: bool, +} + impl DeletedObject { pub fn version_purge_status(&self) -> VersionPurgeStatusType { self.replication_state @@ -341,6 +352,19 @@ pub trait ObjectOperations: Send + Sync + fmt::Debug { objects: Vec, opts: Self::ObjectOptions, ) -> (Vec, Vec>); + /// Delete objects and optionally return commit-time accounting identities. + /// The default preserves the ordinary delete contract for implementations + /// that do not expose storage-level accounting details. + async fn delete_objects_with_accounting( + &self, + bucket: &str, + objects: Vec, + opts: Self::ObjectOptions, + ) -> (Vec, Vec>, Vec>) { + let object_count = objects.len(); + let (deleted, errors) = self.delete_objects(bucket, objects, opts).await; + (deleted, errors, vec![None; object_count]) + } async fn put_object_metadata( &self, bucket: &str, diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 7d55035e9..b51254b23 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1021,7 +1021,7 @@ fn build_list_objects_v2_metadata_output( object: Object { key: Some(encode_list_objects_v2_value(&object.name, encoding_type)), last_modified: object.mod_time.map(Timestamp::from), - size: Some(object.get_actual_size().unwrap_or_default()), + size: Some(object.get_actual_size_or_physical()), e_tag: object.etag.clone().map(|etag| to_s3s_etag(&etag)), storage_class: Some(ObjectStorageClass::from( object diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index a31ec1d19..d1978f2db 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -3969,6 +3969,55 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool { opts.version_id.is_none() && opts.versioned && !opts.version_suspended } +fn delete_removes_current_object(opts: &ObjectOptions) -> bool { + delete_request_targets_current( + opts.version_id + .as_deref() + .and_then(|version_id| Uuid::parse_str(version_id).ok()), + ) +} + +fn delete_request_targets_current(version_id: Option) -> bool { + version_id.is_none() || version_id.is_some_and(|version_id| version_id.is_nil()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeleteMemoryUpdate { + DeleteMarker, + Object { size: u64, removed_current_object: bool }, +} + +fn delete_memory_update( + creates_delete_marker: bool, + committed_delete_marker: bool, + requested_current: bool, + accounting_size: Option, + removed_current_object: bool, +) -> Option { + if creates_delete_marker || (committed_delete_marker && requested_current) { + return Some(DeleteMemoryUpdate::DeleteMarker); + } + + (!committed_delete_marker) + .then_some(accounting_size) + .flatten() + .map(|size| DeleteMemoryUpdate::Object { + size, + removed_current_object, + }) +} + +async fn apply_delete_memory_update(bucket: &str, update: Option) { + match update { + Some(DeleteMemoryUpdate::DeleteMarker) => record_bucket_delete_marker_memory(bucket).await, + Some(DeleteMemoryUpdate::Object { + size, + removed_current_object, + }) => record_bucket_object_delete_memory(bucket, size, removed_current_object).await, + None => {} + } +} + /// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the /// distributed delete path instead of its usual typed missing-object error. fn is_delete_objects_not_found(error: &EcstoreError) -> bool { @@ -8409,8 +8458,6 @@ impl DefaultObjectUsecase { object: ObjectToDelete, versioned: bool, version_suspended: bool, - size: i64, - existing: Option, } // Phase 2 (bounded concurrency, backlog#929 / HP-8): collect the @@ -8428,32 +8475,23 @@ impl DefaultObjectUsecase { skip_stat, } = prepared; let synthetic_version_id = object.version_id.is_none() && is_dir_object(&object.object_name); - let (goi, source_missing) = if skip_stat { - (ObjectInfo::default(), false) - } else { + if !skip_stat { match store_ref.get_object_info(bucket_ref, &object.object_name, &opts).await { - Ok(res) => (res, false), - Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => { - (ObjectInfo::default(), true) - } + Ok(_) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} Err(err) => return Err(ApiError::from(err)), } - }; - - let size = goi.size; + } if synthetic_version_id { object.version_id = Some(Uuid::nil()); } - let existing = (!skip_stat && !source_missing).then_some(goi); Ok::<_, ApiError>(AdmittedDelete { idx, object, versioned: opts.versioned, version_suspended: opts.version_suspended, - size, - existing, }) })) .buffered(DELETE_OBJECTS_PRE_STAT_CONCURRENCY) @@ -8464,15 +8502,11 @@ impl DefaultObjectUsecase { // per-key success/failure reporting is unchanged. let mut object_to_delete = Vec::new(); let mut object_to_delete_idx = Vec::new(); - let mut object_sizes = Vec::new(); - let mut existing_object_infos = Vec::new(); let mut object_versioning = Vec::new(); for admitted in admitted_deletes { - object_sizes.push(admitted.size); object_to_delete_idx.push(admitted.idx); object_versioning.push((admitted.versioned, admitted.version_suspended)); object_to_delete.push(admitted.object); - existing_object_infos.push(admitted.existing); } let cache_adapter = self.object_data_cache(); let cache_keys_before_delete = object_to_delete @@ -8489,8 +8523,8 @@ impl DefaultObjectUsecase { ..Default::default() }; apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?; - let (dobjs, errs) = store - .delete_objects_with_tier_delete_journal(&bucket, object_to_delete.clone(), storage_delete_opts) + let (dobjs, errs, accounting) = store + .delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts) .await; let _manager = get_concurrency_manager(); @@ -8515,17 +8549,16 @@ impl DefaultObjectUsecase { delete_results[didx].delete_object = Some(deleted_object.clone()); let (versioned, version_suspended) = object_versioning[i]; let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended; - if creates_delete_marker { - record_bucket_delete_marker_memory(&bucket).await; - } else { - let size = object_sizes[i].max(0) as u64; - record_bucket_object_delete_memory( - &bucket, - size, - existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(), - ) - .await; - } + let committed_delete_marker = dobjs[i].delete_marker; + let delete_accounting = accounting.get(i).and_then(Option::as_ref); + let update = delete_memory_update( + creates_delete_marker, + committed_delete_marker, + delete_request_targets_current(object_to_delete[i].version_id), + delete_accounting.and_then(|value| value.size), + delete_accounting.is_some_and(|value| value.removed_current_object), + ); + apply_delete_memory_update(&bucket, update).await; } Err(error) => { delete_results[didx].error = Some(error); @@ -8803,12 +8836,24 @@ impl DefaultObjectUsecase { let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; } - // Fast in-memory update for immediate quota and admin usage consistency - if delete_creates_delete_marker(&opts) { - record_bucket_delete_marker_memory(&bucket).await; + // Fast in-memory update for immediate quota and admin usage consistency. + // Prefix/force deletes and synthetic directory entries do not carry one + // committed object identity; leave their cache delta to reconciliation. + let update = if force_delete || obj_info.name.is_empty() || synthetic_version_id { + None } else { - record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await; - } + // The storage commit returns this object's metadata while its + // generation lock is held. Never fall back to a pre-delete stat: + // an overwrite can commit between that stat and this delete. + delete_memory_update( + delete_creates_delete_marker(&opts), + obj_info.delete_marker, + opts.version_id.is_none(), + quota_object_size(&obj_info).ok(), + delete_removes_current_object(&opts), + ) + }; + apply_delete_memory_update(&bucket, update).await; if obj_info.name.is_empty() { if let Some((operation_id, target_arns, generation)) = force_delete_intent { @@ -17861,6 +17906,158 @@ mod tests { assert!(!can_skip_delete_objects_pre_stat(false, &delete_marker_creating_opts(), false)); } + #[test] + fn delete_accounting_recognizes_explicit_null_as_current_object() { + let opts = ObjectOptions { + version_id: Some(Uuid::nil().to_string()), + version_suspended: true, + ..Default::default() + }; + assert!(delete_removes_current_object(&opts)); + assert!(delete_request_targets_current(Some(Uuid::nil()))); + assert!(!delete_request_targets_current(Some(Uuid::new_v4()))); + assert!(!delete_removes_current_object(&ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + })); + } + + #[test] + fn compressed_object_delete_restores_usage_baseline() { + let mut metadata = HashMap::new(); + insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string()); + let object = ObjectInfo { + size: 400, + actual_size: 1000, + user_defined: Arc::new(metadata), + ..Default::default() + }; + let accounting_size = quota_object_size(&object).expect("logical compressed size should be canonical"); + + assert_eq!( + delete_memory_update(false, false, true, Some(accounting_size), true), + Some(DeleteMemoryUpdate::Object { + size: 1000, + removed_current_object: true, + }) + ); + } + + #[test] + fn invalid_accounting_metadata_is_reconciled_without_overflow() { + assert_eq!(delete_memory_update(false, false, true, None, true), None); + assert_eq!( + delete_memory_update(false, true, true, None, true), + Some(DeleteMemoryUpdate::DeleteMarker) + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn compressed_delete_requests_restore_usage_baseline() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let bucket = format!("compressed-delete-request-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create compressed delete request bucket"); + + // Seed the process-local usage with the canonical logical bytes. The + // direct storage PUT below intentionally does not apply an app-layer + // usage delta; the two real DELETE requests must remove exactly this + // amount through their request-layer wiring. + crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 2_000).await; + + for object in ["single", "batch"] { + let mut metadata = HashMap::new(); + insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, "1000".to_string()); + let reader = HashReader::from_stream(std::io::Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false) + .expect("compressed fixture reader should be valid"); + let mut reader = PutObjReader::new(reader); + store + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("compressed fixture object should be written"); + } + + let mut single_req = build_request( + DeleteObjectInput::builder() + .bucket(bucket.clone()) + .key("single".to_string()) + .build() + .expect("single delete input should build"), + Method::DELETE, + ); + single_req.extensions.insert(crate::storage::access::ReqInfo { + cred: Some(rustfs_credentials::Credentials::default()), + is_owner: true, + ..Default::default() + }); + DefaultObjectUsecase::from_global() + .execute_delete_object(single_req) + .await + .expect("single compressed delete should succeed"); + assert_eq!( + crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await, + Some(1_000), + "single delete must subtract the logical accounting size" + ); + + let mut batch_req = build_request( + DeleteObjectsInput::builder() + .bucket(bucket.clone()) + .delete(Delete { + objects: vec![ObjectIdentifier { + key: "batch".to_string(), + ..Default::default() + }], + quiet: None, + }) + .build() + .expect("batch delete input should build"), + Method::POST, + ); + batch_req.extensions.insert(crate::storage::access::ReqInfo { + cred: Some(rustfs_credentials::Credentials::default()), + is_owner: true, + ..Default::default() + }); + DefaultObjectUsecase::from_global() + .execute_delete_objects(batch_req) + .await + .expect("batch compressed delete should succeed"); + assert_eq!( + crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await, + Some(0), + "batch delete must subtract the committed logical accounting size" + ); + + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await + .expect("clean up compressed delete request bucket"); + } + #[tokio::test] async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() { let input = GetObjectAttributesInput::builder() diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index d90cfb6a8..a599cfde6 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -72,6 +72,11 @@ pub(crate) mod data_usage { compute_bucket_usage, live_bucket_usage_computations, seed_bucket_usage_memory_for_test, store_data_usage_in_backend, }; + #[cfg(test)] + pub(crate) async fn get_bucket_usage_memory(bucket: &str) -> Option { + crate::storage::storage_api::ecstore_data_usage::get_bucket_usage_memory(bucket).await + } + pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { crate::storage::storage_api::ecstore_data_usage::record_bucket_object_delete_memory( bucket, @@ -1233,7 +1238,10 @@ pub(crate) mod test { pub(crate) use super::access::ReqInfo; pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS; - pub(crate) use super::{bucket, data_usage, ecfs, object_utils, runtime}; + pub(crate) use super::{bucket, ecfs, object_utils, runtime}; + pub(crate) mod data_usage { + pub(crate) use super::super::data_usage::*; + } pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata}; pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index e105adba4..6599ca107 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -296,7 +296,10 @@ pub(crate) fn build_list_objects_v2_output( let mut obj = Object { key: Some(key), last_modified: v.mod_time.map(Timestamp::from), - size: Some(v.get_actual_size().unwrap_or_default()), + // Compressed legacy objects may retain an unknown (-1) + // logical-size sentinel; never expose that internal value in + // an S3 response. + size: Some(v.get_actual_size_or_physical()), e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), storage_class: v.storage_class.clone().map(ObjectStorageClass::from), ..Default::default() @@ -656,6 +659,45 @@ mod tests { assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2)); } + #[test] + fn list_objects_never_exposes_compressed_unknown_size_sentinel() { + let mut metadata = std::collections::HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let output = build_list_objects_v2_output( + ListObjectsV2Info { + objects: vec![ObjectInfo { + name: "legacy-compressed".to_string(), + size: 128, + actual_size: -1, + user_defined: std::sync::Arc::new(metadata), + ..Default::default() + }], + ..Default::default() + }, + false, + 1000, + "bucket".to_string(), + String::new(), + None, + None, + None, + None, + ); + + assert_eq!( + output + .contents + .as_ref() + .and_then(|objects| objects.first()) + .and_then(|object| object.size), + Some(128) + ); + } + #[test] fn list_responses_report_standard_for_legacy_label_only_file_metadata() { let version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("fixture version ID should be valid"); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 31668783a..e5277c0dd 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -429,6 +429,8 @@ pub(crate) mod ecstore_config { } pub(crate) mod ecstore_data_usage { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::data_usage::get_bucket_usage_memory; pub(crate) use rustfs_ecstore::api::data_usage::{ apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached, load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, From a930152d5a5cc38320effd0231a82288278ff08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sat, 22 Aug 2026 20:43:52 +0800 Subject: [PATCH 05/32] fix(admin): expose per-target disableProxy through remote target admin API (#6376) The read-proxy selector already honors a target's disable_proxy flag (PR #6172), but the admin API still rejected the field, so the only way to set it was importing a MinIO-written bucket-targets.json. - move disableProxy from REMOTE_TARGET_UNSUPPORTED_FIELDS to REMOTE_TARGET_WRITABLE_FIELDS (set-remote-target create accepts it) - add TargetUpdateOp::Proxy so set-remote-target?update=true&proxy=true overlays only the proxy group (MinIO TargetUpdateType parity) - bump REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION 1 -> 2 and update the runtime capability pin tests - keep edge/edgeSyncBeforeExpiry rejected (no implementation behind them) - pin that a published TargetClient carries disable_proxy, the field the proxy-target selector consults Refs rustfs/backlog#1950 --- .../ecstore/src/bucket/bucket_target_sys.rs | 38 +++++++++++++ crates/replication/src/config.rs | 9 ++- rustfs/src/admin/handlers/replication.rs | 56 ++++++++++++++++--- rustfs/src/admin/handlers/system.rs | 25 +++++++-- 4 files changed, 115 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 74bf96d7f..7e5a5df5f 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -3425,6 +3425,44 @@ mod tests { assert!(mutexes.contains_key("second")); } + #[tokio::test] + async fn update_all_targets_publishes_disable_proxy_on_target_client() { + // The read-proxy selector (replication_proxy::get_proxy_targets) skips + // targets whose TargetClient carries disable_proxy — the persisted + // per-target opt-out must survive client publication. + let sys = BucketTargetSys::default(); + let target = |arn: &str, disable_proxy: bool| BucketTarget { + arn: arn.to_string(), + endpoint: "192.168.1.10:9000".to_string(), + target_bucket: "target-bucket".to_string(), + region: "us-east-1".to_string(), + disable_proxy, + credentials: Some(Credentials { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + session_token: None, + expiration: None, + }), + ..Default::default() + }; + let targets = BucketTargets { + targets: vec![target("arn:proxied", false), target("arn:opted-out", true)], + }; + + sys.update_all_targets("bucket", Some(&targets)).await; + + let proxied = sys + .get_remote_target_client("bucket", "arn:proxied") + .await + .expect("client should be published"); + assert!(!proxied.disable_proxy); + let opted_out = sys + .get_remote_target_client("bucket", "arn:opted-out") + .await + .expect("client should be published"); + assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient"); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn target_updates_serialize_client_build_through_publication_per_bucket() { let sys = Arc::new(BucketTargetSys::default()); diff --git a/crates/replication/src/config.rs b/crates/replication/src/config.rs index c8ca328a5..405799f94 100644 --- a/crates/replication/src/config.rs +++ b/crates/replication/src/config.rs @@ -60,7 +60,9 @@ pub const REPLICATION_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[ "Destination.ReplicationTime", ]; -pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 1; +// v2: disableProxy moved from unsupported to writable (per-target read-proxy +// opt-out is accepted by set-remote-target and the `proxy` update op). +pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 2; pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[ "sourcebucket", @@ -83,9 +85,12 @@ pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[ // madmin default of 60s); the per-target health-check interval is not // yet applied — the heartbeat keeps its global env-configured interval. "healthCheckDuration", + // Per-target read-proxy opt-out, consumed by the proxy-target selector + // (contract v2; previously only importable via MinIO bucket-targets.json). + "disableProxy", ]; -pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["disableProxy", "edge", "edgeSyncBeforeExpiry"]; +pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["edge", "edgeSyncBeforeExpiry"]; #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct ObjectOpts { diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 106c25790..229c3c9fd 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -73,6 +73,8 @@ enum TargetUpdateOp { /// Connection group: credentials plus endpoint, target bucket, and TLS settings. Credentials, Sync, + /// Per-target read-proxy opt-out (`disableProxy`). + Proxy, Bandwidth, Path, } @@ -81,12 +83,13 @@ fn parse_remote_target_update_ops(queries: &HashMap) -> S3Result const SUPPORTED_OPS: &[(&str, TargetUpdateOp)] = &[ ("creds", TargetUpdateOp::Credentials), ("sync", TargetUpdateOp::Sync), + ("proxy", TargetUpdateOp::Proxy), ("bandwidth", TargetUpdateOp::Bandwidth), ("path", TargetUpdateOp::Path), ]; // Present in the MinIO wire contract, but they drive target fields this // version rejects as unsupported — fail loudly instead of silently ignoring. - const UNSUPPORTED_OPS: &[&str] = &["proxy", "healthcheck", "edge", "edgeSyncBeforeExpiry"]; + const UNSUPPORTED_OPS: &[&str] = &["healthcheck", "edge", "edgeSyncBeforeExpiry"]; for key in UNSUPPORTED_OPS { if queries.get(*key).is_some_and(|value| value == "true") { @@ -312,11 +315,10 @@ impl RemoteTargetRequest { )); } - for (unsupported, configured) in - REMOTE_TARGET_UNSUPPORTED_FIELDS - .iter() - .copied() - .zip([self.disable_proxy, self.edge, self.edge_sync_before_expiry]) + for (unsupported, configured) in REMOTE_TARGET_UNSUPPORTED_FIELDS + .iter() + .copied() + .zip([self.edge, self.edge_sync_before_expiry]) { if configured { return Err(s3_error!( @@ -702,6 +704,7 @@ impl Operation for SetRemoteTargetHandler { target.deployment_id = remote_target.deployment_id.clone(); } TargetUpdateOp::Sync => target.replication_sync = remote_target.replication_sync, + TargetUpdateOp::Proxy => target.disable_proxy = remote_target.disable_proxy, TargetUpdateOp::Bandwidth => target.bandwidth_limit = remote_target.bandwidth_limit, TargetUpdateOp::Path => target.path = remote_target.path.clone(), } @@ -1520,6 +1523,7 @@ mod tests { ("update", "true"), ("creds", "true"), ("sync", "true"), + ("proxy", "true"), ("bandwidth", "true"), ("path", "true"), ])) @@ -1529,6 +1533,7 @@ mod tests { vec![ TargetUpdateOp::Credentials, TargetUpdateOp::Sync, + TargetUpdateOp::Proxy, TargetUpdateOp::Bandwidth, TargetUpdateOp::Path ] @@ -2070,7 +2075,6 @@ mod tests { ("credentials.session_token", serde_json::json!("session-token")), ("credentials.expiration", serde_json::json!("2026-01-01T00:00:00Z")), ("api", serde_json::json!("s3v2")), - ("disableProxy", serde_json::json!(true)), ("edge", serde_json::json!(true)), ("edgeSyncBeforeExpiry", serde_json::json!(true)), ] { @@ -2300,6 +2304,44 @@ mod tests { assert!(!REMOTE_TARGET_UNSUPPORTED_FIELDS.contains(&"healthCheckDuration")); } + #[test] + fn remote_target_disable_proxy_is_declared_writable_edge_stays_unsupported() { + assert!(REMOTE_TARGET_WRITABLE_FIELDS.contains(&"disableProxy")); + assert!(!REMOTE_TARGET_UNSUPPORTED_FIELDS.contains(&"disableProxy")); + // edge sync has no implementation behind it — it must stay rejected. + assert!(REMOTE_TARGET_UNSUPPORTED_FIELDS.contains(&"edge")); + assert!(REMOTE_TARGET_UNSUPPORTED_FIELDS.contains(&"edgeSyncBeforeExpiry")); + } + + #[test] + fn remote_target_create_accepts_disable_proxy() { + let mut request = valid_remote_target_request(); + request["disableProxy"] = serde_json::json!(true); + + let target = serde_json::from_value::(request) + .expect("request should deserialize") + .into_bucket_target() + .expect("disableProxy is a supported per-target read-proxy opt-out"); + + assert!(target.disable_proxy); + } + + #[test] + fn update_body_with_proxy_op_toggles_disable_proxy_without_credentials() { + // Mirrors the other partial-update groups: a proxy-only update body may + // omit the connection fields entirely. + let body = serde_json::json!({ + "arn": "arn:rustfs:replication:us-east-1:dep:target", + "type": "replication", + "disableProxy": true + }); + let request: RemoteTargetRequest = serde_json::from_value(body).expect("partial update body should deserialize"); + let target = request + .into_update_bucket_target(&[TargetUpdateOp::Proxy]) + .expect("proxy-only update must not require credentials"); + assert!(target.disable_proxy); + } + #[test] fn remote_target_capability_fields_do_not_overlap() { for field in REMOTE_TARGET_UNSUPPORTED_FIELDS { diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 9df481c11..981baaefb 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -1262,7 +1262,9 @@ mod tests { assert_eq!(response.summary.manual_transition_jobs.state, CapabilityState::Supported); assert_eq!(response.replication.contract_version, 1); assert_eq!(response.replication.bucket_replication.contract_version, 1); - assert_eq!(response.replication.remote_targets.contract_version, 1); + // v2: disableProxy moved from unsupported to writable (per-target + // read-proxy opt-out reached the admin API). + assert_eq!(response.replication.remote_targets.contract_version, 2); assert_eq!(response.replication.bucket_replication.status.state, CapabilityState::Supported); assert_eq!(response.replication.remote_targets.status.state, CapabilityState::Supported); assert_eq!( @@ -1293,7 +1295,15 @@ mod tests { .remote_targets .fields .iter() - .any(|field| field.name == "disableProxy" && field.state == super::ReplicationFieldState::Unsupported) + .any(|field| field.name == "disableProxy" && field.state == super::ReplicationFieldState::Supported) + ); + assert!( + response + .replication + .remote_targets + .fields + .iter() + .any(|field| field.name == "edge" && field.state == super::ReplicationFieldState::Unsupported) ); assert!( response @@ -1364,7 +1374,7 @@ mod tests { assert_eq!(value["summary"]["manual_transition_jobs"]["state"], "supported"); assert_eq!(value["replication"]["contract_version"], 1); assert_eq!(value["replication"]["bucket_replication"]["contract_version"], 1); - assert_eq!(value["replication"]["remote_targets"]["contract_version"], 1); + assert_eq!(value["replication"]["remote_targets"]["contract_version"], 2); assert_eq!(value["replication"]["bucket_replication"]["status"]["state"], "supported"); assert_eq!(value["replication"]["remote_targets"]["status"]["state"], "supported"); assert_eq!( @@ -1383,7 +1393,14 @@ mod tests { .as_array() .expect("remote target fields should be an array") .iter() - .any(|field| field["name"] == "disableProxy" && field["state"] == "unsupported") + .any(|field| field["name"] == "disableProxy" && field["state"] == "supported") + ); + assert!( + value["replication"]["remote_targets"]["fields"] + .as_array() + .expect("remote target fields should be an array") + .iter() + .any(|field| field["name"] == "edge" && field["state"] == "unsupported") ); assert!( value["replication"]["remote_targets"]["fields"] From da90d02c150306364020d146f6fe709e74a6e85e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 22 Aug 2026 20:45:03 +0800 Subject: [PATCH 06/32] test(ecstore): cover suspended-owner heal semantics (#6348) * test(ecstore): cover suspended-owner heal semantics * test(heal): cover suspended owner production path --- crates/ecstore/src/store/heal.rs | 278 ++++++++++++++++++++++++++++++- 1 file changed, 276 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index d10abe740..ffac77751 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -297,10 +297,16 @@ impl ECStore { #[cfg(test)] mod tests { use super::*; + use crate::bucket::metadata_sys; use crate::core::pools::{PoolDecommissionInfo, PoolStatus}; - use crate::disk::{DiskOption, format::FormatV3, new_disk}; - use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk}; + use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::runtime::instance::InstanceContext; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations}; use crate::store::init_format::{load_format_erasure, save_format_file}; + use crate::store::init_local_disks_with_instance_ctx; + use tokio_util::sync::CancellationToken; async fn minimal_heal_pool(pool_idx: usize) -> Arc { let format = FormatV3::new(1, 1); @@ -347,6 +353,51 @@ mod tests { } } + async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc, CancellationToken) { + let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created"); + let mut pool_endpoints = Vec::new(); + for pool_index in 0..2 { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("multi-pool heal test disk should be created"); + let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")) + .expect("test endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pool_endpoints.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: format!("heal-owner-pool-{pool_index}"), + platform: "test".to_string(), + }); + } + + let endpoint_pools = EndpointServerPools::from(pool_endpoints); + let instance_ctx = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("multi-pool local disks should initialize"); + let shutdown = CancellationToken::new(); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + shutdown.clone(), + instance_ctx, + ) + .await + .expect("multi-pool test store should initialize"); + metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + (temp_dir, store, shutdown) + } + #[tokio::test] async fn heal_object_pool_scope_selects_only_requested_pool() { let store = minimal_heal_store().await; @@ -506,6 +557,229 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn unscoped_heal_object_suspended_owner_semantics() { + let (_temp_dir, store, shutdown) = multi_pool_heal_store().await; + let bucket = format!("heal-owner-{}", Uuid::new_v4().simple()); + let active_object = "active-owner"; + let suspended_only_object = "suspended-only"; + let duplicate_object = "duplicate-owner"; + let marker_object = "marker-owner"; + let quorum_object = "quorum-owner"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in all pools"); + + let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec()); + store.pools[0] + .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default()) + .await + .expect("active owner object should be written"); + let active_disks = store.pools[0].disk_set[0].disks.read().await.clone(); + let missing_active_disk = active_disks[0].clone().expect("active disk should be online"); + missing_active_disk + .delete( + &bucket, + active_object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("active owner shard should be removed for repair"); + assert!( + missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(), + "the active owner fixture must start with one missing metadata copy" + ); + + let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec()); + store.pools[1] + .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default()) + .await + .expect("suspended owner object should be written"); + for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() { + let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes()); + store.pools[pool_index] + .put_object( + &bucket, + duplicate_object, + &mut duplicate_reader, + &ObjectOptions { + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)), + ..Default::default() + }, + ) + .await + .expect("duplicate owner object should be written"); + } + let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0] + .clone() + .expect("duplicate active owner disk should be online"); + duplicate_missing_disk + .delete( + &bucket, + duplicate_object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("duplicate active owner shard should be removed for repair"); + let history_version = Uuid::new_v4(); + let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec()); + store.pools[0] + .put_object( + &bucket, + marker_object, + &mut history_reader, + &ObjectOptions { + versioned: true, + version_id: Some(history_version.to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)), + ..Default::default() + }, + ) + .await + .expect("versioned marker history should be written"); + store.pools[0] + .delete_object( + &bucket, + marker_object, + ObjectOptions { + versioned: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)), + ..Default::default() + }, + ) + .await + .expect("delete marker should be written"); + let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec()); + store.pools[0] + .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default()) + .await + .expect("quorum boundary object should be written"); + { + let mut pool_meta = store.pool_meta.write().await; + let mut next = PoolMeta::new(&store.pools, &pool_meta); + next.pools[1].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }); + *pool_meta = next; + } + + let (_, duplicate_owner) = store + .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default()) + .await + .expect("duplicate owner should resolve"); + assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible"); + let (_, active_duplicate_owner) = store + .get_latest_object_info_with_idx( + &bucket, + duplicate_object, + &ObjectOptions { + skip_decommissioned: true, + ..Default::default() + }, + ) + .await + .expect("active duplicate owner should resolve"); + assert_eq!( + active_duplicate_owner, 0, + "suspended duplicate must be excluded from active owner selection" + ); + let (duplicate_result, duplicate_err) = store + .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default()) + .await + .expect("duplicate owner heal should complete through the production path"); + assert_eq!(duplicate_result.object, duplicate_object); + assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}"); + assert!( + duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(), + "production heal must repair the active duplicate owner rather than the suspended owner" + ); + let (marker_info, marker_owner) = store + .get_latest_object_info_with_idx( + &bucket, + marker_object, + &ObjectOptions { + skip_decommissioned: true, + versioned: true, + ..Default::default() + }, + ) + .await + .expect("latest delete marker should resolve"); + assert_eq!(marker_owner, 0); + assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics"); + + let (active_result, active_err) = store + .handle_heal_object(&bucket, active_object, "", &HealOpts::default()) + .await + .expect("unscoped active-owner heal should complete"); + assert_eq!(active_result.object, active_object); + assert!(active_err.is_none(), "active owner must be selected even with a suspended pool"); + assert!( + missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(), + "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}" + ); + assert!( + store.pools[1] + .get_object_info(&bucket, active_object, &ObjectOptions::default()) + .await + .is_err(), + "the suspended pool must not be written for an active-owner object" + ); + + let (suspended_result, suspended_err) = store + .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default()) + .await + .expect("unscoped suspended-only heal should return a terminal result"); + assert!(suspended_result.object.is_empty()); + assert!(matches!(suspended_err, Some(Error::FileNotFound))); + assert!( + store.pools[1] + .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default()) + .await + .is_ok(), + "suspended-only data must remain untouched when unscoped heal reports absent" + ); + + let (_, explicit_err) = store + .handle_heal_object( + &bucket, + suspended_only_object, + "", + &HealOpts { + pool: Some(1), + ..Default::default() + }, + ) + .await + .expect("explicit suspended-owner heal should return a mapped error"); + assert!(matches!(explicit_err, Some(Error::SlowDown))); + + let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone(); + let surviving_quorum_disk = original_quorum_disks[3].clone(); + *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk]; + let (_, quorum_err) = store + .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default()) + .await + .expect("quorum boundary heal should return a mapped result"); + *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks; + assert!( + matches!(quorum_err, Some(Error::ErasureReadQuorum)), + "quorum-boundary heal must preserve quorum error, got {quorum_err:?}" + ); + shutdown.cancel(); + } + #[tokio::test] async fn handle_heal_format_continues_after_a_pool_error() { let canonical_format = FormatV3::new(1, 3); From 6b5e0feef6b7e161564bde2080288a0c4b4a2d2b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 22 Aug 2026 21:52:08 +0800 Subject: [PATCH 07/32] test(e2e): wait for heal peers after node rejoin (#6359) --- .../src/heal_erasure_disk_rebuild_test.rs | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index dac1eef5e..dd82512a5 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -380,10 +380,24 @@ mod tests { cluster.start_node(1).await?; let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url); - let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?; - assert!( - !status_body.contains("MissingContentLength"), - "background heal status should not fail without an explicit Content-Length: {status_body}" + let mut recovered = serde_json::Value::Null; + for _ in 0..60 { + let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?; + assert!( + !status_body.contains("MissingContentLength"), + "background heal status should not fail without an explicit Content-Length: {status_body}" + ); + recovered = serde_json::from_str(&status_body) + .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?; + if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) { + break; + } + sleep(Duration::from_secs(1)).await; + } + assert_eq!( + recovered["clusterStatusComplete"], + serde_json::Value::Bool(true), + "cluster heal status should recover before root heal starts: {recovered}" ); let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; From 12a9e654b5e3471ee61f22152aa0af038f0a3810 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 22 Aug 2026 21:56:58 +0800 Subject: [PATCH 08/32] refactor(data-usage): rename ReplicationStats to ReplicationTargetUsage (#6345) * refactor(data-usage): ReplicationStats -> ReplicationTargetUsage Rename the data-usage crate's ReplicationStats to ReplicationTargetUsage. Serde field names are byte-identical (only the Rust type name changed; field identifiers that rmp encodes are untouched). An rmp round-trip test guards against future drift. Scanner test imports updated to match. * style: cargo fmt --- crates/data-usage/src/data_usage.rs | 82 +++++++++++++++---- crates/scanner/src/data_usage_define/tests.rs | 4 +- .../src/scanner_io/publish_gate_tests.rs | 4 +- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index 9c4692a21..81a313125 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -585,9 +585,12 @@ impl VersionsHistogram { } } -/// Replication statistics for a single target -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationStats { +/// Replication statistics for a single target. +/// +/// Renamed from `ReplicationStats`; serde field names are preserved +/// byte-identically to maintain wire compatibility with existing snapshots. +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReplicationTargetUsage { pub pending_size: u64, pub replicated_size: u64, pub failed_size: u64, @@ -600,7 +603,7 @@ pub struct ReplicationStats { pub replicated_count: u64, } -impl ReplicationStats { +impl ReplicationTargetUsage { pub fn is_empty(&self) -> bool { let Self { pending_size, @@ -636,7 +639,7 @@ impl ReplicationStats { /// Replication statistics for all targets #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct ReplicationAllStats { - pub targets: HashMap, + pub targets: HashMap, pub replica_size: u64, pub replica_count: u64, } @@ -649,7 +652,7 @@ impl ReplicationAllStats { targets, } = self; - *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty) + *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty) } #[deprecated(note = "use is_empty instead")] @@ -2466,7 +2469,7 @@ mod tests { #[test] fn replication_stats_empty_checks_every_field() { - type SetField = fn(&mut ReplicationStats); + type SetField = fn(&mut ReplicationTargetUsage); let cases: [(&str, SetField); 10] = [ ("pending_size", |stats| stats.pending_size = 1), @@ -2481,9 +2484,9 @@ mod tests { ("replicated_count", |stats| stats.replicated_count = 1), ]; - assert!(ReplicationStats::default().is_empty()); + assert!(ReplicationTargetUsage::default().is_empty()); for (field, set_nonzero) in cases { - let mut stats = ReplicationStats::default(); + let mut stats = ReplicationTargetUsage::default(); set_nonzero(&mut stats); assert!(!stats.is_empty(), "{field} must make replication stats non-empty"); } @@ -2514,17 +2517,17 @@ mod tests { } let empty_targets = ReplicationAllStats { - targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]), + targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]), ..Default::default() }; assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty"); let stats = ReplicationAllStats { targets: HashMap::from([ - ("arn:test:empty".to_string(), ReplicationStats::default()), + ("arn:test:empty".to_string(), ReplicationTargetUsage::default()), ( "arn:test:non-empty".to_string(), - ReplicationStats { + ReplicationTargetUsage { pending_count: 1, ..Default::default() }, @@ -2565,7 +2568,7 @@ mod tests { replication_stats: Some(ReplicationAllStats { targets: HashMap::from([( "arn:test:pending".to_string(), - ReplicationStats { + ReplicationTargetUsage { pending_count: 1, ..Default::default() }, @@ -2714,7 +2717,7 @@ mod tests { targets: HashMap::from([ ( "arn:self-only".to_string(), - ReplicationStats { + ReplicationTargetUsage { pending_size: 7, pending_count: 1, ..Default::default() @@ -2722,7 +2725,7 @@ mod tests { ), ( "arn:shared".to_string(), - ReplicationStats { + ReplicationTargetUsage { failed_size: 3, failed_count: 1, missed_threshold_size: 2, @@ -2741,7 +2744,7 @@ mod tests { targets: HashMap::from([ ( "arn:shared".to_string(), - ReplicationStats { + ReplicationTargetUsage { failed_size: 5, failed_count: 2, after_threshold_size: 4, @@ -2751,7 +2754,7 @@ mod tests { ), ( "arn:other-only".to_string(), - ReplicationStats { + ReplicationTargetUsage { replicated_size: 11, replicated_count: 3, ..Default::default() @@ -2993,7 +2996,9 @@ mod tests { fn replication_target_deserialization_preserves_large_historical_maps() { let mut stats = ReplicationAllStats::default(); for index in 0..=1024 { - stats.targets.insert(format!("target-{index}"), ReplicationStats::default()); + stats + .targets + .insert(format!("target-{index}"), ReplicationTargetUsage::default()); } let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode"); let decoded = rmp_serde::from_slice::(&encoded) @@ -3002,6 +3007,47 @@ mod tests { assert_eq!(decoded.targets.len(), stats.targets.len()); } + /// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back + /// must produce the exact same value. This guards against accidental serde + /// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage` + /// rename. Wire-level field names are the serialized Rust field identifiers, + /// which must remain byte-identical. + #[test] + fn replication_target_usage_rmp_round_trip() { + let original = ReplicationTargetUsage { + pending_size: 100, + replicated_size: 2_000, + failed_size: 50, + failed_count: 3, + pending_count: 7, + missed_threshold_size: 11, + after_threshold_size: 22, + missed_threshold_count: 1, + after_threshold_count: 2, + replicated_count: 99, + }; + + let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack"); + let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack"); + assert_eq!(original, decoded, "round-trip through rmp must preserve every field"); + + // Also verify that encoding as an unnamed sequence and then decoding + // with named fields produces the correct mapping (this catches reordering). + let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning"); + // Spot-check that known field names appear in the named encoding. + let named_str = String::from_utf8_lossy(&named_buf); + assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename"); + assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename"); + assert!( + named_str.contains("missed_threshold_size"), + "field 'missed_threshold_size' must survive the rename" + ); + assert!( + named_str.contains("after_threshold_count"), + "field 'after_threshold_count' must survive the rename" + ); + } + #[test] fn checked_merge_rejects_noncanonical_histograms_without_mutation() { let mut entry = DataUsageEntry { diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index ed3fcd544..bdccb11c1 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt; use super::*; use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO}; use crate::{ScannerGetObjectReader, ScannerPutObjReader}; -use rustfs_data_usage::{ReplicationAllStats, ReplicationStats}; +use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage}; use serde_json::Value; use std::io::Cursor; use std::pin::Pin; @@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() { replication_stats: Some(ReplicationAllStats { targets: HashMap::from([( "arn:test:threshold".to_string(), - ReplicationStats { + ReplicationTargetUsage { after_threshold_count: 1, ..Default::default() }, diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index 56961c89e..c6acdea1a 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use rustfs_data_usage::{ReplicationAllStats, ReplicationStats}; +use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage}; const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]); @@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() { replication_stats: Some(ReplicationAllStats { targets: HashMap::from([( "arn:target".to_string(), - ReplicationStats { + ReplicationTargetUsage { replicated_size: 2048, replicated_count: 2, ..Default::default() From 0e79106c2ff7d3f3c289030904c1188348c943a2 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 22 Aug 2026 22:05:20 +0800 Subject: [PATCH 09/32] fix(storageclass): use div_ceil for inline threshold to match shard size calc (#6390) The inline_block threshold used floor division (DEFAULT_INLINE_OBJECT_BUDGET / data_shards) while shard_file_size uses ceiling division (div_ceil). For EC 12:4 with 256KiB objects, this caused a 1-byte discrepancy: - inline_block = 262144 / 12 = 21845 (floor) - shard_file_size = 262144.div_ceil(12) = 21846 (ceil) - should_inline(21846, 12, false) = false (wrong!) Fix by using div_ceil for the inline_block calculation, so both sides use the same rounding and the inline path is correctly triggered. Co-authored-by: heihutu --- crates/ecstore/src/config/storageclass.rs | 5 ++++- crates/ecstore/src/set_disk/ops/object.rs | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index 043bf508e..361b15702 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -248,10 +248,13 @@ impl Config { let shard_size = shard_size as usize; // Keep the historical two-data-shard object budget while preventing // wider EC layouts from multiplying the maximum inline object size. + // Use div_ceil to match the shard_file_size calculation (which also uses + // div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline + // for objects right at the threshold. let inline_block = if self.initialized && self.inline_block_explicit { self.inline_block } else { - (DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK) + DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK) }; if versioned { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index faacca3a8..f5a55a6b4 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -2123,14 +2123,27 @@ impl SetDisks { let erasure = Arc::new(erasure_from_file_info(&fi, false)?); let put_object_size = known_put_object_storage_size(data.size()); + let shard_file_size_raw = erasure.shard_file_size(put_object_size); let is_inline_buffer = - storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned); + storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned); let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled(); - let shard_file_size = erasure.shard_file_size(put_object_size); + let shard_file_size = shard_file_size_raw; let shard_size = erasure.shard_size(); let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size); let direct_inline_commit = matches!(write_path, SmallWritePath::Inline); + { + use std::io::Write; + let msg = format!( + "INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n", + bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size, + is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned + ); + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") { + let _ = f.write_all(msg.as_bytes()); + } + let _ = std::io::stderr().write_all(msg.as_bytes()); + } rustfs_io_metrics::record_put_object_path(write_path.metric_label()); let writer_setup_stage_start = collect_stage_timing.then(Instant::now); let (mut writers, errors) = if direct_inline_commit { From 9815694301f447a20ab783813f41487b6e6b2638 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 22 Aug 2026 22:33:24 +0800 Subject: [PATCH 10/32] fix(ecstore): supervise decommission worker cleanup (#6372) --- crates/ecstore/src/core/pools.rs | 1092 +++++++++++++++++++++++------- crates/ecstore/src/store/mod.rs | 4 +- 2 files changed, 842 insertions(+), 254 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 04a94cc05..66fe2bbe9 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -58,7 +58,6 @@ use http::HeaderMap; #[cfg(test)] use rmp_serde::Deserializer; use rmp_serde::Serializer; -use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; @@ -66,13 +65,14 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, Replicatio use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; +use std::future::Future; #[cfg(test)] use std::io::Cursor; use std::io::Write; use std::path::PathBuf; use std::sync::{ Arc, - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, }; use time::{Duration, OffsetDateTime}; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -96,6 +96,7 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4; const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); +const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1); /// Background decommission walks must tolerate slow object migrations; the /// stall timeout is the drive-health bound, not the total listing duration. const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); @@ -104,6 +105,74 @@ pub const POOL_META_NAME: &str = "pool.bin"; pub const POOL_META_FORMAT: u16 = 1; pub const POOL_META_VERSION: u16 = 1; +#[derive(Clone, Debug)] +pub struct DecommissionCanceler { + operation: Arc, +} + +#[derive(Debug)] +struct DecommissionOperation { + token: CancellationToken, + active: AtomicBool, +} + +impl DecommissionCanceler { + fn new(token: CancellationToken) -> Self { + Self { + operation: Arc::new(DecommissionOperation { + token, + active: AtomicBool::new(true), + }), + } + } + + fn token(&self) -> &CancellationToken { + &self.operation.token + } + + fn is_active(&self) -> bool { + self.operation.active.load(Ordering::Acquire) + } + + #[cfg(test)] + fn is_cancelled(&self) -> bool { + self.token().is_cancelled() + } + + fn cancel(&self) { + self.token().cancel(); + } + + fn release(&self) { + self.cancel(); + self.operation.active.store(false, Ordering::Release); + } + + fn owns_same_operation(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.operation, &other.operation) + } +} + +struct DecommissionCancelerGuard { + canceler: DecommissionCanceler, +} + +impl DecommissionCancelerGuard { + fn new(canceler: DecommissionCanceler) -> Self { + Self { canceler } + } + + fn canceler(&self) -> &DecommissionCanceler { + &self.canceler + } +} + +impl Drop for DecommissionCancelerGuard { + fn drop(&mut self) { + self.canceler.release(); + } +} + fn dedup_indices(indices: &[usize]) -> Vec { let mut seen = HashSet::with_capacity(indices.len()); let mut output = Vec::with_capacity(indices.len()); @@ -119,18 +188,18 @@ fn dedup_indices(indices: &[usize]) -> Vec { fn bind_decommission_cancelers( indices: &[usize], parent: &CancellationToken, - cancelers: &mut [Option], -) -> Vec<(usize, CancellationToken)> { + cancelers: &mut [Option], +) -> Vec<(usize, DecommissionCanceler)> { let mut bound = Vec::with_capacity(indices.len()); for idx in indices { if let Some(slot) = cancelers.get_mut(*idx) { if let Some(existing) = slot.take() { - existing.cancel(); + existing.release(); } - let token = parent.child_token(); - *slot = Some(token.clone()); - bound.push((*idx, token)); + let canceler = DecommissionCanceler::new(parent.child_token()); + *slot = Some(canceler.clone()); + bound.push((*idx, canceler)); } } @@ -140,47 +209,104 @@ fn bind_decommission_cancelers( fn bind_missing_decommission_cancelers( indices: &[usize], parent: &CancellationToken, - cancelers: &mut [Option], -) -> Vec<(usize, CancellationToken)> { + cancelers: &mut [Option], +) -> Vec<(usize, DecommissionCanceler)> { let mut bound = Vec::with_capacity(indices.len()); for idx in indices { let Some(slot) = cancelers.get_mut(*idx) else { continue; }; - if slot.is_some() { + if slot.as_ref().is_some_and(DecommissionCanceler::is_active) { break; } - let token = parent.child_token(); - *slot = Some(token.clone()); - bound.push((*idx, token)); + if let Some(stale) = slot.take() { + stale.release(); + } + let canceler = DecommissionCanceler::new(parent.child_token()); + *slot = Some(canceler.clone()); + bound.push((*idx, canceler)); } bound } -fn take_decommission_canceler(cancelers: &mut [Option], idx: usize) -> Option { +fn take_decommission_canceler(cancelers: &mut [Option], idx: usize) -> Option { cancelers.get_mut(idx).and_then(Option::take) } -fn has_active_decommission_canceler(cancelers: &[Option]) -> bool { - cancelers.iter().any(Option::is_some) +fn take_decommission_canceler_for_operation( + cancelers: &mut [Option], + idx: usize, + owner: &DecommissionCanceler, +) -> Option { + let slot = cancelers.get_mut(idx)?; + if slot.as_ref().is_some_and(|canceler| canceler.owns_same_operation(owner)) { + slot.take() + } else { + None + } } -fn cancel_decommission_canceler(canceler: Option) -> bool { +fn decommission_canceler_is_owned_by( + cancelers: &[Option], + idx: usize, + owner: &DecommissionCanceler, +) -> bool { + cancelers + .get(idx) + .and_then(Option::as_ref) + .is_some_and(|canceler| canceler.owns_same_operation(owner)) +} + +fn update_decommission_for_operation( + cancelers: &[Option], + pool_meta: &mut PoolMeta, + idx: usize, + owner: Option<&DecommissionCanceler>, + update: impl FnOnce(&mut PoolMeta) -> T, +) -> Option { + if let Some(owner) = owner + && !decommission_canceler_is_owned_by(cancelers, idx, owner) + { + owner.release(); + return None; + } + + Some(update(pool_meta)) +} + +fn has_active_decommission_canceler(cancelers: &[Option]) -> bool { + cancelers.iter().flatten().any(DecommissionCanceler::is_active) +} + +fn cancel_decommission_canceler(canceler: Option) -> bool { if let Some(canceler) = canceler { - canceler.cancel(); + canceler.release(); true } else { false } } -fn take_and_cancel_decommission_canceler(cancelers: &mut [Option], idx: usize) -> bool { +fn take_and_cancel_decommission_canceler(cancelers: &mut [Option], idx: usize) -> bool { let canceler = take_decommission_canceler(cancelers, idx); cancel_decommission_canceler(canceler) } +fn take_and_cancel_decommission_canceler_for_operation( + cancelers: &mut [Option], + idx: usize, + owner: &DecommissionCanceler, +) -> bool { + let canceler = take_decommission_canceler_for_operation(cancelers, idx, owner); + if canceler.is_none() { + owner.release(); + return false; + } + cancel_decommission_canceler(canceler) +} + fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> { if bound_count == 0 || bound_count != expected_count { return Err(Error::other(format!( @@ -191,6 +317,36 @@ fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: us Ok(()) } +fn guard_decommission_cancelers(index_cancelers: Vec<(usize, DecommissionCanceler)>) -> Vec<(usize, DecommissionCancelerGuard)> { + index_cancelers + .into_iter() + .map(|(idx, canceler)| (idx, DecommissionCancelerGuard::new(canceler))) + .collect() +} + +async fn await_decommission_worker(idx: usize, worker: tokio::task::JoinHandle>) -> Result<()> { + worker + .await + .map_err(|err| Error::other(format!("decommission worker {idx} task join error: {err}")))? +} + +fn reserve_decommission_start_cancelers( + pool_meta: &PoolMeta, + indices: &[usize], + local_indices: &[usize], + parent: &CancellationToken, + cancelers: &mut [Option], +) -> Result> { + ensure_decommission_start_pool_states(pool_meta, indices)?; + if local_indices.is_empty() { + return Ok(Vec::new()); + } + let bound = bind_decommission_cancelers(local_indices, parent, cancelers); + let guards = guard_decommission_cancelers(bound); + ensure_decommission_routines_scheduled(guards.len(), local_indices.len())?; + Ok(guards) +} + fn default_decommission_bucket_concurrency(cpu_count: usize) -> usize { cpu_count.clamp(1, DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP) } @@ -309,11 +465,15 @@ fn first_resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec { indices } -fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option]) -> Vec { +fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option]) -> Vec { let mut missing = Vec::with_capacity(indices.len()); for idx in indices { - if cancelers.get(*idx).and_then(Option::as_ref).is_some() { + if cancelers + .get(*idx) + .and_then(Option::as_ref) + .is_some_and(DecommissionCanceler::is_active) + { break; } missing.push(*idx); @@ -360,29 +520,25 @@ fn build_decommission_start_state( fn spawn_decommission_index_cancelers( store: Arc, rx: CancellationToken, - index_cancelers: Vec<(usize, CancellationToken)>, -) { + index_cancelers: Vec<(usize, DecommissionCancelerGuard)>, +) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut stop_queue = false; - for (idx, canceler) in index_cancelers { + for (idx, canceler_guard) in index_cancelers { + let canceler = canceler_guard.canceler().clone(); if stop_queue || rx.is_cancelled() { canceler.cancel(); - if let Err(err) = store.decommission_cancel(idx).await { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "queued_cancel_failed", - error = %err, - "Failed to cancel queued decommission" - ); - } + store.retry_decommission_cancel_for_operation(idx, &canceler).await; continue; } - if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { + let worker = tokio::spawn({ + let store = store.clone(); + let canceler = canceler.clone(); + async move { store.do_decommission_in_routine(canceler, idx).await } + }); + if let Err(err) = await_decommission_worker(idx, worker).await { error!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -392,6 +548,7 @@ fn spawn_decommission_index_cancelers( error = %err, "Decommission routine failed" ); + store.retry_decommission_failed_for_operation(idx, &canceler).await; stop_queue = true; continue; } @@ -401,7 +558,7 @@ fn spawn_decommission_index_cancelers( !should_continue_decommission_queue(&pool_meta, idx) }; } - }); + }) } fn decommission_meta_bucket_options() -> MakeBucketOptions { @@ -700,16 +857,6 @@ fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}"))) } -fn resolve_decommission_spawn_failure_result(spawn_err: Error, rollback_err: Option) -> Error { - if let Some(rollback_err) = rollback_err { - Error::other(format!( - "decommission spawn routines failed: {spawn_err}; rollback failed: {rollback_err}" - )) - } else { - spawn_err - } -} - fn decommission_item_size(size: T) -> usize where usize: TryFrom, @@ -2747,7 +2894,10 @@ impl ECStore { let active_workers = { let cancelers = self.decommission_cancelers.read().await; - cancelers.iter().map(Option::is_some).collect::>() + cancelers + .iter() + .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)) + .collect::>() }; let mut pool_meta = self.pool_meta.write().await; @@ -2783,9 +2933,98 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn decommission_cancel(&self, idx: usize) -> Result<()> { - ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; + self.decommission_cancel_with_owner(idx, None).await + } - let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta) = { + async fn decommission_cancel_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> { + self.decommission_cancel_with_owner(idx, Some(owner)).await + } + + async fn release_decommission_canceler_slot(&self, idx: usize, owner: &DecommissionCanceler) { + let mut cancelers = self.decommission_cancelers.write().await; + take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner); + } + + async fn decommission_terminal_retryable_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> bool { + let _start_guard = self.start_gate.lock().await; + let mut cancelers = self.decommission_cancelers.write().await; + if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) { + owner.release(); + return false; + } + + let retryable = { + let pool_meta = self.pool_meta.read().await; + pool_meta + .pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled) + }; + if !retryable { + take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner); + } + retryable + } + + async fn retry_decommission_cancel_for_operation(&self, idx: usize, owner: &DecommissionCanceler) { + let mut attempt = 0usize; + loop { + let Err(err) = self.decommission_cancel_for_operation(idx, owner).await else { + return; + }; + if !self.decommission_terminal_retryable_for_operation(idx, owner).await { + return; + } + attempt += 1; + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_save_retry", + terminal = "canceled", + attempt, + error = %err, + "Decommission terminal save will be retried" + ); + tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await; + } + } + + async fn retry_decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) { + let mut attempt = 0usize; + loop { + let Err(err) = self.decommission_failed_for_operation(idx, owner).await else { + return; + }; + if !self.decommission_terminal_retryable_for_operation(idx, owner).await { + return; + } + attempt += 1; + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_save_retry", + terminal = "failed", + attempt, + error = %err, + "Decommission terminal save will be retried" + ); + tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await; + } + } + + async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { + ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; + let _start_guard = self.start_gate.lock().await; + + // Lock order: decommission_cancelers before pool_meta. Holding both makes + // owner validation and the terminal transition one atomic operation. + let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = { + let cancelers = self.decommission_cancelers.read().await; let mut lock = self.pool_meta.write().await; let mut already_canceled = false; let (pool_present, decommission_present, terminal) = if let Some(pool) = lock.pools.get(idx) { @@ -2805,19 +3044,28 @@ impl ECStore { ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?; let previous_pool_meta = lock.clone(); - let changed = lock.decommission_cancel(idx); + let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut lock, idx, owner, |pool_meta| { + pool_meta.decommission_cancel(idx) + }) else { + return Ok(()); + }; + let terminal_canceler = if let Some(owner) = owner { + Some(owner.clone()) + } else { + cancelers.get(idx).and_then(Option::as_ref).cloned() + }; + if let Some(canceler) = terminal_canceler.as_ref() { + canceler.cancel(); + } ( changed, should_retry_decommission_cancel_reload(changed, already_canceled), already_canceled, changed.then_some(previous_pool_meta), + terminal_canceler, ) }; - - let canceled_worker = { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx) - }; + let canceled_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active); if !canceled_worker && !already_canceled { warn!( event = EVENT_DECOMMISSION_STATE, @@ -2838,6 +3086,10 @@ impl ECStore { return Err(err); } + if let Some(canceler) = terminal_canceler.as_ref() { + self.release_decommission_canceler_slot(idx, canceler).await; + } + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_cancel for pool {idx}"); resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; @@ -2849,6 +3101,7 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn clear_decommission(&self, idx: usize) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?; + let _start_guard = self.start_gate.lock().await; let (should_reload_pool_meta, previous_pool_meta) = { let mut pool_meta = self.pool_meta.write().await; @@ -2857,11 +3110,6 @@ impl ECStore { (changed, changed.then_some(previous_pool_meta)) }; - { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); - } - if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta().await { if let Some(previous_pool_meta) = previous_pool_meta { let mut pool_meta = self.pool_meta.write().await; @@ -2870,6 +3118,11 @@ impl ECStore { return Err(err); } + { + let mut cancelers = self.decommission_cancelers.write().await; + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); + } + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("clear_decommission for pool {idx}"); resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; @@ -2936,26 +3189,53 @@ impl ECStore { is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) } + async fn reserve_decommission_routines( + &self, + rx: &CancellationToken, + indices: &[usize], + ) -> Result> { + let indices = dedup_indices(indices); + if indices.is_empty() { + return Ok(Vec::new()); + } + + let _start_guard = self.start_gate.lock().await; + let indices = { + let pool_meta = self.pool_meta.read().await; + first_resumable_decommission_queue_indices(&pool_meta) + .into_iter() + .filter(|idx| indices.contains(idx)) + .collect::>() + }; + if indices.is_empty() { + return Ok(Vec::new()); + } + + let index_cancelers = { + let mut cancelers = self.decommission_cancelers.write().await; + let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice()); + if missing.is_empty() { + return Ok(Vec::new()); + } + let bound = bind_missing_decommission_cancelers(missing.as_slice(), rx, cancelers.as_mut_slice()); + let guards = guard_decommission_cancelers(bound); + ensure_decommission_routines_scheduled(guards.len(), missing.len())?; + guards + }; + Ok(index_cancelers) + } + pub(crate) async fn spawn_decommission_routines( &self, store: Arc, rx: CancellationToken, indices: Vec, ) -> Result<()> { - let indices = dedup_indices(&indices); - if indices.is_empty() { - return Ok(()); + let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; + if !index_cancelers.is_empty() { + std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); } - let index_cancelers = { - let mut cancelers = self.decommission_cancelers.write().await; - bind_decommission_cancelers(indices.as_slice(), &rx, cancelers.as_mut_slice()) - }; - - ensure_decommission_routines_scheduled(index_cancelers.len(), indices.len())?; - - spawn_decommission_index_cancelers(store, rx, index_cancelers); - Ok(()) } @@ -2970,17 +3250,12 @@ impl ECStore { } let rx = CancellationToken::new(); - let index_cancelers = { - let mut cancelers = self.decommission_cancelers.write().await; - let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice()); - bind_missing_decommission_cancelers(missing.as_slice(), &rx, cancelers.as_mut_slice()) - }; - + let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; if index_cancelers.is_empty() { return Ok(()); } - spawn_decommission_index_cancelers(self.clone(), rx, index_cancelers); + std::mem::drop(spawn_decommission_index_cancelers(self.clone(), rx, index_cancelers)); Ok(()) } @@ -3002,28 +3277,10 @@ impl ECStore { let store = require_decommission_store(runtime_sources::object_store_handle(), "start decommission")?; let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?; - - self.start_decommission(indices.clone()).await?; - if let Err(err) = self.spawn_decommission_routines(store, rx, local_indices).await { - let mut rollback_err: Option = None; - for idx in indices { - if let Err(cancel_err) = self.decommission_cancel(idx).await { - error!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "rollback_failed", - error = ?cancel_err, - "Decommission rollback failed after spawn error" - ); - if rollback_err.is_none() { - rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}"))); - } - } - } - return Err(resolve_decommission_spawn_failure_result(err, rollback_err)); - } + let index_cancelers = self + .start_decommission_with_routines(indices, &rx, local_indices.as_slice()) + .await?; + std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); Ok(()) } @@ -3766,24 +4023,24 @@ impl ECStore { Ok(()) } - #[tracing::instrument(skip(self, rx))] - pub async fn do_decommission_in_routine(self: &Arc, rx: CancellationToken, idx: usize) -> Result<()> { - defer!(|| async { - let mut cancelers = self.decommission_cancelers.write().await; - if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "canceler_already_cleared", - "Decommission canceler already cleared" - ); - } - }); + #[tracing::instrument(skip(self, canceler))] + pub async fn do_decommission_in_routine(self: &Arc, canceler: DecommissionCanceler, idx: usize) -> Result<()> { + let rx = canceler.token().clone(); + self.run_decommission_in_routine(rx, idx, &canceler).await + } + async fn run_decommission_in_routine( + self: &Arc, + rx: CancellationToken, + idx: usize, + canceler: &DecommissionCanceler, + ) -> Result<()> { if let Err(err) = self.promote_queued_decommission(idx).await { - resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, canceler).await, + idx, + &err, + )?; return Err(err); } if rx.is_cancelled() { @@ -3802,8 +4059,12 @@ impl ECStore { ); return Ok(()); } - if let Err(err) = self.decommission_cancel(idx).await { - resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + if let Err(err) = self.decommission_cancel_for_operation(idx, canceler).await { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, canceler).await, + idx, + &err, + )?; return Err(err); } return Ok(()); @@ -3862,7 +4123,11 @@ impl ECStore { return Ok(()); } - resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, canceler).await, + idx, + &err, + )?; warn!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -3909,7 +4174,11 @@ impl ECStore { "Decommission completion verification started" ); if let Err(err) = self.check_after_decommission(idx).await { - resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; + resolve_decommission_terminal_mark_result( + self.decommission_failed_for_operation(idx, canceler).await, + "failed", + &cmd_line, + )?; return Err(Error::other(format!( "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}" ))); @@ -3924,7 +4193,11 @@ impl ECStore { state = "marking_completed", "Decommission marking completed state" ); - resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?; + resolve_decommission_terminal_mark_result( + self.complete_decommission_for_operation(idx, canceler).await, + "completed", + &cmd_line, + )?; } DecommissionFinalState::Failed => { warn!( @@ -3936,7 +4209,11 @@ impl ECStore { state = "marking_failed", "Decommission marking failed state" ); - resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; + resolve_decommission_terminal_mark_result( + self.decommission_failed_for_operation(idx, canceler).await, + "failed", + &cmd_line, + )?; } } @@ -3954,63 +4231,97 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn decommission_failed(&self, idx: usize) -> Result<()> { - ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; + self.decommission_failed_with_owner(idx, None).await + } - let (should_reload_pool_meta, previous_pool_meta) = { + async fn decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> { + self.decommission_failed_with_owner(idx, Some(owner)).await + } + + async fn decommission_failed_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { + self.decommission_failed_with_owner_and_save(idx, owner, self.save_current_pool_meta()) + .await + } + + async fn decommission_failed_with_owner_and_save( + &self, + idx: usize, + owner: Option<&DecommissionCanceler>, + save_pool_meta: SaveFuture, + ) -> Result<()> + where + SaveFuture: Future>, + { + ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; + let _start_guard = self.start_gate.lock().await; + + // Lock order: decommission_cancelers before pool_meta. Holding both makes + // owner validation and the terminal transition one atomic operation. + let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { + let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); - let changed = pool_meta.decommission_failed(idx); - (changed, changed.then_some(previous_pool_meta)) + let Some(changed) = + update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| { + pool_meta.decommission_failed(idx) + }) + else { + return Ok(()); + }; + let terminal_canceler = if let Some(owner) = owner { + Some(owner.clone()) + } else { + cancelers.get(idx).and_then(Option::as_ref).cloned() + }; + (changed, changed.then_some(previous_pool_meta), terminal_canceler) }; - { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); - } - - if should_reload_pool_meta { - if let Err(err) = self.save_current_pool_meta().await { - if let Some(previous_pool_meta) = previous_pool_meta { - let mut pool_meta = self.pool_meta.write().await; - rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); - } - return Err(err); + if should_reload_pool_meta && let Err(err) = save_pool_meta.await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); } + return Err(err); + } + if should_reload_pool_meta { { let mut pool_meta = self.pool_meta.write().await; pool_meta.mark_decommission_progress_saved(); } - if let Some(notification_sys) = runtime_sources::notification_sys() { - let stage = format!("decommission_failed for pool {idx}"); - if let Some(err) = observe_decommission_terminal_reload_result( - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), - stage.as_str(), - ) { - if let Err(record_err) = self - .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone()) - .await - { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "terminal_reload_record_failed", - error = %record_err, - original_error = %err, - "Decommission terminal reload failure record failed" - ); - } + } + if let Some(canceler) = terminal_canceler.as_ref() { + self.release_decommission_canceler_slot(idx, canceler).await; + } + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { + let stage = format!("decommission_failed for pool {idx}"); + if let Some(err) = observe_decommission_terminal_reload_result( + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), + stage.as_str(), + ) { + if let Err(record_err) = self + .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone()) + .await + { warn!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_POOLS, pool_index = idx, - state = "terminal_reload_failed", - error = %err, - "Decommission terminal state saved but pool meta reload failed" + state = "terminal_reload_record_failed", + error = %record_err, + original_error = %err, + "Decommission terminal reload failure record failed" ); } + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission terminal state saved but pool meta reload failed" + ); } } @@ -4019,63 +4330,84 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn complete_decommission(&self, idx: usize) -> Result<()> { - ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; + self.complete_decommission_with_owner(idx, None).await + } - let (should_reload_pool_meta, previous_pool_meta) = { + async fn complete_decommission_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> { + self.complete_decommission_with_owner(idx, Some(owner)).await + } + + async fn complete_decommission_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { + ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; + let _start_guard = self.start_gate.lock().await; + + // Lock order: decommission_cancelers before pool_meta. Holding both makes + // owner validation and the terminal transition one atomic operation. + let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { + let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); - let changed = pool_meta.decommission_complete(idx); - (changed, changed.then_some(previous_pool_meta)) + let Some(changed) = + update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| { + pool_meta.decommission_complete(idx) + }) + else { + return Ok(()); + }; + let terminal_canceler = if let Some(owner) = owner { + Some(owner.clone()) + } else { + cancelers.get(idx).and_then(Option::as_ref).cloned() + }; + (changed, changed.then_some(previous_pool_meta), terminal_canceler) }; - { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); - } - - if should_reload_pool_meta { - if let Err(err) = self.save_current_pool_meta().await { - if let Some(previous_pool_meta) = previous_pool_meta { - let mut pool_meta = self.pool_meta.write().await; - rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); - } - return Err(err); + if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta().await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); } + return Err(err); + } + if should_reload_pool_meta { { let mut pool_meta = self.pool_meta.write().await; pool_meta.mark_decommission_progress_saved(); } - if let Some(notification_sys) = runtime_sources::notification_sys() { - let stage = format!("complete_decommission for pool {idx}"); - if let Some(err) = observe_decommission_terminal_reload_result( - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), - stage.as_str(), - ) { - if let Err(record_err) = self - .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone()) - .await - { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "terminal_reload_record_failed", - error = %record_err, - original_error = %err, - "Decommission terminal reload failure record failed" - ); - } + } + if let Some(canceler) = terminal_canceler.as_ref() { + self.release_decommission_canceler_slot(idx, canceler).await; + } + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { + let stage = format!("complete_decommission for pool {idx}"); + if let Some(err) = observe_decommission_terminal_reload_result( + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), + stage.as_str(), + ) { + if let Err(record_err) = self + .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone()) + .await + { warn!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_POOLS, pool_index = idx, - state = "terminal_reload_failed", - error = %err, - "Decommission terminal state saved but pool meta reload failed" + state = "terminal_reload_record_failed", + error = %record_err, + original_error = %err, + "Decommission terminal reload failure record failed" ); } + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission terminal state saved but pool meta reload failed" + ); } } @@ -4189,6 +4521,23 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn start_decommission(&self, indices: Vec) -> Result<()> { + self.start_decommission_inner(indices, None).await.map(|_| ()) + } + + async fn start_decommission_with_routines( + &self, + indices: Vec, + rx: &CancellationToken, + local_indices: &[usize], + ) -> Result> { + self.start_decommission_inner(indices, Some((rx, local_indices))).await + } + + async fn start_decommission_inner( + &self, + indices: Vec, + reservation: Option<(&CancellationToken, &[usize])>, + ) -> Result> { let indices = dedup_indices(&indices); validate_start_decommission_request(&indices, self.single_pool())?; @@ -4231,11 +4580,19 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?; - { + let index_cancelers = if let Some((rx, local_indices)) = reservation { + // Lock order matches terminal transitions: decommission_cancelers + // before pool_meta while start_gate excludes another start. + let mut cancelers = self.decommission_cancelers.write().await; + let pool_meta = self.pool_meta.read().await; + ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; + reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())? + } else { let pool_meta = self.pool_meta.read().await; ensure_decommission_start_pool_states(&pool_meta, &indices)?; ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; - } + Vec::new() + }; let mut space_infos = Vec::with_capacity(indices.len()); for (idx, pi) in all_space_infos.iter().copied() { @@ -4311,7 +4668,7 @@ impl ECStore { return Err(Error::other(format!("{err}; decommission start rollback succeeded"))); } - Ok(()) + Ok(index_cancelers) } async fn get_buckets_to_decommission(&self) -> Result> { @@ -5457,10 +5814,13 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi #[cfg(test)] mod pools_tests { + use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; + use super::record_decommission_entry_error; + use super::resolve_decommission_listing_error; use super::{ - DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF, - DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, - PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, + DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, + DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, + PoolStatus, apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, @@ -5470,35 +5830,36 @@ mod pools_tests { ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, get_by_index, - has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, + guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, merge_pool_status_refresh, missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, - pool_meta_has_active_decommission, record_decommission_entry_error, require_decommission_store, + pool_meta_has_active_decommission, require_decommission_store, reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, - resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_error, + resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, resolve_decommission_partial_listing_entry, resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, - resolve_decommission_spawn_failure_result, resolve_decommission_terminal_mark_after_error_result, - resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, - resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, - run_decommission_buckets_bounded, run_decommission_listing_with_retry, should_cleanup_decommission_source_entry, - should_continue_decommission_queue, should_count_decommission_version_complete, + resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, + resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, + rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, + should_cleanup_decommission_source_entry, should_continue_decommission_queue, should_count_decommission_version_complete, should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, - split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler, - track_decommission_current_object, track_decommission_current_object_stage, validate_start_decommission_request, - wait_decommission_listing_retry, wait_decommission_worker_drain, with_decommission_entry_context, + spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler, + take_decommission_canceler, track_decommission_current_object, track_decommission_current_object_stage, + update_decommission_for_operation, validate_start_decommission_request, wait_decommission_listing_retry, + wait_decommission_worker_drain, with_decommission_entry_context, }; use crate::data_movement; use crate::disk::endpoint::Endpoint; use crate::error::{Error, StorageError}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::runtime::instance::InstanceContext; use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; - use rustfs_filemeta::{ - FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo, - }; + use crate::store::ECStore; + use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo}; + use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams}; use rustfs_rio::Index; use std::sync::{ Arc, @@ -5513,6 +5874,24 @@ mod pools_tests { Arc::new(|_| Box::pin(async {})) } + fn decommission_worker_test_store(pool_meta: PoolMeta, cancelers: Vec>) -> Arc { + let ctx = Arc::new(InstanceContext::new()); + let endpoint_pools = EndpointServerPools::default(); + Arc::new(ECStore { + id: uuid::Uuid::new_v4(), + disk_map: std::collections::HashMap::new(), + pools: Vec::new(), + peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()), + pool_meta: tokio::sync::RwLock::new(pool_meta), + rebalance_meta: tokio::sync::RwLock::new(None), + decommission_cancelers: tokio::sync::RwLock::new(cancelers), + start_gate: tokio::sync::Mutex::new(()), + pool_meta_save_gate: tokio::sync::Mutex::new(()), + ctx, + bucket_fence_registry: Arc::default(), + }) + } + fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints { let port = 9000usize + idx; let mut endpoint = @@ -6383,20 +6762,6 @@ mod pools_tests { assert!(message.contains(Error::SlowDown.to_string().as_str())); } - #[test] - fn test_resolve_decommission_spawn_failure_result_keeps_primary_without_rollback_error() { - let err = resolve_decommission_spawn_failure_result(Error::SlowDown, None); - assert!(matches!(err, Error::SlowDown)); - } - - #[test] - fn test_resolve_decommission_spawn_failure_result_wraps_rollback_error() { - let err = resolve_decommission_spawn_failure_result(Error::SlowDown, Some(Error::OperationCanceled)); - let message = err.to_string(); - assert!(message.contains("decommission spawn routines failed")); - assert!(message.contains("rollback failed")); - } - #[test] fn test_decommission_item_size_converts_positive_values() { assert_eq!(decommission_item_size(42_i64), 42); @@ -8127,7 +8492,7 @@ mod pools_tests { #[test] fn test_bind_decommission_cancelers_replaces_existing_slot() { let parent = CancellationToken::new(); - let existing = CancellationToken::new(); + let existing = DecommissionCanceler::new(CancellationToken::new()); let mut cancelers = vec![Some(existing.clone())]; let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice()); @@ -8144,7 +8509,7 @@ mod pools_tests { #[test] fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() { let parent = CancellationToken::new(); - let existing = CancellationToken::new(); + let existing = DecommissionCanceler::new(CancellationToken::new()); let mut cancelers = vec![None, Some(existing.clone()), None]; let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice()); @@ -8157,6 +8522,38 @@ mod pools_tests { assert!(!existing.is_cancelled()); } + #[test] + fn test_serialized_decommission_double_start_preserves_first_operation() { + let mut pool_meta = PoolMeta { + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + let first_parent = CancellationToken::new(); + let second_parent = CancellationToken::new(); + let mut cancelers = vec![None, None]; + + let first = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &first_parent, cancelers.as_mut_slice()) + .expect("first start should reserve its worker"); + pool_meta + .decommission( + 0, + PoolSpaceInfo { + total: 100, + free: 40, + used: 60, + }, + ) + .expect("first start should install active metadata"); + + let second = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &second_parent, cancelers.as_mut_slice()); + + assert!(matches!(second, Err(Error::DecommissionAlreadyRunning))); + let current = cancelers[0].as_ref().expect("first operation should retain the slot"); + assert!(current.owns_same_operation(first[0].1.canceler())); + assert!(current.is_active()); + assert!(!first_parent.is_cancelled()); + } + #[test] fn test_local_decommission_queue_prefix_stops_at_remote_leader() { let endpoints = EndpointServerPools::from(vec![ @@ -8207,7 +8604,7 @@ mod pools_tests { #[test] fn test_missing_decommission_worker_prefix_stops_at_active_worker() { - let cancelers = vec![None, Some(CancellationToken::new()), None]; + let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new())), None]; let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice()); @@ -8316,8 +8713,8 @@ mod pools_tests { #[test] fn test_take_decommission_canceler_takes_and_clears_slot() { - let token = CancellationToken::new(); - let mut cancelers = vec![Some(token)]; + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let mut cancelers = vec![Some(canceler)]; let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0); assert!(taken.is_some()); @@ -8326,13 +8723,13 @@ mod pools_tests { #[test] fn test_take_decommission_canceler_returns_none_for_missing_slot() { - let mut cancelers: Vec> = Vec::new(); + let mut cancelers: Vec> = Vec::new(); assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none()); } #[test] fn test_has_active_decommission_canceler_true_when_any_slot_present() { - let cancelers = vec![None, Some(CancellationToken::new())]; + let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new()))]; assert!(has_active_decommission_canceler(cancelers.as_slice())); } @@ -8344,11 +8741,12 @@ mod pools_tests { #[test] fn test_cancel_decommission_canceler_cancels_when_present() { - let token = CancellationToken::new(); - let canceled = cancel_decommission_canceler(Some(token.clone())); + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let canceled = cancel_decommission_canceler(Some(canceler.clone())); assert!(canceled); - assert!(token.is_cancelled()); + assert!(canceler.is_cancelled()); + assert!(!canceler.is_active()); } #[test] @@ -8358,12 +8756,13 @@ mod pools_tests { #[test] fn test_take_and_cancel_decommission_canceler_clears_slot() { - let token = CancellationToken::new(); - let mut cancelers = vec![Some(token.clone())]; + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let mut cancelers = vec![Some(canceler.clone())]; assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0)); assert!(cancelers[0].is_none()); - assert!(token.is_cancelled()); + assert!(canceler.is_cancelled()); + assert!(!canceler.is_active()); } #[test] @@ -8374,6 +8773,195 @@ mod pools_tests { assert!(cancelers[0].is_none()); } + #[test] + fn test_guarded_decommission_future_releases_without_first_poll() { + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let cancelers = vec![Some(canceler.clone())]; + let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]); + let unpolled = async move { + let _guards = guards; + std::future::pending::<()>().await; + }; + + drop(unpolled); + + assert!(canceler.is_cancelled()); + assert!(!has_active_decommission_canceler(cancelers.as_slice())); + } + + #[test] + fn test_partial_decommission_spawn_reservation_releases_bound_slot() { + let parent = CancellationToken::new(); + let mut cancelers = vec![None]; + let bound = bind_decommission_cancelers(&[0, 1], &parent, cancelers.as_mut_slice()); + let guards = guard_decommission_cancelers(bound); + + let result = super::ensure_decommission_routines_scheduled(guards.len(), 2); + drop(guards); + + assert!(result.is_err()); + assert!(!has_active_decommission_canceler(cancelers.as_slice())); + } + + #[tokio::test] + async fn test_decommission_supervisor_observes_worker_abort() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let worker = tokio::spawn(async move { + started_tx.send(()).expect("worker start should be observed"); + std::future::pending::<()>().await; + #[allow(unreachable_code)] + Ok(()) + }); + + started_rx.await.expect("worker start should be observed"); + worker.abort(); + let err = await_decommission_worker(3, worker) + .await + .expect_err("supervisor should observe aborted worker"); + + assert!(err.to_string().contains("decommission worker 3 task join error")); + } + + #[tokio::test] + async fn test_decommission_supervisor_observes_worker_panic() { + let worker = tokio::spawn(async move { + panic!("injected decommission worker panic"); + #[allow(unreachable_code)] + Ok(()) + }); + + let err = await_decommission_worker(4, worker) + .await + .expect_err("supervisor should observe panicked worker"); + + assert!(err.to_string().contains("decommission worker 4 task join error")); + } + + #[tokio::test] + async fn test_decommission_worker_metadata_missing_releases_owned_slot() { + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]); + canceler.cancel(); + + let err = store + .do_decommission_in_routine(canceler.clone(), 0) + .await + .expect_err("missing worker metadata should fail the routine"); + + assert!(err.to_string().contains("target pool was not found")); + assert!(!canceler.is_active()); + assert!(store.decommission_cancelers.read().await[0].is_none()); + } + + #[tokio::test] + async fn test_decommission_supervisor_failure_cancels_queued_successor() { + let first = DecommissionCanceler::new(CancellationToken::new()); + let queued = DecommissionCanceler::new(CancellationToken::new()); + let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]); + let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]); + + spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards) + .await + .expect("decommission supervisor should finish after queued cleanup"); + + assert!(!first.is_active()); + assert!(!queued.is_active()); + assert!(queued.is_cancelled()); + assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none)); + } + + #[tokio::test] + async fn test_decommission_failed_save_failure_preserves_owner_until_retry_succeeds() { + let canceler = DecommissionCanceler::new(CancellationToken::new()); + let pool_meta = PoolMeta { + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }), + )], + ..Default::default() + }; + let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]); + + store + .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Err(Error::SlowDown) }) + .await + .expect_err("injected terminal save failure should be returned"); + + { + let cancelers = store.decommission_cancelers.read().await; + let current = cancelers[0].as_ref().expect("failed save must retain the exact owner slot"); + assert!(current.owns_same_operation(&canceler)); + assert!(current.is_active()); + } + { + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("rollback must retain active decommission metadata"); + assert!(info.has_decommission_state()); + assert!(!info.failed); + assert!(!info.complete); + assert!(!info.canceled); + } + assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await); + + store + .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Ok(()) }) + .await + .expect("terminal retry should commit"); + + let pool_meta = store.pool_meta.read().await; + assert!( + pool_meta.pools[0] + .decommission + .as_ref() + .expect("terminal metadata should remain") + .failed + ); + drop(pool_meta); + assert!(store.decommission_cancelers.read().await[0].is_none()); + assert!(!canceler.is_active()); + assert!(canceler.is_cancelled()); + } + + #[test] + fn test_stale_decommission_operation_cannot_cancel_replacement() { + let stale = DecommissionCanceler::new(CancellationToken::new()); + let replacement = DecommissionCanceler::new(CancellationToken::new()); + let cancelers = vec![Some(replacement.clone())]; + let mut pool_meta = PoolMeta { + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }), + )], + ..Default::default() + }; + + let changed = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, 0, Some(&stale), |pool_meta| { + pool_meta.decommission_cancel(0) + }); + + assert!(changed.is_none()); + assert!( + !pool_meta.pools[0] + .decommission + .as_ref() + .expect("replacement metadata should remain") + .canceled + ); + assert!(replacement.is_active()); + assert!(!replacement.is_cancelled()); + assert!(!stale.is_active()); + assert!(stale.is_cancelled()); + } + #[test] fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() { assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok()); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 4b8a9cb71..757be0c53 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::cluster::rpc::{RemoteClient, S3PeerSys}; use crate::config::storageclass; -use crate::core::pools::PoolMeta; +use crate::core::pools::{DecommissionCanceler, PoolMeta}; use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; use crate::error::{Error, Result}; @@ -176,7 +176,7 @@ pub struct ECStore { // pub local_disks: Vec, pub pool_meta: RwLock, pub rebalance_meta: RwLock>, - pub decommission_cancelers: RwLock>>, + pub decommission_cancelers: RwLock>>, /// Serializes rebalance/decommission start transitions. /// /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`, From 2fccfdeabe9a28a66a36c8787edbc09d8bd0ebe2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 22 Aug 2026 22:38:18 +0800 Subject: [PATCH 11/32] fix(scanner): fence timed out scan cycles (#6352) * fix(scanner): fence timed out scan cycles * fix(scanner): cancel scan workers with cycle scope * fix(scanner): reject persisted timer overflow * fix(scanner): reject terminal leadership epochs * fix(scanner): reject trailing cycle state bytes --- crates/common/src/metrics.rs | 56 +++++ crates/config/README.md | 6 + crates/config/src/constants/scanner.rs | 9 +- .../ecstore/src/services/metrics_realtime.rs | 12 + crates/madmin/src/metrics.rs | 16 ++ crates/scanner/src/runtime_config.rs | 118 ++++++++-- crates/scanner/src/scanner.rs | 217 ++++++++++++++++-- crates/scanner/src/scanner/cycle_state.rs | 60 +++++ crates/scanner/src/scanner/tests.rs | 189 ++++++++++++++- crates/scanner/src/scanner_budget.rs | 161 +++++++++++-- crates/scanner/src/scanner_io.rs | 1 + crates/scanner/src/scanner_io/io_cache.rs | 8 +- crates/scanner/src/scanner_io/io_cycle.rs | 4 +- crates/utils/src/envs.rs | 4 +- docs/operations/scanner-runtime-controls.md | 21 +- 15 files changed, 803 insertions(+), 79 deletions(-) diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 51eef967a..813e39ac3 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -901,6 +901,10 @@ pub struct Metrics { scanner_cycle_max_duration_millis: AtomicU64, scanner_cycle_max_objects: AtomicU64, scanner_cycle_max_directories: AtomicU64, + scanner_cycle_timeout_total: AtomicU64, + scanner_cycle_recovery_required_total: AtomicU64, + scanner_cycle_last_progress_age_seconds: AtomicU64, + scanner_leader_lease_without_progress: AtomicBool, scanner_bitrot_cycle_enabled: AtomicBool, scanner_bitrot_cycle_millis: AtomicU64, scanner_checkpoint: Mutex>, @@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport { #[serde(default)] pub cycle_max_directories: u64, #[serde(default)] + pub cycle_timeout_total: u64, + #[serde(default)] + pub cycle_recovery_required_total: u64, + #[serde(default)] + pub cycle_last_progress_age: u64, + #[serde(default)] + pub leader_lease_without_progress: bool, + #[serde(default)] pub bitrot_cycle_enabled: bool, #[serde(default)] pub bitrot_cycle_seconds: f64, @@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total"; const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds"; const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds"; +const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total"; +const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age"; +const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress"; fn scan_cycle_result_label(result: u8) -> &'static str { match result { @@ -1913,6 +1928,10 @@ impl Metrics { scanner_cycle_max_duration_millis: AtomicU64::new(0), scanner_cycle_max_objects: AtomicU64::new(0), scanner_cycle_max_directories: AtomicU64::new(0), + scanner_cycle_timeout_total: AtomicU64::new(0), + scanner_cycle_recovery_required_total: AtomicU64::new(0), + scanner_cycle_last_progress_age_seconds: AtomicU64::new(0), + scanner_leader_lease_without_progress: AtomicBool::new(false), scanner_bitrot_cycle_enabled: AtomicBool::new(false), scanner_bitrot_cycle_millis: AtomicU64::new(0), scanner_checkpoint: Mutex::new(None), @@ -2412,12 +2431,29 @@ impl Metrics { .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed); self.scanner_cycle_max_directories .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed); + self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed); + self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed); + metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0); + metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0); self.scanner_bitrot_cycle_enabled .store(bitrot_cycle.is_some(), Ordering::Relaxed); self.scanner_bitrot_cycle_millis .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed); } + pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) { + self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed); + if recovery_required { + self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed); + } + self.scanner_cycle_last_progress_age_seconds + .store(progress_age.as_secs(), Ordering::Relaxed); + self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed); + metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1); + metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64()); + metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0); + } + pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option, queued: Option, active: Option) { if let Some(concurrency_limit) = concurrency_limit { self.scanner_set_scan_concurrency_limit @@ -3265,6 +3301,10 @@ impl Metrics { m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0; m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed); m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed); + m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed); + m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed); + m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed); + m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed); m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed); m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0; m.scan_checkpoint = match self.scanner_checkpoint.lock() { @@ -4926,4 +4966,20 @@ mod tests { assert!(!report.bitrot_cycle_enabled); assert_eq!(report.bitrot_cycle_seconds, 0.0); } + + #[tokio::test] + async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() { + let metrics = Metrics::new(); + metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17)); + let timed_out = metrics.report().await; + assert_eq!(timed_out.cycle_timeout_total, 1); + assert_eq!(timed_out.cycle_last_progress_age, 17); + assert!(timed_out.leader_lease_without_progress); + + metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None); + let current = metrics.report().await; + assert_eq!(current.cycle_timeout_total, 1); + assert_eq!(current.cycle_last_progress_age, 0); + assert!(!current.leader_lease_without_progress); + } } diff --git a/crates/config/README.md b/crates/config/README.md index 02cf81d52..338b85dbf 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -84,6 +84,12 @@ Current guidance: - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical) - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical) +Scanner cycle budget controls: + +- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance. +- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`. +- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted. + ## Mmap read environment aliases - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical) diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index 8086c3229..953ef789b 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS"; /// Default scanner speed preset. pub const DEFAULT_SCANNER_SPEED: &str = "default"; -/// Default scanner cycle runtime budget. -/// `0` keeps the existing unbounded per-cycle behavior. -pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0; +/// Default scanner cycle runtime budget when no override is configured. +/// +/// An explicit `0` remains the compatibility escape hatch for an unbounded +/// cycle. Keeping the unset default finite prevents a stalled scanner I/O +/// operation from holding the leader lease forever. +pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60; /// Default scanner per-cycle object budget. /// `0` keeps the existing unbounded per-cycle behavior. diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index e4fc919e6..6e2e5cd7d 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo cycle_max_duration_seconds: metrics.cycle_max_duration_seconds, cycle_max_objects: metrics.cycle_max_objects, cycle_max_directories: metrics.cycle_max_directories, + cycle_timeout_total: metrics.cycle_timeout_total, + cycle_recovery_required_total: metrics.cycle_recovery_required_total, + cycle_last_progress_age: metrics.cycle_last_progress_age, + leader_lease_without_progress: metrics.leader_lease_without_progress, bitrot_cycle_enabled: metrics.bitrot_cycle_enabled, bitrot_cycle_seconds: metrics.bitrot_cycle_seconds, scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport { @@ -611,6 +615,10 @@ mod test { current_started: chrono_to_jiff_timestamp(current_started), last_cycle_partial_source: "usage".to_string(), last_cycle_partial_source_code: 1, + cycle_timeout_total: 3, + cycle_recovery_required_total: 2, + cycle_last_progress_age: 17, + leader_lease_without_progress: true, partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot { source: "usage".to_string(), cycles: 2, @@ -622,6 +630,10 @@ mod test { assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started)); assert_eq!(scanner.last_cycle_partial_source, "usage"); assert_eq!(scanner.last_cycle_partial_source_code, 1); + assert_eq!(scanner.cycle_timeout_total, 3); + assert_eq!(scanner.cycle_recovery_required_total, 2); + assert_eq!(scanner.cycle_last_progress_age, 17); + assert!(scanner.leader_lease_without_progress); let usage = scanner .partial_cycles_by_source .iter() diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 49e1e5ff2..4e55563a5 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -689,6 +689,14 @@ pub struct ScannerMetrics { pub cycle_max_objects: u64, #[serde(rename = "cycle_max_directories", default)] pub cycle_max_directories: u64, + #[serde(rename = "cycle_timeout_total", default)] + pub cycle_timeout_total: u64, + #[serde(rename = "cycle_recovery_required_total", default)] + pub cycle_recovery_required_total: u64, + #[serde(rename = "cycle_last_progress_age", default)] + pub cycle_last_progress_age: u64, + #[serde(rename = "leader_lease_without_progress", default)] + pub leader_lease_without_progress: bool, #[serde(rename = "bitrot_cycle_enabled", default)] pub bitrot_cycle_enabled: bool, #[serde(rename = "bitrot_cycle_seconds", default)] @@ -764,6 +772,8 @@ impl ScannerMetrics { self.cycle_max_duration_seconds = other.cycle_max_duration_seconds; self.cycle_max_objects = other.cycle_max_objects; self.cycle_max_directories = other.cycle_max_directories; + self.cycle_last_progress_age = other.cycle_last_progress_age; + self.leader_lease_without_progress = other.leader_lease_without_progress; self.bitrot_cycle_enabled = other.bitrot_cycle_enabled; self.bitrot_cycle_seconds = other.bitrot_cycle_seconds; } @@ -857,6 +867,12 @@ impl ScannerMetrics { .saturating_add(other.last_cycle_replication_checks); self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves); self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles); + self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total); + self.cycle_recovery_required_total = self + .cycle_recovery_required_total + .saturating_add(other.cycle_recovery_required_total); + self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age); + self.leader_lease_without_progress |= other.leader_lease_without_progress; self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles); self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown); self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime); diff --git a/crates/scanner/src/runtime_config.rs b/crates/scanner/src/runtime_config.rs index 48fb40d2d..2fef6f9f4 100644 --- a/crates/scanner/src/runtime_config.rs +++ b/crates/scanner/src/runtime_config.rs @@ -125,7 +125,10 @@ impl Default for ScannerRuntimeConfig { cycle_interval_source: ScannerRuntimeConfigSource::Default, bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)), bitrot_cycle_source: ScannerRuntimeConfigSource::Default, - cycle_budget: ScannerCycleBudgetConfig::default(), + cycle_budget: ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)), + ..Default::default() + }, cycle_max_duration_source: ScannerRuntimeConfigSource::Default, cycle_max_objects_source: ScannerRuntimeConfigSource::Default, cycle_max_directories_source: ScannerRuntimeConfigSource::Default, @@ -374,7 +377,10 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<() } validate_optional_config_u64(scanner_kvs, SCANNER_START_DELAY, "")?; validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE, "")?; - validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)?; + if let Some(value) = config_value(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) { + let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?; + cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)?; + } validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS)?; validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES)?; if let Some(value) = config_value(heal_kvs, HEAL_BITROT_CYCLE, DEFAULT_HEAL_BITROT_CYCLE_SECS) { @@ -436,19 +442,46 @@ fn lookup_max_wait( Ok((speed.max_sleep(), speed_source)) } -fn lookup_optional_seconds( - kvs: Option<&KVS>, - key: &'static str, - env_key: &'static str, - default: u64, -) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { - if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) { - return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env)); +fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { + match rustfs_utils::get_env_parse_outcome::(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) { + rustfs_utils::EnvParseOutcome::Parsed(secs) => { + return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs) + .map(|duration| (duration, ScannerRuntimeConfigSource::Env)); + } + rustfs_utils::EnvParseOutcome::Invalid => { + // Do not include the raw environment value in the typed error: + // deployments occasionally put sensitive material in inherited + // environment snapshots. The key still identifies the control. + return Err(invalid_value( + ENV_SCANNER_CYCLE_MAX_DURATION_SECS, + "", + "expected unsigned integer seconds", + )); + } + rustfs_utils::EnvParseOutcome::Absent => {} } - if let Some(value) = config_value(kvs, key, default) { - return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config)); + + if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) { + let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?; + return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs) + .map(|duration| (duration, ScannerRuntimeConfigSource::Config)); } - Ok((None, ScannerRuntimeConfigSource::Default)) + + Ok(( + Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)), + ScannerRuntimeConfigSource::Default, + )) +} + +fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result, ScannerRuntimeConfigError> { + if secs == 0 { + return Ok(None); + } + let duration = Duration::from_secs(secs); + if std::time::Instant::now().checked_add(duration).is_none() { + return Err(invalid_value(key, "", "duration exceeds the timer range")); + } + Ok(Some(duration)) } fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { @@ -553,12 +586,7 @@ pub(crate) fn lookup_scanner_runtime_config( (speed.cycle_interval(), speed_source) }; - let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds( - scanner_kvs, - SCANNER_CYCLE_MAX_DURATION, - ENV_SCANNER_CYCLE_MAX_DURATION_SECS, - DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, - )?; + let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?; let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget( scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, @@ -863,10 +891,10 @@ mod tests { use rustfs_config::server_config::{Config as ServerConfig, KVS}; use rustfs_config::{ DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, - ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, - HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, - SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, - SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, + ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, + ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, + SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, + SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, }; use std::collections::HashMap; use std::time::Duration; @@ -941,6 +969,50 @@ mod tests { }); } + #[test] + fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() { + let config = server_config_with_scanner(&[]); + with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800))); + assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default); + }); + + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]); + with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + assert_eq!(resolved.cycle_budget.max_duration, None); + assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config); + }); + } + + #[test] + fn cycle_budget_invalid_or_overflow_config_is_rejected() { + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || { + let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected"); + assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS)); + assert!(error.to_string().contains("")); + assert!(!error.to_string().contains(": invalid (")); + }); + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || { + assert!(lookup_scanner_runtime_config(None).is_err()); + }); + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || { + assert!(lookup_scanner_runtime_config(None).is_err()); + }); + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]); + assert!(lookup_scanner_runtime_config(Some(&config)).is_err()); + } + + #[test] + fn scanner_runtime_config_validation_rejects_overflow_persisted_duration() { + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "18446744073709551615")]); + + let error = validate_scanner_runtime_config(&config) + .expect_err("persisted duration that exceeds the timer range must be rejected"); + assert!(error.to_string().contains(SCANNER_CYCLE_MAX_DURATION)); + } + #[test] fn scanner_runtime_config_normalizes_persisted_default_speed() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 7a4a482cf..059de1fb4 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -52,6 +52,7 @@ use rustfs_config::{ }; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; use rustfs_data_usage::observed_data_usage_is_newer; +use rustfs_lock::NamespaceLockGuard; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; use tokio::sync::{Notify, mpsc}; @@ -1037,20 +1038,116 @@ fn data_usage_persist_timeout() -> Duration { DataUsageCache::persistence_timeout() } +#[cfg(not(test))] +const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] +const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50); + +async fn fence_scanner_epoch_after_cycle_timeout( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + cycle_revision: &mut DataUsageCacheRevision, + leader_epoch: &mut u64, + lock_lost: LockLost, +) -> bool +where + Store: ScannerObjectIO, + LockLost: Future, +{ + let fence_ctx = ctx.child_token(); + let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch); + tokio::pin!(claim); + tokio::pin!(lock_lost); + tokio::select! { + biased; + _ = &mut lock_lost => { + fence_ctx.cancel(); + false + } + result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => { + result.unwrap_or(false) && !fence_ctx.is_cancelled() + } + } +} + +struct ScannerCycleDeadlineState<'a> { + cycle_info: &'a mut CurrentCycle, + cycle_revision: &'a mut DataUsageCacheRevision, + leader_epoch: &'a mut u64, + cycle_budget: &'a ScannerCycleBudget, +} + +fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool { + !worker_stopped || !cycle_state_persisted || !generation_fenced +} + +async fn handle_scanner_cycle_deadline( + ctx: &CancellationToken, + storeapi: Arc, + state: ScannerCycleDeadlineState<'_>, + worker_stopped: bool, + guard: &mut NamespaceLockGuard, +) where + Store: ScannerObjectIO, +{ + let fenced = fence_scanner_epoch_after_cycle_timeout( + ctx, + storeapi, + state.cycle_info, + state.cycle_revision, + state.leader_epoch, + guard.lock_lost_notified(), + ) + .await; + let cycle_state_persisted = state.cycle_budget.cycle_state_persisted(); + let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced); + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cycle_timeout", + worker_stopped, + cycle_state_persisted, + generation_fenced = fenced, + recovery_required, + "Scanner cycle deadline expired; durable cursor/generation fencing completed when possible" + ); + global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age()); + // Stop renewing before releasing the lease. A new leader can then claim the + // higher persisted generation instead of inheriting the expired worker. + guard.release(); + global_metrics().set_cycle(None).await; +} + async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) { cycle_info.current = 0; global_metrics().clear_current_scan_mode(); cycle_metrics_guard.finish(cycle_info.clone()).await; } -#[instrument(skip_all)] -#[hotpath::measure] +#[cfg(test)] async fn run_data_scanner_cycle( ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle, cycle_revision: &mut DataUsageCacheRevision, leader_epoch: u64, +) -> ScannerCycleOutcome { + let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config()); + run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await +} + +#[instrument(skip_all)] +#[hotpath::measure] +async fn run_data_scanner_cycle_with_budget( + ctx: &CancellationToken, + storeapi: &Arc, + cycle_info: &mut CurrentCycle, + cycle_revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_budget: Arc, ) -> ScannerCycleOutcome { let _activity_guard = ScannerActivityGuard::new(); if let Err(err) = refresh_scanner_runtime_config_from_global() { @@ -1066,7 +1163,11 @@ async fn run_data_scanner_cycle( } let configured_cycle_interval = scanner_cycle_interval(); let configured_bitrot_cycle = scanner_bitrot_cycle(); - let cycle_budget_config = scanner_cycle_budget_config(); + let cycle_budget_config = ScannerCycleBudgetConfig { + max_duration: cycle_budget.max_duration(), + max_objects: cycle_budget.max_objects(), + max_directories: cycle_budget.max_directories(), + }; let usage_persist_timeout = data_usage_persist_timeout(); global_metrics().record_scanner_cycle_config( configured_cycle_interval, @@ -1137,7 +1238,6 @@ async fn run_data_scanner_cycle( let (sender, receiver) = mpsc::channel::(1); let done_cycle = Metrics::time(Metric::ScanCycle); - let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config); let scan_result = storeapi .clone() .nsscanner_with_status( @@ -1277,7 +1377,7 @@ async fn run_data_scanner_cycle( "Scanner cycle is recovering to a newer durable cache generation" ); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - return if persist_required_scanner_cycle_floor( + let persisted = persist_required_scanner_cycle_floor( ctx, storeapi.clone(), cycle_info, @@ -1286,8 +1386,9 @@ async fn run_data_scanner_cycle( required_cycle, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1345,7 +1446,7 @@ async fn run_data_scanner_cycle( scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - return if finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle( ctx, storeapi.clone(), cycle_info, @@ -1353,8 +1454,9 @@ async fn run_data_scanner_cycle( leader_epoch, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1429,7 +1531,7 @@ async fn run_data_scanner_cycle( ); } emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - return if finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle( ctx, storeapi.clone(), cycle_info, @@ -1437,8 +1539,9 @@ async fn run_data_scanner_cycle( leader_epoch, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1479,6 +1582,7 @@ async fn run_data_scanner_cycle( ) .await { + cycle_budget.mark_cycle_state_persisted(); emit_scan_cycle_superseded(cycle_start.elapsed()); return ScannerCycleOutcome::Superseded; } @@ -1511,6 +1615,7 @@ async fn run_data_scanner_cycle( emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; } + cycle_budget.mark_cycle_state_persisted(); done_cycle(); emit_scan_cycle_complete(true, cycle_start.elapsed()); @@ -1575,7 +1680,7 @@ async fn run_data_scanner_with_maintenance_state( ) -> Result<(), ScannerError> { reset_scanner_cycle_schedule(); // Acquire leader lock (write lock) to ensure only one scanner runs - let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { + let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await { Ok(guard) => { record_scanner_leader_lock_state("acquired"); @@ -1740,13 +1845,49 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } let cycle_ctx = ctx.child_token(); - let initial_outcome = await_scanner_cycle_with_lock_fence( + let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); + let initial_outcome = match await_scanner_cycle_with_budget_fence( &cycle_ctx, - run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + &cycle_budget, + run_data_scanner_cycle_with_budget( + &cycle_ctx, + &storeapi, + &mut cycle_info, + &mut cycle_revision, + leader_epoch, + cycle_budget.clone(), + ), guard.lock_lost_notified(), ) .await - .unwrap_or(ScannerCycleOutcome::Failed); + { + ScannerCycleWaitOutcome::Completed(outcome) => outcome, + ScannerCycleWaitOutcome::LockLost => { + record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Cancelled => { + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Deadline { worker_stopped } => { + handle_scanner_cycle_deadline( + &ctx, + storeapi.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &cycle_budget, + }, + worker_stopped, + &mut guard, + ) + .await; + return Ok(()); + } + }; superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; @@ -1952,13 +2093,49 @@ async fn run_data_scanner_with_maintenance_state( } let dirty_generation_before_cycle = dirty_usage_generation(); let cycle_ctx = ctx.child_token(); - let outcome = await_scanner_cycle_with_lock_fence( + let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); + let outcome = match await_scanner_cycle_with_budget_fence( &cycle_ctx, - run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + &cycle_budget, + run_data_scanner_cycle_with_budget( + &cycle_ctx, + &storeapi, + &mut cycle_info, + &mut cycle_revision, + leader_epoch, + cycle_budget.clone(), + ), guard.lock_lost_notified(), ) .await - .unwrap_or(ScannerCycleOutcome::Failed); + { + ScannerCycleWaitOutcome::Completed(outcome) => outcome, + ScannerCycleWaitOutcome::LockLost => { + record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Cancelled => { + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Deadline { worker_stopped } => { + handle_scanner_cycle_deadline( + &ctx, + storeapi.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &cycle_budget, + }, + worker_stopped, + &mut guard, + ) + .await; + return Ok(()); + } + }; superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 32c893fdf..6459ae6e5 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -1581,3 +1581,63 @@ where output = &mut cycle => Some(output), } } + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum ScannerCycleWaitOutcome { + Completed(T), + LockLost, + Cancelled, + Deadline { worker_stopped: bool }, +} + +pub(super) async fn await_scanner_cycle_with_budget_fence( + cycle_ctx: &CancellationToken, + budget: &ScannerCycleBudget, + cycle: Cycle, + lock_lost: LockLost, +) -> ScannerCycleWaitOutcome +where + Cycle: Future, + LockLost: Future, +{ + tokio::pin!(cycle); + tokio::pin!(lock_lost); + let deadline = async { + if let Some(deadline) = budget.deadline() { + tokio::time::sleep_until(deadline).await; + } else { + std::future::pending::<()>().await; + } + }; + tokio::pin!(deadline); + tokio::select! { + biased; + _ = &mut lock_lost => { + cycle_ctx.cancel(); + let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await; + ScannerCycleWaitOutcome::LockLost + } + _ = &mut deadline => { + budget.cancel_for_runtime(); + // Let the budget cancellation reach the scanner first so it can + // persist a partial cursor. Only an uncooperative worker gets the + // parent cancellation, and it is dropped after the bounded window; + // the caller fences its epoch next. + let worker_stopped = if tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle) + .await + .is_ok() + { + true + } else { + cycle_ctx.cancel(); + false + }; + ScannerCycleWaitOutcome::Deadline { worker_stopped } + } + _ = cycle_ctx.cancelled() => { + let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await; + ScannerCycleWaitOutcome::Cancelled + } + output = &mut cycle => ScannerCycleWaitOutcome::Completed(output), + } +} diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 5430e9d29..d73d63cde 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -26,6 +26,7 @@ use std::task::Poll; use temp_env::{with_var, with_var_unset}; use tokio::io::AsyncReadExt; use tokio::sync::Mutex; +use tokio::time::{Duration, advance}; const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60; @@ -118,6 +119,178 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { assert!(cycle_ctx.is_cancelled()); } +#[tokio::test(start_paused = true)] +async fn cycle_budget_fences_late_writer_after_timeout() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let outcome = { + let cycle = std::future::pending::<()>(); + let lock_lost = std::future::pending::<()>(); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, cycle, lock_lost); + tokio::pin!(waiter); + tokio::task::yield_now().await; + advance(Duration::from_secs(5)).await; + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + waiter.await + }; + assert_eq!(outcome, ScannerCycleWaitOutcome::Deadline { worker_stopped: false }); + assert!(cycle_ctx.is_cancelled()); + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime)); + + // A newer leadership epoch is the durable fence that rejects a late + // writer after the timed-out future has been dropped. + let store = Arc::new(MemoryConfigStore::default()); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + ..Default::default() + }; + let persist_ctx = CancellationToken::new(); + assert!(persist_scanner_cycle_state(&persist_ctx, store.clone(), &mut cycle, &mut revision, 1).await); + let newer = encode_scanner_cycle_state(&cycle, 2).expect("new epoch fence should encode"); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.interleaving_puts.lock().await.insert(key, (2, newer)); + let mut late_cycle = CurrentCycle { next: 13, ..cycle }; + assert!(!persist_scanner_cycle_state(&persist_ctx, store, &mut late_cycle, &mut revision, 1).await); +} + +#[tokio::test(start_paused = true)] +async fn cycle_budget_parent_cancellation_is_not_reported_as_timeout() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending()); + tokio::pin!(waiter); + tokio::task::yield_now().await; + cycle_ctx.cancel(); + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + assert_eq!(waiter.await, ScannerCycleWaitOutcome::Cancelled); +} + +#[tokio::test(start_paused = true)] +async fn cycle_budget_deadline_wins_same_tick_as_parent_cancellation() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending()); + tokio::pin!(waiter); + tokio::task::yield_now().await; + advance(Duration::from_secs(5)).await; + cycle_ctx.cancel(); + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + + assert_eq!(waiter.await, ScannerCycleWaitOutcome::Deadline { worker_stopped: false }); + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime)); +} + +#[tokio::test] +async fn cycle_budget_persist_cursor_failure_is_recovery_required() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.fail_put_number.lock().await.insert(key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }; + let mut leader_epoch = 1; + let fenced = fence_scanner_epoch_after_cycle_timeout( + &ctx, + store, + &mut cycle, + &mut revision, + &mut leader_epoch, + std::future::pending(), + ) + .await; + assert!(!fenced, "a failed cursor/generation write must require recovery"); + let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); + assert!(cycle_timeout_requires_recovery(true, budget.cycle_state_persisted(), fenced)); + + let metrics = Metrics::new(); + metrics.record_scanner_cycle_timeout(!fenced, Duration::from_secs(17)); + let report = metrics.report().await; + assert_eq!(report.cycle_timeout_total, 1); + assert_eq!(report.cycle_recovery_required_total, 1); + assert_eq!(report.cycle_last_progress_age, 17); + assert!(report.leader_lease_without_progress); +} + +#[tokio::test] +async fn cycle_budget_deadline_handler_fences_and_releases_guard() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let lock = store + .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock") + .await + .expect("scanner leader lock should be created"); + let mut guard = lock + .get_write_lock(Duration::from_secs(1)) + .await + .expect("scanner leader lock should be acquired"); + + let ctx = CancellationToken::new(); + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }; + let mut cycle_revision = DataUsageCacheRevision::Missing; + let mut leader_epoch = 1; + let budget = ScannerCycleBudget::new( + &ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + ..Default::default() + }, + ); + budget.mark_cycle_state_persisted(); + + handle_scanner_cycle_deadline( + &ctx, + store.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &budget, + }, + true, + &mut guard, + ) + .await; + + assert!(guard.is_released()); + let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("deadline handler should persist a fenced cursor"); + let (_, persisted_epoch) = decode_scanner_cycle_state(&persisted).expect("fenced cursor should decode"); + assert_eq!(persisted_epoch, 2); + global_metrics().set_cycle(None).await; +} + #[tokio::test] async fn scanner_cycle_recovery_wake_survives_wait_registration_race() { notify_scanner_cycle_recovery_wake(); @@ -428,13 +601,6 @@ fn test_scanner_cycle_max_duration_uses_env() { }); } -#[test] -fn test_scanner_cycle_max_duration_default_is_disabled() { - with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { - assert_eq!(scanner_cycle_max_duration(), None); - }); -} - #[tokio::test] async fn test_scanner_cycle_budget_cancels_after_duration() { let parent = CancellationToken::new(); @@ -2242,7 +2408,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { } #[tokio::test] -async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { +async fn cycle_budget_lease_takeover_rejects_old_generation() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); let mut revision = DataUsageCacheRevision::Missing; @@ -2287,12 +2453,17 @@ async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { .await ); - let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) .await .expect("replacement leadership claim should persist"); let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode"); assert_eq!(claimed_cycle.next, 14); assert_eq!(claimed_epoch, 2); + + let mut stale_cycle = CurrentCycle { next: 15, ..cycle }; + let mut stale_revision = DataUsageCacheRevision::Etag("memory-2".to_string()); + let stale_ctx = CancellationToken::new(); + assert!(!persist_scanner_cycle_state(&stale_ctx, store, &mut stale_cycle, &mut stale_revision, 1,).await); } #[tokio::test] diff --git a/crates/scanner/src/scanner_budget.rs b/crates/scanner/src/scanner_budget.rs index 43ce732d5..65743439f 100644 --- a/crates/scanner/src/scanner_budget.rs +++ b/crates/scanner/src/scanner_budget.rs @@ -14,17 +14,16 @@ use std::sync::{ Arc, - atomic::{AtomicU8, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, }; -use std::time::Instant; - -use tokio::time::Duration; +use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; const BUDGET_REASON_NONE: u8 = 0; const BUDGET_REASON_RUNTIME: u8 = 1; const BUDGET_REASON_OBJECTS: u8 = 2; const BUDGET_REASON_DIRECTORIES: u8 = 3; +const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct ScannerCycleBudgetConfig { @@ -63,29 +62,51 @@ pub struct ScannerCycleBudget { token: CancellationToken, reason: Arc, started_at: Instant, + deadline: Option, max_duration: Option, max_objects: Option, max_directories: Option, track_progress: bool, + track_unbounded_counts: bool, objects_scanned: AtomicU64, directories_started: AtomicU64, entries_visited: AtomicU64, + last_progress_millis: AtomicU64, + cycle_state_persisted: AtomicBool, } impl ScannerCycleBudget { + #[cfg(test)] pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { - Self::new_inner(parent, config, false) + Self::new_inner(parent, config, false, false) } pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { - Self::new_inner(parent, config, true) + Self::new_inner(parent, config, true, true) } - fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc { + pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { + let track_progress = config.max_duration.is_some(); + Self::new_inner(parent, config, track_progress, false) + } + + fn new_inner( + parent: &CancellationToken, + config: ScannerCycleBudgetConfig, + track_progress: bool, + track_unbounded_counts: bool, + ) -> Arc { let token = parent.child_token(); let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE)); + let started_at = Instant::now(); + let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) { + Some(deadline) => deadline, + // Runtime config rejects this range, but keep programmatic callers + // fail-closed instead of panicking or silently disabling the wall clock. + None => started_at, + }); - if let Some(duration) = config.max_duration { + if let Some(deadline) = deadline { let parent = parent.clone(); let token_wait = token.clone(); let token_cancel = token.clone(); @@ -94,7 +115,7 @@ impl ScannerCycleBudget { tokio::select! { _ = parent.cancelled() => {} _ = token_wait.cancelled() => {} - _ = tokio::time::sleep(duration) => { + _ = tokio::time::sleep_until(deadline) => { Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime); } } @@ -104,14 +125,18 @@ impl ScannerCycleBudget { Arc::new(Self { token, reason, - started_at: Instant::now(), + started_at, + deadline, max_duration: config.max_duration, max_objects: config.max_objects, max_directories: config.max_directories, track_progress, + track_unbounded_counts, objects_scanned: AtomicU64::new(0), directories_started: AtomicU64::new(0), entries_visited: AtomicU64::new(0), + last_progress_millis: AtomicU64::new(0), + cycle_state_persisted: AtomicBool::new(false), }) } @@ -131,6 +156,14 @@ impl ScannerCycleBudget { self.max_duration } + pub(crate) fn deadline(&self) -> Option { + self.deadline + } + + pub(crate) fn cancel_for_runtime(&self) { + self.cancel_for(ScannerCycleBudgetReason::Runtime); + } + pub(crate) fn max_objects(&self) -> Option { self.max_objects } @@ -173,15 +206,43 @@ impl ScannerCycleBudget { self.entries_visited.load(Ordering::Relaxed) } + pub(crate) fn mark_cycle_state_persisted(&self) { + self.cycle_state_persisted.store(true, Ordering::Release); + } + + pub(crate) fn cycle_state_persisted(&self) -> bool { + self.cycle_state_persisted.load(Ordering::Acquire) + } + + pub(crate) fn progress_age(&self) -> Duration { + let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let last_progress = self.last_progress_millis.load(Ordering::Relaxed); + Duration::from_millis(elapsed_millis.saturating_sub(last_progress)) + } + + fn record_progress_sample(&self, event: u64) { + // Clock reads are sampled at batch/count boundaries; the scanner's + // per-object path does not add a second progress atomic. + if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) { + return; + } + let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed); + } + pub(crate) fn record_entries_visited(&self, entries_visited: u64) { if self.track_progress { - saturating_fetch_add(&self.entries_visited, entries_visited); + let entries = saturating_fetch_add(&self.entries_visited, entries_visited); + self.record_progress_sample(entries); } } pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) { if self.track_progress || self.max_objects.is_some() { let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned); + if self.track_progress { + self.record_progress_sample(objects); + } if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { self.cancel_for(ScannerCycleBudgetReason::Objects); } @@ -189,9 +250,12 @@ impl ScannerCycleBudget { if self.track_progress || self.max_directories.is_some() { let directories = saturating_fetch_add(&self.directories_started, directories_started); + if self.track_progress { + self.record_progress_sample(directories); + } if self .max_directories - .is_some_and(|max_directories| directories > max_directories) + .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories)) { self.cancel_for(ScannerCycleBudgetReason::Directories); } @@ -207,14 +271,17 @@ impl ScannerCycleBudget { } pub(crate) fn try_start_directory(&self) -> bool { - if !self.track_progress && self.max_directories.is_none() { + if self.max_directories.is_none() && !self.track_unbounded_counts { return true; } let directories = saturating_fetch_add(&self.directories_started, 1); + if self.track_progress { + self.record_progress_sample(directories); + } if self .max_directories - .is_some_and(|max_directories| directories > max_directories) + .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories)) { self.cancel_for(ScannerCycleBudgetReason::Directories); return false; @@ -224,11 +291,14 @@ impl ScannerCycleBudget { } pub(crate) fn record_object_scanned(&self) { - if !self.track_progress && self.max_objects.is_none() { + if self.max_objects.is_none() && !self.track_unbounded_counts { return; } let objects = saturating_fetch_add(&self.objects_scanned, 1); + if self.track_progress { + self.record_progress_sample(objects); + } if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { self.cancel_for(ScannerCycleBudgetReason::Objects); } @@ -259,6 +329,13 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 { } } +fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool { + // Saturation hides a remote max+1 update when the configured limit is the + // largest representable counter. Treat that boundary as exhausted rather + // than allowing work to continue indefinitely. + directories > max_directories || (directories == u64::MAX && max_directories == u64::MAX) +} + impl Drop for ScannerCycleBudget { fn drop(&mut self) { self.token.cancel(); @@ -401,6 +478,35 @@ mod tests { assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories)); } + #[test] + fn directory_budget_fails_closed_when_progress_saturates() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(u64::MAX), + ..Default::default() + }, + ); + + budget.record_remote_progress(0, u64::MAX); + + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Directories)); + assert!(budget.token().is_cancelled()); + + let local_budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(u64::MAX), + ..Default::default() + }, + ); + local_budget.record_remote_progress(0, u64::MAX - 1); + assert!(!local_budget.budget_elapsed()); + assert!(!local_budget.try_start_directory()); + assert_eq!(local_budget.reason(), Some(ScannerCycleBudgetReason::Directories)); + } + #[test] fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() { let parent = CancellationToken::new(); @@ -461,4 +567,29 @@ mod tests { assert!(object_limited.requires_serial_progress_accounting()); assert!(directory_limited.requires_serial_progress_accounting()); } + + #[tokio::test(start_paused = true)] + async fn progress_age_uses_virtual_time_and_sampled_progress() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_runtime_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + ..Default::default() + }, + ); + + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(budget.progress_age(), Duration::from_secs(5)); + budget.record_entries_visited(1); + assert_eq!(budget.progress_age(), Duration::ZERO); + + tokio::time::advance(Duration::from_secs(2)).await; + for _ in 0..126 { + budget.record_entries_visited(1); + } + assert_eq!(budget.progress_age(), Duration::from_secs(2)); + budget.record_entries_visited(1); + assert_eq!(budget.progress_age(), Duration::ZERO); + } } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 677719a69..c104fb263 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -48,6 +48,7 @@ use time::OffsetDateTime; use tokio::sync::{Mutex, Notify, Semaphore, mpsc}; use tokio::time::Duration; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 3151b4c0c..aa0339e4b 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -314,7 +314,7 @@ impl ScannerIOCache for SetDisks { let ctx_clone = ctx.clone(); let completed_bucket_count = Arc::new(AtomicUsize::new(0)); let completed_bucket_count_clone = completed_bucket_count.clone(); - let collect_bucket_results_fut = tokio::spawn(async move { + let collect_bucket_results_fut = AbortOnDropHandle::new(tokio::spawn(async move { let mut cancelled = false; loop { @@ -333,7 +333,7 @@ impl ScannerIOCache for SetDisks { } } } - }); + })); let mut futs = Vec::new(); @@ -365,7 +365,7 @@ impl ScannerIOCache for SetDisks { NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch), NamespaceScannerWorkerMode::Coordinator => None, }; - futs.push(tokio::spawn(async move { + futs.push(AbortOnDropHandle::new(tokio::spawn(async move { let remote_session_id = uuid::Uuid::new_v4(); let mut remote_session_sequence = 0_u64; loop { @@ -1038,7 +1038,7 @@ impl ScannerIOCache for SetDisks { ); } } - })); + }))); } drop(bucket_tx); drop(bucket_result_tx); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index c2a8eb253..64763655f 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -242,7 +242,7 @@ impl ScannerIOCycle for ECStore { results[results_index_clone] = result; } }); - wait_futs.push(receiver_fut); + wait_futs.push(AbortOnDropHandle::new(receiver_fut)); let scan_plan = ScannerBucketScanPlan { buckets: set_buckets, @@ -318,7 +318,7 @@ impl ScannerIOCycle for ECStore { record_set_scan_failure(&mut first_err, e); } }); - wait_futs.push(scanner_fut); + wait_futs.push(AbortOnDropHandle::new(scanner_fut)); } } diff --git a/crates/utils/src/envs.rs b/crates/utils/src/envs.rs index 2d039dec5..5e572e5c0 100644 --- a/crates/utils/src/envs.rs +++ b/crates/utils/src/envs.rs @@ -268,7 +268,7 @@ where .parse::() .map_err(|_| { log_once(&format!("env_invalid_value:{used_key}"), || { - format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::()) + format!("Invalid {} value for {used_key}. Treating as unset.", type_name::()) }); }) .ok() @@ -570,7 +570,7 @@ where Ok(parsed) => EnvParseOutcome::Parsed(parsed), Err(_) => { log_once(&format!("env_invalid_value:{used_key}"), || { - format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::()) + format!("Invalid {} value for {used_key}. Treating as unset.", type_name::()) }); EnvParseOutcome::Invalid } diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 33206d699..de54b8f40 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. | | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. | | `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. | -| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. | +| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. | | `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. | | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. | | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | @@ -70,6 +70,21 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. +When the cycle duration control is unset, RustFS uses a finite 1800-second +(30-minute) default, matching the scanner benchmark guidance. An explicit `0` +preserves the compatibility behavior of an unbounded cycle; object and +directory budgets likewise remain unbounded when explicitly set to `0`. Invalid +or overflowing duration environment values are configuration errors rather than +silent fallback values. + +When a finite deadline expires, RustFS cancels cooperative scanner work and +waits only for the existing bounded shutdown window. A non-yielding I/O future +is dropped after that window. RustFS then attempts a higher leadership epoch so +late cycle, usage, cache, and remote writes from the old generation fail closed. +If the worker cannot stop cooperatively, the cycle state was not confirmed +durable, or that epoch fence cannot be durably persisted, the scanner reports +`recovery-required`; it does not claim an uncooperative cursor was saved. + An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle cadence: dirty-usage notifications do not bypass that configured interval. The default adaptive policy continues to use dirty-usage notifications to wake @@ -144,6 +159,10 @@ metrics.maintenance_control.primary_control metrics.source_work metrics.replication_repair metrics.scan_checkpoint +metrics.cycle_timeout_total +metrics.cycle_last_progress_age +metrics.leader_lease_without_progress +metrics.cycle_recovery_required_total ``` ## Reading Pacing Pressure From cc412914d59c338783b0a13ae0db8468174b35dd Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 00:03:23 +0800 Subject: [PATCH 12/32] feat(connect): emit durable heartbeats (#6383) --- protocol/agent/v1/fixtures/fixture-sets.json | 2 +- .../v1/fixtures/heartbeat/MANIFEST.sha256 | 5 + .../v1/fixtures/heartbeat/duplicate.json | 9 + .../agent/v1/fixtures/heartbeat/overflow.json | 11 + .../agent/v1/fixtures/heartbeat/stale.json | 9 + .../agent/v1/fixtures/heartbeat/unknown.json | 18 + .../agent/v1/fixtures/heartbeat/valid.json | 21 + rustfs/src/connect/config.rs | 173 +++++ rustfs/src/connect/heartbeat.rs | 585 +++++++++++++++ rustfs/src/connect/mod.rs | 12 +- rustfs/src/connect/runtime.rs | 156 ++++ rustfs/src/startup_lifecycle.rs | 4 + rustfs/src/startup_services.rs | 21 + rustfs/tests/connect_heartbeat.rs | 687 ++++++++++++++++++ 14 files changed, 1709 insertions(+), 4 deletions(-) create mode 100644 protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 create mode 100644 protocol/agent/v1/fixtures/heartbeat/duplicate.json create mode 100644 protocol/agent/v1/fixtures/heartbeat/overflow.json create mode 100644 protocol/agent/v1/fixtures/heartbeat/stale.json create mode 100644 protocol/agent/v1/fixtures/heartbeat/unknown.json create mode 100644 protocol/agent/v1/fixtures/heartbeat/valid.json create mode 100644 rustfs/src/connect/config.rs create mode 100644 rustfs/src/connect/heartbeat.rs create mode 100644 rustfs/src/connect/runtime.rs create mode 100644 rustfs/tests/connect_heartbeat.rs diff --git a/protocol/agent/v1/fixtures/fixture-sets.json b/protocol/agent/v1/fixtures/fixture-sets.json index d3beb8f23..72c4c1919 100644 --- a/protocol/agent/v1/fixtures/fixture-sets.json +++ b/protocol/agent/v1/fixtures/fixture-sets.json @@ -25,7 +25,7 @@ }, { "name": "heartbeat", - "status": "reserved", + "status": "populated", "purpose": "Heartbeat payloads, Connect receive time, and freshness window behavior." }, { diff --git a/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 b/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 new file mode 100644 index 000000000..7aa8e3191 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 @@ -0,0 +1,5 @@ +975c1ca53eefeef6766a6fc0b3d3281f7408255342b0686e5e2aee5ad055414c duplicate.json +963529a38a02849c6c2acc6d72668dca9f63218b49c89fae41a451b584850411 overflow.json +e3adeee1c8a19aa17e70894896fb79c072e3785bea3611b93c11e79f039ed5af stale.json +35b9cebd8525389a701e8fe69fbe96407bcb31aa28392fe95babf4a4886985ad unknown.json +37941735dbd6ad3d238258a7b2cae6f0b3aa0ecaae1d8817817c3d718d11d633 valid.json diff --git a/protocol/agent/v1/fixtures/heartbeat/duplicate.json b/protocol/agent/v1/fixtures/heartbeat/duplicate.json new file mode 100644 index 000000000..11cad8ac1 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/duplicate.json @@ -0,0 +1,9 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "duplicate", + "description": "An exact requestId replay returns the first result and creates no second heartbeat.", + "first": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "replay": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "expected": {"decision": "DUPLICATE", "heartbeatWrites": 1, "events": 1, "sameResponse": true} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/overflow.json b/protocol/agent/v1/fixtures/heartbeat/overflow.json new file mode 100644 index 000000000..7b62b5b45 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/overflow.json @@ -0,0 +1,11 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "overflow", + "description": "Values beyond frozen bounds are rejected before persistence.", + "vectors": [ + {"field": "sequence", "value": 9007199254740992, "maximum": 9007199254740991}, + {"field": "coarseNodeSummary.total", "value": 4097, "maximum": 4096} + ], + "expected": {"decision": "REJECT", "httpStatus": 422, "status": "INVALID_ARGUMENT"} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/stale.json b/protocol/agent/v1/fixtures/heartbeat/stale.json new file mode 100644 index 000000000..aabe98a6f --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/stale.json @@ -0,0 +1,9 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "stale", + "description": "A lower heartbeat sequence is retained as history and cannot replace the current projection.", + "head": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "late": {"requestId": "7c4d2e10-9f83-4a5b-b6c7-d8e9f0a1b2c3", "sequence": 9}, + "expected": {"decision": "ACCEPT_HISTORY", "currentSequence": 42, "historySequence": 9} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/unknown.json b/protocol/agent/v1/fixtures/heartbeat/unknown.json new file mode 100644 index 000000000..6639cc0ff --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/unknown.json @@ -0,0 +1,18 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "unknown", + "description": "Unknown optional members and capabilities are accepted, discarded before hashing, and never stored or echoed.", + "requestAdditions": { + "telemetryProfile": "extended", + "authorization": "Bearer non-functional-example", + "capabilities": ["heartbeat", "future.capability"], + "coarseNodeSummary": {"rackNames": ["customer-rack"]} + }, + "expected": { + "decision": "ACCEPT", + "storedCapabilities": ["heartbeat"], + "discarded": ["authorization", "future.capability", "telemetryProfile", "coarseNodeSummary.rackNames"], + "echoed": [] + } +} diff --git a/protocol/agent/v1/fixtures/heartbeat/valid.json b/protocol/agent/v1/fixtures/heartbeat/valid.json new file mode 100644 index 000000000..ed3561b80 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/valid.json @@ -0,0 +1,21 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "valid", + "description": "A bounded L0 heartbeat. clientTime is advisory; Connect's receivedAt is online authority.", + "request": { + "protocolVersion": "v1", + "requestId": "550e8400-e29b-41d4-a716-446655440000", + "agentVersion": "rustfs-agent/1.19.4", + "capabilities": ["heartbeat", "inventory"], + "sequence": 42, + "clientTime": "2026-08-22T01:02:03Z", + "coarseNodeSummary": {"total": 8, "healthy": 7, "degraded": 1} + }, + "expected": { + "decision": "ACCEPT", + "acceptedVersion": "v1", + "responseFields": ["serverTime", "acceptedVersion", "capabilityHints"], + "onlineAuthority": "serverTime" + } +} diff --git a/rustfs/src/connect/config.rs b/rustfs/src/connect/config.rs new file mode 100644 index 000000000..391d4ef45 --- /dev/null +++ b/rustfs/src/connect/config.rs @@ -0,0 +1,173 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use super::{CredentialStore, IdentityStore}; + +pub const ENV_CONNECT_ENDPOINT: &str = "RUSTFS_CONNECT_ENDPOINT"; +pub const ENV_CONNECT_ROOT_CA_FILE: &str = "RUSTFS_CONNECT_ROOT_CA_FILE"; +pub const ENV_CONNECT_STATE_DIR: &str = "RUSTFS_CONNECT_STATE_DIR"; + +#[derive(Clone, Copy, Debug)] +pub struct HeartbeatSchedule { + pub cadence: Duration, + pub jitter: Duration, + pub timeout: Duration, + pub initial_backoff: Duration, + pub max_backoff: Duration, +} + +impl Default for HeartbeatSchedule { + fn default() -> Self { + Self { + cadence: Duration::from_secs(30), + jitter: Duration::from_secs(3), + timeout: Duration::from_secs(5), + initial_backoff: Duration::from_secs(1), + max_backoff: Duration::from_secs(5 * 60), + } + } +} + +#[derive(Clone, Debug)] +pub struct HeartbeatConfig { + pub endpoint: String, + pub root_ca_pem: Vec, + pub identity_store: IdentityStore, + pub credential_store: CredentialStore, + pub state_path: PathBuf, + pub schedule: HeartbeatSchedule, +} + +impl HeartbeatConfig { + pub fn new( + endpoint: impl Into, + root_ca_pem: impl Into>, + identity_store: IdentityStore, + credential_store: CredentialStore, + state_path: impl Into, + ) -> Self { + Self { + endpoint: endpoint.into(), + root_ca_pem: root_ca_pem.into(), + identity_store, + credential_store, + state_path: state_path.into(), + schedule: HeartbeatSchedule::default(), + } + } + + pub fn from_env() -> Result, HeartbeatConfigError> { + Self::from_env_values( + env::var_os(ENV_CONNECT_ENDPOINT), + env::var_os(ENV_CONNECT_ROOT_CA_FILE), + env::var_os(ENV_CONNECT_STATE_DIR), + ) + } + + fn from_env_values( + endpoint: Option, + root_ca_file: Option, + state_dir: Option, + ) -> Result, HeartbeatConfigError> { + let configured = endpoint.is_some() || root_ca_file.is_some() || state_dir.is_some(); + if !configured { + return Ok(None); + } + let (Some(endpoint), Some(root_ca_file), Some(state_dir)) = (endpoint, root_ca_file, state_dir) else { + return Err(HeartbeatConfigError::Partial); + }; + let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?; + let root_ca_file = PathBuf::from(root_ca_file); + let state_dir = PathBuf::from(state_dir); + if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() || state_dir.as_os_str().is_empty() { + return Err(HeartbeatConfigError::Partial); + } + let root_ca_pem = fs::read(&root_ca_file).map_err(|source| HeartbeatConfigError::RootCertificate { + path: root_ca_file, + source, + })?; + Ok(Some(Self::new( + endpoint, + root_ca_pem, + IdentityStore::new(state_dir.join("identity")), + CredentialStore::new(state_dir.join("credential")), + state_dir.join("heartbeat/state.json"), + ))) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum HeartbeatConfigError { + #[error( + "Connect heartbeat configuration requires RUSTFS_CONNECT_ENDPOINT, RUSTFS_CONNECT_ROOT_CA_FILE, and RUSTFS_CONNECT_STATE_DIR" + )] + Partial, + #[error("RUSTFS_CONNECT_ENDPOINT is not valid UTF-8")] + EndpointEncoding, + #[error("failed to read the Connect root CA at {path}: {source}")] + RootCertificate { + path: PathBuf, + #[source] + source: std::io::Error, + }, +} + +#[cfg(test)] +mod tests { + use super::{HeartbeatConfig, HeartbeatConfigError}; + use std::ffi::OsString; + + #[test] + fn absent_environment_is_disabled_without_side_effects() { + assert!( + HeartbeatConfig::from_env_values(None, None, None) + .expect("absent config") + .is_none() + ); + } + + #[test] + fn partial_environment_is_rejected() { + assert!(matches!( + HeartbeatConfig::from_env_values(Some(OsString::from("https://connect.example/agent/")), None, None), + Err(HeartbeatConfigError::Partial) + )); + } + + #[test] + fn complete_environment_builds_the_durable_paths() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().join("root.pem"); + std::fs::write(&root, b"root certificate").expect("root CA"); + let state = temp.path().join("state"); + let config = HeartbeatConfig::from_env_values( + Some(OsString::from("https://connect.example/agent/")), + Some(root.into_os_string()), + Some(state.clone().into_os_string()), + ) + .expect("complete config") + .expect("enabled config"); + + assert_eq!(config.endpoint, "https://connect.example/agent/"); + assert_eq!(config.root_ca_pem, b"root certificate"); + assert_eq!(config.state_path, state.join("heartbeat/state.json")); + assert!(!state.exists(), "parsing configuration must not create state"); + } +} diff --git a/rustfs/src/connect/heartbeat.rs b/rustfs/src/connect/heartbeat.rs new file mode 100644 index 000000000..2eeb8c135 --- /dev/null +++ b/rustfs/src/connect/heartbeat.rs @@ -0,0 +1,585 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fs; +use std::io::{self, Write as _}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use chrono::{DateTime, SecondsFormat, Utc}; +use reqwest::{Client, StatusCode, Url, header}; +use rustls::RootCertStore; +use rustls::pki_types::{CertificateDer, pem::PemObject as _}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use zeroize::Zeroizing; + +use super::config::HeartbeatConfig; +use super::credential_store::{CredentialStoreError, DeviceCredential}; +use super::identity::IdentityError; +use super::identity_store::StoreError; +use super::registration::{CredentialValidationError, validate_stored_credential}; + +const PROTOCOL_VERSION: &str = "v1"; +const AGENT_VERSION: &str = concat!("rustfs-agent/", env!("CARGO_PKG_VERSION")); +const MAX_SEQUENCE: u64 = 9_007_199_254_740_991; +const MAX_RESPONSE_BYTES: usize = 64 * 1024; +#[cfg(unix)] +const FILE_MODE: u32 = 0o600; +static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct CoarseNodeSummary { + total: u16, + healthy: u16, + degraded: u16, +} + +impl CoarseNodeSummary { + pub fn new(total: u16, healthy: u16, degraded: u16) -> Result { + let summary = Self { + total, + healthy, + degraded, + }; + if !summary.is_valid() { + return Err(HeartbeatError::NodeSummary); + } + Ok(summary) + } + + fn is_valid(&self) -> bool { + self.total != 0 + && self.total <= 4096 + && self.healthy <= 4096 + && self.degraded <= 4096 + && self.healthy.saturating_add(self.degraded) <= self.total + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HeartbeatStatus { + Starting, + Online { server_time: String }, + BackingOff { delay: Duration }, + AuthenticationStopped { status: u16, reason: Option }, + Failed { reason: String }, + Stopped, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct PendingHeartbeat { + protocol_version: String, + request_id: String, + agent_version: String, + capabilities: [String; 1], + sequence: u64, + client_time: String, + coarse_node_summary: CoarseNodeSummary, +} + +impl PendingHeartbeat { + fn is_valid(&self) -> bool { + self.protocol_version == PROTOCOL_VERSION + && self.agent_version == AGENT_VERSION + && self.capabilities[0] == "heartbeat" + && self.sequence <= MAX_SEQUENCE + && self.coarse_node_summary.is_valid() + && is_exact_utc_seconds(&self.client_time) + && Uuid::parse_str(&self.request_id) + .is_ok_and(|request_id| request_id.get_version_num() == 4 && request_id.to_string() == self.request_id) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct HeartbeatResponse { + server_time: String, + accepted_version: String, + #[serde(default)] + capability_hints: Vec, +} + +pub(crate) enum Delivery { + Accepted { server_time: String }, + Retry { retry_after: Option }, + AuthenticationStopped { status: u16, reason: Option }, + Rejected { status: u16, reason: Option }, +} + +pub(crate) struct HeartbeatSender { + endpoint: Url, + root_store: RootCertStore, + roots: Vec>, + config: HeartbeatConfig, +} + +impl HeartbeatSender { + pub(crate) fn new(config: HeartbeatConfig) -> Result { + let mut endpoint = Url::parse(&config.endpoint).map_err(|_| HeartbeatError::Endpoint)?; + if endpoint.scheme() != "https" + || endpoint.cannot_be_a_base() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.query().is_some() + || endpoint.fragment().is_some() + { + return Err(HeartbeatError::Endpoint); + } + if !endpoint.path().ends_with('/') { + endpoint.set_path(&format!("{}/", endpoint.path())); + } + let roots = CertificateDer::pem_slice_iter(&config.root_ca_pem) + .collect::, _>>() + .map_err(|_| HeartbeatError::RootCertificate)?; + if roots.is_empty() { + return Err(HeartbeatError::RootCertificate); + } + let mut root_store = RootCertStore::empty(); + let (accepted, rejected) = root_store.add_parsable_certificates(roots.clone()); + if accepted != roots.len() || rejected != 0 { + return Err(HeartbeatError::RootCertificate); + } + let schedule = config.schedule; + if schedule.cadence.is_zero() + || schedule.timeout.is_zero() + || schedule.timeout > Duration::from_secs(5) + || schedule.initial_backoff.is_zero() + || schedule.max_backoff < schedule.initial_backoff + || schedule.max_backoff > Duration::from_secs(5 * 60) + || schedule.jitter > schedule.cadence + { + return Err(HeartbeatError::Schedule); + } + Ok(Self { + endpoint, + root_store, + roots, + config, + }) + } + + pub(crate) async fn send(&self, heartbeat: &PendingHeartbeat) -> Result { + let (cluster_uid, client) = { + let _lock = self.config.credential_store.lock().await?; + let credential = self.config.credential_store.load()?.ok_or(HeartbeatError::NotRegistered)?; + let identity = self.config.identity_store.load()?.ok_or(HeartbeatError::IdentityMissing)?; + validate_stored_credential(&credential, &identity, &self.root_store, &self.roots)?; + let now = Utc::now().timestamp(); + if now < credential.not_before_unix || now >= credential.not_after_unix { + return Err(HeartbeatError::CredentialExpired); + } + let cluster_uid = cluster_uid(&credential)?.to_owned(); + let client = self.client(&credential, &identity.to_pkcs8_pem()?)?; + (cluster_uid, client) + }; + let url = self.endpoint.join(&format!("clusters/{cluster_uid}/heartbeats"))?; + let response = match client.post(url).json(heartbeat).send().await { + Ok(response) => response, + Err(error) if error.is_timeout() || error.is_connect() || error.is_request() => { + return Ok(Delivery::Retry { retry_after: None }); + } + Err(error) => return Err(error.into()), + }; + let status = response.status(); + if status == StatusCode::TOO_MANY_REQUESTS { + return Ok(Delivery::Retry { + retry_after: retry_after(response.headers(), Utc::now(), self.config.schedule.max_backoff), + }); + } + if status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() { + return Ok(Delivery::Retry { retry_after: None }); + } + if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + return Ok(Delivery::AuthenticationStopped { + status: status.as_u16(), + reason: response_reason(response).await, + }); + } + if status != StatusCode::OK { + return Ok(Delivery::Rejected { + status: status.as_u16(), + reason: response_reason(response).await, + }); + } + let accepted: HeartbeatResponse = + serde_json::from_slice(&bounded_body(response).await?).map_err(|_| HeartbeatError::Response)?; + if accepted.accepted_version != PROTOCOL_VERSION + || accepted.capability_hints.len() > 32 + || accepted.capability_hints.iter().any(|hint| hint.len() > 32) + || !is_exact_utc_seconds(&accepted.server_time) + { + return Err(HeartbeatError::Response); + } + Ok(Delivery::Accepted { + server_time: accepted.server_time, + }) + } + + fn client(&self, credential: &DeviceCredential, key: &Zeroizing) -> Result { + let mut pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + key.len() + 1)); + pem.extend_from_slice(credential.certificate_chain.as_bytes()); + pem.push(b'\n'); + pem.extend_from_slice(key.as_bytes()); + let identity = reqwest::Identity::from_pem(&pem).map_err(|_| HeartbeatError::IdentityCertificate)?; + let roots = self + .roots + .iter() + .map(|root| reqwest::Certificate::from_der(root.as_ref())) + .collect::, _>>()?; + Client::builder() + .https_only(true) + .redirect(reqwest::redirect::Policy::none()) + .timeout(self.config.schedule.timeout) + .tls_certs_only(roots) + .identity(identity) + .build() + .map_err(Into::into) + } +} + +#[derive(Clone)] +pub(crate) struct HeartbeatStateStore { + path: PathBuf, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct HeartbeatState { + next_sequence: u64, + pending: Option, +} + +impl HeartbeatStateStore { + pub(crate) fn new(path: PathBuf) -> Self { + Self { path } + } + + pub(crate) fn try_runtime_lock(&self) -> Result { + let directory = parent(&self.path)?; + fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?; + let name = filename(&self.path)?; + let path = directory.join(format!(".{name}.lock")); + let mut options = fs::OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(FILE_MODE); + } + let lock = options.open(&path).map_err(|source| state_io(&path, source))?; + check_mode(&path)?; + lock.try_lock().map_err(|_| HeartbeatError::AlreadyRunning)?; + Ok(lock) + } + + pub(crate) async fn prepare( + &self, + summary: CoarseNodeSummary, + now: DateTime, + ) -> Result { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.prepare_sync(summary, now)) + .await + .map_err(|source| state_io(&self.path, io::Error::other(source)))? + } + + pub(crate) async fn mark_accepted(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> { + let store = self.clone(); + let accepted = accepted.clone(); + tokio::task::spawn_blocking(move || store.mark_accepted_sync(&accepted)) + .await + .map_err(|source| state_io(&self.path, io::Error::other(source)))? + } + + fn prepare_sync(&self, summary: CoarseNodeSummary, now: DateTime) -> Result { + let mut state = self.read()?; + if let Some(pending) = state.pending { + return Ok(pending); + } + if state.next_sequence > MAX_SEQUENCE { + return Err(HeartbeatError::SequenceExhausted); + } + let pending = PendingHeartbeat { + protocol_version: PROTOCOL_VERSION.to_owned(), + request_id: Uuid::new_v4().to_string(), + agent_version: AGENT_VERSION.to_owned(), + capabilities: ["heartbeat".to_owned()], + sequence: state.next_sequence, + client_time: now.to_rfc3339_opts(SecondsFormat::Secs, true), + coarse_node_summary: summary, + }; + state.pending = Some(pending.clone()); + self.write(&state)?; + Ok(pending) + } + + fn mark_accepted_sync(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> { + let mut state = self.read()?; + if state.pending.as_ref() != Some(accepted) { + return Err(HeartbeatError::StateConflict); + } + state.next_sequence = accepted.sequence.checked_add(1).ok_or(HeartbeatError::SequenceExhausted)?; + state.pending = None; + self.write(&state) + } + + fn read(&self) -> Result { + let bytes = match fs::read(&self.path) { + Ok(bytes) => bytes, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(HeartbeatState::default()), + Err(source) => return Err(state_io(&self.path, source)), + }; + check_mode(&self.path)?; + let state: HeartbeatState = serde_json::from_slice(&bytes).map_err(|source| HeartbeatError::StateInvalid { + path: self.path.clone(), + source, + })?; + if state.next_sequence > MAX_SEQUENCE + 1 + || state + .pending + .as_ref() + .is_some_and(|pending| pending.sequence != state.next_sequence || !pending.is_valid()) + { + return Err(HeartbeatError::StateCorrupt { path: self.path.clone() }); + } + Ok(state) + } + + fn write(&self, state: &HeartbeatState) -> Result<(), HeartbeatError> { + let bytes = serde_json::to_vec(state).map_err(|source| HeartbeatError::StateInvalid { + path: self.path.clone(), + source, + })?; + let directory = parent(&self.path)?; + fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?; + let temp = stage(directory, &self.path, &bytes)?; + let result = fs::rename(&temp, &self.path) + .map_err(|source| state_io(&self.path, source)) + .and_then(|()| fsync_dir(directory).map_err(|source| state_io(directory, source))); + if result.is_err() { + let _ = fs::remove_file(temp); + } + result + } +} + +fn cluster_uid(credential: &DeviceCredential) -> Result<&str, HeartbeatError> { + let mut parts = credential.name.split('/'); + let valid = parts.next() == Some("organizations"); + let organization_uid = parts.next(); + let valid = valid && parts.next() == Some("clusters"); + let cluster_uid = parts.next(); + let valid = valid && parts.next() == Some("clusterDevices"); + let device_uid = parts.next(); + if !valid + || organization_uid.is_none_or(str::is_empty) + || cluster_uid.is_none_or(str::is_empty) + || device_uid != Some(credential.uid.as_str()) + || parts.next().is_some() + { + return Err(HeartbeatError::CredentialName); + } + cluster_uid.ok_or(HeartbeatError::CredentialName) +} + +fn retry_after(headers: &header::HeaderMap, now: DateTime, maximum: Duration) -> Option { + let value = headers.get(header::RETRY_AFTER)?.to_str().ok()?; + let delay = value.parse::().ok().map(Duration::from_secs).or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .and_then(|at| (at.with_timezone(&Utc) - now).to_std().ok()) + })?; + Some(delay.min(maximum)) +} + +fn is_exact_utc_seconds(value: &str) -> bool { + DateTime::parse_from_rfc3339(value).is_ok_and(|time| { + time.offset().local_minus_utc() == 0 + && value.ends_with('Z') + && time.with_timezone(&Utc).to_rfc3339_opts(SecondsFormat::Secs, true) == value + }) +} + +async fn response_reason(response: reqwest::Response) -> Option { + #[derive(Deserialize)] + struct Envelope { + #[serde(default)] + details: Vec, + } + #[derive(Deserialize)] + struct Detail { + #[serde(default)] + reason: String, + } + + serde_json::from_slice::(&bounded_body(response).await.ok()?) + .ok()? + .details + .into_iter() + .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason)) +} + +async fn bounded_body(mut response: reqwest::Response) -> Result, HeartbeatError> { + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await? { + if body.len().saturating_add(chunk.len()) > MAX_RESPONSE_BYTES { + return Err(HeartbeatError::ResponseTooLarge); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +fn parent(path: &Path) -> Result<&Path, HeartbeatError> { + path.parent() + .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state path has no parent"))) +} + +fn filename(path: &Path) -> Result<&str, HeartbeatError> { + path.file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state filename is invalid"))) +} + +fn stage(directory: &Path, destination: &Path, bytes: &[u8]) -> Result { + let name = filename(destination)?; + loop { + let path = directory.join(format!( + ".{name}.{}.{}.tmp", + std::process::id(), + STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(FILE_MODE); + } + let mut file = match options.open(&path) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue, + Err(source) => return Err(state_io(&path, source)), + }; + if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) { + let _ = fs::remove_file(&path); + return Err(state_io(&path, source)); + } + return Ok(path); + } +} + +fn state_io(path: &Path, source: io::Error) -> HeartbeatError { + HeartbeatError::StateIo { + path: path.to_path_buf(), + source, + } +} + +#[cfg(unix)] +fn check_mode(path: &Path) -> Result<(), HeartbeatError> { + use std::os::unix::fs::PermissionsExt as _; + + let mode = fs::metadata(path) + .map_err(|source| state_io(path, source))? + .permissions() + .mode() + & 0o7777; + if mode != FILE_MODE { + return Err(HeartbeatError::StatePermissions { + path: path.to_path_buf(), + mode, + expected: FILE_MODE, + }); + } + Ok(()) +} + +#[cfg(not(unix))] +fn check_mode(_path: &Path) -> Result<(), HeartbeatError> { + Ok(()) +} + +fn fsync_dir(directory: &Path) -> io::Result<()> { + #[cfg(unix)] + fs::File::open(directory)?.sync_all()?; + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum HeartbeatError { + #[error("Connect heartbeat endpoint must be an HTTPS base URL without credentials, query, or fragment")] + Endpoint, + #[error("Connect heartbeat root CA configuration is invalid")] + RootCertificate, + #[error("Connect heartbeat schedule is invalid")] + Schedule, + #[error("RustFS is not registered with Connect")] + NotRegistered, + #[error("the Connect device private key is missing")] + IdentityMissing, + #[error("the stored Connect certificate and device private key cannot form a TLS identity")] + IdentityCertificate, + #[error("the stored Connect credential name is invalid")] + CredentialName, + #[error("the stored Connect device certificate is not currently valid")] + CredentialExpired, + #[error("the Connect heartbeat node summary is outside protocol bounds")] + NodeSummary, + #[error("the Connect heartbeat sequence is exhausted")] + SequenceExhausted, + #[error("a Connect heartbeat runtime already owns this state")] + AlreadyRunning, + #[error("the persisted Connect heartbeat changed while delivery was in flight")] + StateConflict, + #[error("Connect heartbeat state I/O failed at {path}: {source}")] + StateIo { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("Connect heartbeat state at {path} is invalid: {source}")] + StateInvalid { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("Connect heartbeat state at {path} violates the protocol invariants")] + StateCorrupt { path: PathBuf }, + #[cfg(unix)] + #[error("Connect heartbeat state at {path} has mode {mode:o}, expected {expected:o}")] + StatePermissions { path: PathBuf, mode: u32, expected: u32 }, + #[error("Connect heartbeat response exceeded 64 KiB")] + ResponseTooLarge, + #[error("Connect returned an invalid heartbeat response")] + Response, + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Transport(#[from] reqwest::Error), + #[error(transparent)] + Identity(#[from] IdentityError), + #[error(transparent)] + IdentityStore(#[from] StoreError), + #[error(transparent)] + CredentialStore(#[from] CredentialStoreError), + #[error(transparent)] + CredentialValidation(#[from] CredentialValidationError), +} diff --git a/rustfs/src/connect/mod.rs b/rustfs/src/connect/mod.rs index 350cceb38..3ce2ad975 100644 --- a/rustfs/src/connect/mod.rs +++ b/rustfs/src/connect/mod.rs @@ -21,20 +21,26 @@ //! canonical transcript frozen by //! `protocol/agent/v1/registration-proof.md`. //! -//! Nothing here contacts the network or starts a task. A deployment that has -//! not been enrolled into a Connect control plane never calls into it, so an -//! unconfigured server generates no key and holds no identity. +//! Enrolled deployments may start the optional outbound heartbeat runtime. +//! An unconfigured server starts no Connect task, generates no key, and holds +//! no Connect identity. pub mod client; +pub mod config; pub mod credential_store; +pub mod heartbeat; pub mod identity; pub mod identity_store; pub mod offline; pub mod registration; +pub mod runtime; pub use client::{ClientError, ConnectClient, ConnectConfig}; +pub use config::{HeartbeatConfig, HeartbeatConfigError, HeartbeatSchedule}; pub use credential_store::{CredentialStore, DeviceCredential}; +pub use heartbeat::{CoarseNodeSummary, HeartbeatError, HeartbeatStatus}; pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript}; pub use identity_store::{IdentityStore, StoreError}; pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge}; pub use registration::{RegistrationToken, TokenError}; +pub use runtime::{HeartbeatRuntime, spawn_heartbeat_runtime}; diff --git a/rustfs/src/connect/runtime.rs b/rustfs/src/connect/runtime.rs new file mode 100644 index 000000000..f0d919a07 --- /dev/null +++ b/rustfs/src/connect/runtime.rs @@ -0,0 +1,156 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future::Future; +use std::time::Duration; + +use chrono::Utc; +use rand::RngExt as _; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::config::HeartbeatConfig; +use super::heartbeat::{CoarseNodeSummary, Delivery, HeartbeatError, HeartbeatSender, HeartbeatStateStore, HeartbeatStatus}; + +pub struct HeartbeatRuntime { + shutdown: CancellationToken, + status: watch::Receiver, + task: Option>, +} + +impl HeartbeatRuntime { + pub fn status(&self) -> watch::Receiver { + self.status.clone() + } + + pub async fn shutdown(mut self) { + self.shutdown.cancel(); + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +impl Drop for HeartbeatRuntime { + fn drop(&mut self) { + self.shutdown.cancel(); + } +} + +pub fn spawn_heartbeat_runtime( + config: Option, + parent_shutdown: &CancellationToken, + sample: F, +) -> Result, HeartbeatError> +where + F: Fn() -> CoarseNodeSummary + Send + Sync + 'static, +{ + let Some(config) = config else { + return Ok(None); + }; + let sender = HeartbeatSender::new(config.clone())?; + let store = HeartbeatStateStore::new(config.state_path.clone()); + let lock = store.try_runtime_lock()?; + let schedule = config.schedule; + let shutdown = parent_shutdown.child_token(); + let task_shutdown = shutdown.clone(); + let (status_tx, status_rx) = watch::channel(HeartbeatStatus::Starting); + let task = tokio::spawn(async move { + let _lock = lock; + let mut backoff = schedule.initial_backoff; + loop { + if task_shutdown.is_cancelled() { + break; + } + let pending = match store.prepare(sample(), Utc::now()).await { + Ok(pending) => pending, + Err(error) => return failed(&status_tx, error), + }; + let delivery = match cancellable(&task_shutdown, sender.send(&pending)).await { + Some(Ok(delivery)) => delivery, + Some(Err(error)) => return failed(&status_tx, error), + None => break, + }; + let delay = match delivery { + Delivery::Accepted { server_time } => { + if let Err(error) = store.mark_accepted(&pending).await { + return failed(&status_tx, error); + } + backoff = schedule.initial_backoff; + let _ = status_tx.send(HeartbeatStatus::Online { server_time }); + schedule.cadence.saturating_add(jitter(schedule.jitter)) + } + Delivery::Retry { retry_after } => { + let delay = retry_after + .unwrap_or(backoff) + .clamp(schedule.initial_backoff, schedule.max_backoff); + backoff = backoff.saturating_mul(2).min(schedule.max_backoff); + let _ = status_tx.send(HeartbeatStatus::BackingOff { delay }); + delay + } + Delivery::AuthenticationStopped { status, reason } => { + let _ = status_tx.send(HeartbeatStatus::AuthenticationStopped { status, reason }); + return; + } + Delivery::Rejected { status, reason } => { + let suffix = reason.map_or_else(String::new, |reason| format!("; reason={reason}")); + let _ = status_tx.send(HeartbeatStatus::Failed { + reason: format!("Connect rejected heartbeat with HTTP {status}{suffix}"), + }); + return; + } + }; + if sleep_or_cancel(&task_shutdown, delay).await { + break; + } + } + let _ = status_tx.send(HeartbeatStatus::Stopped); + }); + Ok(Some(HeartbeatRuntime { + shutdown, + status: status_rx, + task: Some(task), + })) +} + +fn failed(status: &watch::Sender, error: HeartbeatError) { + let _ = status.send(HeartbeatStatus::Failed { + reason: error.to_string(), + }); +} + +fn jitter(maximum: Duration) -> Duration { + if maximum.is_zero() { + Duration::ZERO + } else { + maximum.mul_f64(rand::rng().random_range(0.0..=1.0)) + } +} + +async fn cancellable(shutdown: &CancellationToken, future: impl Future) -> Option { + tokio::select! { + biased; + () = shutdown.cancelled() => None, + value = future => Some(value), + } +} + +async fn sleep_or_cancel(shutdown: &CancellationToken, delay: Duration) -> bool { + tokio::select! { + biased; + () = shutdown.cancelled() => true, + () = tokio::time::sleep(delay) => false, + } +} diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index c4d09872b..ad5681366 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -128,6 +128,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec } = lifecycle; let StartupServiceRuntime { optional_runtimes, + heartbeat, iam_bootstrap, enable_scanner, } = service_runtime; @@ -162,6 +163,9 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec shutdown_token, ) .await; + if let Some(heartbeat) = heartbeat { + heartbeat.shutdown().await; + } if let Err(err) = event_notifier_reconciler.await { tracing::warn!( target: "rustfs::main::run", diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index 585e9677a..c07db0f67 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -16,6 +16,7 @@ use crate::site_replication_reconcile::spawn_site_replication_reconcile_task; use crate::storage_api::startup::services::{ECStore, EndpointServerPools, ServerContextSlot}; use crate::{ config::Config, + connect::{CoarseNodeSummary, HeartbeatConfig, HeartbeatRuntime, spawn_heartbeat_runtime}, init::{init_buffer_profile_system, init_kms_system}, server::ServiceStateManager, startup_audit::init_audit_runtime, @@ -35,6 +36,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct StartupServiceRuntime { pub(crate) optional_runtimes: OptionalRuntimeServices, + pub(crate) heartbeat: Option, pub(crate) iam_bootstrap: IamBootstrapDisposition, pub(crate) enable_scanner: bool, } @@ -73,6 +75,8 @@ pub(crate) async fn init_startup_runtime_services( init_kms_system(config).await?; let optional_runtimes = init_optional_runtime_services().await?; + let heartbeat_config = HeartbeatConfig::from_env().map_err(std::io::Error::other)?; + let heartbeat_nodes = heartbeat_config.as_ref().map(|_| endpoint_pools.get_nodes().len()); init_buffer_profile_system(config); init_deadlock_detector_runtime(); @@ -92,10 +96,27 @@ pub(crate) async fn init_startup_runtime_services( init_notification_runtime(endpoint_pools, buckets).await?; let enable_scanner = init_background_service_runtime(store.clone()).await?; init_observability_runtime(store.clone(), ctx.clone()).await; + let heartbeat = start_heartbeat_runtime(heartbeat_config, heartbeat_nodes, &ctx)?; Ok(StartupServiceRuntime { optional_runtimes, + heartbeat, iam_bootstrap, enable_scanner, }) } + +fn start_heartbeat_runtime( + config: Option, + node_count: Option, + shutdown: &CancellationToken, +) -> Result> { + let Some(config) = config else { + return Ok(None); + }; + let summary = u16::try_from(node_count.unwrap_or_default()) + .ok() + .and_then(|total| CoarseNodeSummary::new(total, 0, 0).ok()) + .ok_or_else(|| std::io::Error::other("Connect heartbeat node count is outside protocol bounds"))?; + spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(std::io::Error::other) +} diff --git a/rustfs/tests/connect_heartbeat.rs b/rustfs/tests/connect_heartbeat.rs new file mode 100644 index 000000000..9414cce46 --- /dev/null +++ b/rustfs/tests/connect_heartbeat.rs @@ -0,0 +1,687 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::VecDeque; +use std::fs; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full}; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use rcgen::{ + BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, + KeyUsagePurpose, SanType, +}; +use rustfs::connect::{ + CoarseNodeSummary, CredentialStore, DeviceCredential, HeartbeatConfig, HeartbeatSchedule, HeartbeatStatus, IdentityStore, + spawn_heartbeat_runtime, +}; +use rustls::RootCertStore; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; +use rustls::server::WebPkiClientVerifier; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use tokio::net::TcpListener; +use tokio::sync::watch; +use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; + +const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70"; +const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81"; +const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92"; + +struct TestPki { + root_params: CertificateParams, + root_key: KeyPair, + root_der: CertificateDer<'static>, + root_pem: String, + server_der: CertificateDer<'static>, + server_key: PrivatePkcs8KeyDer<'static>, +} + +impl TestPki { + fn new() -> Self { + let now = OffsetDateTime::now_utc(); + let root_key = KeyPair::generate().expect("generate root key"); + let mut root_params = CertificateParams::default(); + root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root_params.not_before = now - time::Duration::days(30); + root_params.not_after = now + time::Duration::days(30); + root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature]; + let root = root_params.self_signed(&root_key).expect("sign root"); + + let server_key = KeyPair::generate().expect("generate server key"); + let mut server_params = CertificateParams::default(); + server_params.not_before = now - time::Duration::hours(1); + server_params.not_after = now + time::Duration::days(2); + server_params + .subject_alt_names + .push(SanType::DnsName("localhost".try_into().expect("valid DNS name"))); + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let server = server_params + .signed_by(&server_key, &Issuer::from_params(&root_params, &root_key)) + .expect("sign server certificate"); + Self { + root_params, + root_key, + root_der: root.der().clone(), + root_pem: root.pem(), + server_der: server.der().clone(), + server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()), + } + } + + fn server_config(&self) -> rustls::ServerConfig { + let mut roots = RootCertStore::empty(); + roots.add(self.root_der.clone()).expect("add client root"); + let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .expect("client verifier"); + rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key())) + .expect("server TLS") + } + + fn stores(&self, temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) { + let now = OffsetDateTime::now_utc(); + self.stores_with_certificate(temp, now - time::Duration::hours(1), now + time::Duration::hours(23), true) + } + + fn stores_with_certificate( + &self, + temp: &tempfile::TempDir, + not_before: OffsetDateTime, + not_after: OffsetDateTime, + bind_identity: bool, + ) -> (IdentityStore, CredentialStore) { + let identity_store = IdentityStore::new(temp.path().join("identity")); + let identity = identity_store.load_or_create().expect("create identity"); + let private_key = PrivatePkcs8KeyDer::from(identity.to_pkcs8_der().expect("serialize key").to_vec()); + let device_key = if bind_identity { + KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("device key") + } else { + KeyPair::generate().expect("mismatched device key") + }; + let mut params = CertificateParams::default(); + params.not_before = not_before; + params.not_after = not_after; + params.serial_number = Some(vec![1; 16].into()); + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + params.distinguished_name = DistinguishedName::new(); + params.distinguished_name.push(DnType::CommonName, DEVICE_UID); + params.subject_alt_names.push(SanType::URI( + format!("urn:rustfs:connect:device:{DEVICE_UID}") + .try_into() + .expect("device URI"), + )); + let certificate = params + .signed_by(&device_key, &Issuer::from_params(&self.root_params, &self.root_key)) + .expect("device certificate"); + let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}"); + let credential = DeviceCredential { + name: format!("{cluster}/clusterDevices/{DEVICE_UID}"), + uid: DEVICE_UID.to_owned(), + protocol_version: "v1".to_owned(), + key_id: format!("x509-{}", "01".repeat(16)), + certificate_serial: "01".repeat(16), + certificate: certificate.pem(), + certificate_chain: certificate.pem(), + not_before_unix: not_before.unix_timestamp(), + not_after_unix: not_after.unix_timestamp(), + }; + let directory = temp.path().join("credential"); + fs::create_dir_all(&directory).expect("credential directory"); + let path = directory.join("device.crt.json"); + fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("write credential"); + private_mode(&path); + (identity_store, CredentialStore::new(directory)) + } +} + +#[derive(Clone)] +struct Reply { + status: StatusCode, + body: Value, + retry_after: Option<&'static str>, + delay: Duration, +} + +impl Reply { + fn ok(time: &str) -> Self { + Self { + status: StatusCode::OK, + body: json!({ + "serverTime": time, + "acceptedVersion": "v1", + "capabilityHints": [], + "futureField": true + }), + retry_after: None, + delay: Duration::ZERO, + } + } + + fn error(status: StatusCode) -> Self { + Self { + status, + body: json!({"details": []}), + retry_after: None, + delay: Duration::ZERO, + } + } +} + +struct TestServer { + endpoint: String, + seen: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn server(pki: &TestPki, replies: Vec) -> TestServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind server"); + let address = listener.local_addr().expect("server address"); + let acceptor = TlsAcceptor::from(Arc::new(pki.server_config())); + let replies = Arc::new(Mutex::new(VecDeque::from(replies))); + let seen = Arc::new(Mutex::new(Vec::new())); + let captured = seen.clone(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + let replies = replies.clone(); + let seen = captured.clone(); + tokio::spawn(async move { + let Ok(stream) = acceptor.accept(stream).await else { return }; + let service = service_fn(move |request: Request| { + let replies = replies.clone(); + let seen = seen.clone(); + async move { + assert_eq!(request.uri().path(), format!("/agent/clusters/{CLUSTER_UID}/heartbeats")); + let body = request.into_body().collect().await.expect("request body").to_bytes(); + seen.lock() + .expect("seen lock") + .push(serde_json::from_slice(&body).expect("request JSON")); + let reply = replies + .lock() + .expect("reply lock") + .pop_front() + .unwrap_or_else(|| Reply::error(StatusCode::SERVICE_UNAVAILABLE)); + if !reply.delay.is_zero() { + tokio::time::sleep(reply.delay).await; + } + let mut builder = Response::builder() + .status(reply.status) + .header("content-type", "application/json"); + if let Some(value) = reply.retry_after { + builder = builder.header("retry-after", value); + } + Ok::<_, hyper::Error>( + builder + .body(Full::new(Bytes::from(serde_json::to_vec(&reply.body).expect("reply JSON")))) + .expect("reply"), + ) + } + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + TestServer { + endpoint: format!("https://localhost:{}/agent/", address.port()), + seen, + task, + } +} + +fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> HeartbeatConfig { + let (identity_store, credential_store) = pki.stores(temp); + config_with_stores(temp, pki, server, identity_store, credential_store) +} + +fn config_with_stores( + temp: &tempfile::TempDir, + pki: &TestPki, + server: &TestServer, + identity_store: IdentityStore, + credential_store: CredentialStore, +) -> HeartbeatConfig { + HeartbeatConfig { + endpoint: server.endpoint.clone(), + root_ca_pem: pki.root_pem.as_bytes().to_vec(), + identity_store, + credential_store, + state_path: temp.path().join("heartbeat/state.json"), + schedule: HeartbeatSchedule { + cadence: Duration::from_millis(40), + jitter: Duration::ZERO, + timeout: Duration::from_millis(200), + initial_backoff: Duration::from_millis(20), + max_backoff: Duration::from_millis(80), + }, + } +} + +fn rewrite_credential(temp: &tempfile::TempDir, update: impl FnOnce(&mut DeviceCredential)) { + let path = temp.path().join("credential/device.crt.json"); + let mut credential: DeviceCredential = + serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("parse credential"); + update(&mut credential); + fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("rewrite credential"); + private_mode(&path); +} + +fn summary() -> CoarseNodeSummary { + CoarseNodeSummary::new(8, 7, 1).expect("node summary") +} + +async fn wait_for( + status: &mut watch::Receiver, + predicate: impl Fn(&HeartbeatStatus) -> bool, +) -> HeartbeatStatus { + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let current = status.borrow_and_update().clone(); + if predicate(¤t) { + return current; + } + status.changed().await.expect("status channel"); + } + }) + .await + .expect("heartbeat status timeout") +} + +async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer, expected: &str) { + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert!(matches!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await, + HeartbeatStatus::Failed { reason } if reason.contains(expected) + )); + assert!(server.seen.lock().expect("seen lock").is_empty()); + runtime.shutdown().await; +} + +#[tokio::test] +async fn connect_config_absent_starts_no_task() { + let shutdown = CancellationToken::new(); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let sampled = calls.clone(); + let runtime = spawn_heartbeat_runtime(None, &shutdown, move || { + sampled.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + summary() + }) + .expect("absent config"); + + assert!(runtime.is_none()); + tokio::task::yield_now().await; + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn duplicate_runtime_is_rejected_without_a_second_task() { + let pki = TestPki::new(); + let mut reply = Reply::ok("2026-08-22T01:02:03Z"); + reply.delay = Duration::from_secs(5); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) + .expect("first runtime") + .expect("configured runtime"); + + assert!(matches!( + spawn_heartbeat_runtime(Some(config), &shutdown, summary), + Err(rustfs::connect::HeartbeatError::AlreadyRunning) + )); + runtime.shutdown().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn dropped_runtime_keeps_the_lock_until_its_task_stops() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) + .expect("first runtime") + .expect("configured runtime"); + + drop(runtime); + assert!(matches!( + spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary), + Err(rustfs::connect::HeartbeatError::AlreadyRunning) + )); + + let replacement = tokio::time::timeout(Duration::from_secs(3), async { + loop { + match spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) { + Ok(Some(runtime)) => break runtime, + Err(rustfs::connect::HeartbeatError::AlreadyRunning) => tokio::task::yield_now().await, + Ok(None) => panic!("configured replacement returned no runtime"), + Err(error) => panic!("unexpected replacement error: {error}"), + } + } + }) + .await + .expect("dropped runtime releases its lock after stopping"); + replacement.shutdown().await; +} + +#[tokio::test] +async fn corrupt_persisted_state_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let directory = config.state_path.parent().expect("state directory"); + fs::create_dir_all(directory).expect("create state directory"); + fs::write( + &config.state_path, + br#"{"nextSequence":0,"pending":{"protocolVersion":"v1","requestId":"550e8400-e29b-41d4-a716-446655440000","agentVersion":"rustfs-agent/1.0.0-rc.3","capabilities":["heartbeat"],"sequence":0,"clientTime":"2026-08-22T01:02:03Z","coarseNodeSummary":{"total":0,"healthy":0,"degraded":0}}}"#, + ) + .expect("write corrupt state"); + private_mode(&config.state_path); + let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + + assert!(matches!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await, + HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants") + )); + assert!(server.seen.lock().expect("seen lock").is_empty()); + runtime.shutdown().await; +} + +#[tokio::test] +async fn invalid_stored_resource_name_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let config = config(&temp, &pki, &server); + rewrite_credential(&temp, |credential| { + credential.name = format!("organizations/{ORGANIZATION_UID}/clusters/not-a-uuid/clusterDevices/{DEVICE_UID}"); + }); + + assert_credential_failure(config, &server, "wrong device identity").await; +} + +#[tokio::test] +async fn invalid_stored_protocol_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let config = config(&temp, &pki, &server); + rewrite_credential(&temp, |credential| credential.protocol_version = "v2".to_owned()); + + assert_credential_failure(config, &server, "wrong device identity").await; +} + +#[tokio::test] +async fn stored_certificate_key_mismatch_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let now = OffsetDateTime::now_utc(); + let (identity_store, credential_store) = + pki.stores_with_certificate(&temp, now - time::Duration::hours(1), now + time::Duration::hours(23), false); + let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store); + + assert_credential_failure(config, &server, "different device key").await; +} + +#[tokio::test] +async fn expired_stored_certificate_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let now = OffsetDateTime::now_utc(); + let (identity_store, credential_store) = + pki.stores_with_certificate(&temp, now - time::Duration::days(2), now - time::Duration::days(1), true); + let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store); + + assert_credential_failure(config, &server, "not currently valid").await; +} + +#[tokio::test] +async fn sends_only_l0_fields_and_accepts_additive_response_fields() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2038-01-19T03:14:07Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await, + HeartbeatStatus::Online { + server_time: "2038-01-19T03:14:07Z".to_owned() + } + ); + runtime.shutdown().await; + let seen = server.seen.lock().expect("seen lock"); + let request = &seen[0]; + let mut keys = request + .as_object() + .expect("heartbeat object") + .keys() + .map(String::as_str) + .collect::>(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "agentVersion", + "capabilities", + "clientTime", + "coarseNodeSummary", + "protocolVersion", + "requestId", + "sequence" + ] + ); + assert_eq!(request["capabilities"], json!(["heartbeat"])); + assert_eq!(request["coarseNodeSummary"], json!({"total": 8, "healthy": 7, "degraded": 1})); + assert_ne!(request["clientTime"], "2038-01-19T03:14:07Z"); + assert!(request.get("authorization").is_none()); +} + +#[tokio::test] +async fn restart_replays_pending_request_then_advances_sequence() { + let pki = TestPki::new(); + let first_server = server(&pki, vec![Reply::error(StatusCode::SERVICE_UNAVAILABLE)]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let first_config = config(&temp, &pki, &first_server); + let runtime = spawn_heartbeat_runtime(Some(first_config.clone()), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await; + runtime.shutdown().await; + let first = first_server.seen.lock().expect("seen lock")[0].clone(); + drop(first_server); + + let second_server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z"), Reply::ok("2026-08-22T01:02:04Z")]).await; + let mut second_config = first_config; + second_config.endpoint = second_server.endpoint.clone(); + let runtime = spawn_heartbeat_runtime(Some(second_config), &shutdown, summary) + .expect("restart runtime") + .expect("configured runtime"); + tokio::time::timeout(Duration::from_secs(3), async { + while second_server.seen.lock().expect("seen lock").len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("two heartbeats"); + runtime.shutdown().await; + + let seen = second_server.seen.lock().expect("seen lock"); + assert_eq!(seen[0]["requestId"], first["requestId"]); + assert_eq!(seen[0]["sequence"], first["sequence"]); + assert_ne!(seen[1]["requestId"], seen[0]["requestId"]); + assert_eq!(seen[1]["sequence"].as_u64(), seen[0]["sequence"].as_u64().map(|value| value + 1)); +} + +#[tokio::test] +async fn retry_after_is_respected_with_the_local_upper_bound() { + let pki = TestPki::new(); + let mut reply = Reply::error(StatusCode::TOO_MANY_REQUESTS); + reply.retry_after = Some("300"); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await, + HeartbeatStatus::BackingOff { + delay: Duration::from_millis(80) + } + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn disconnects_use_exponential_backoff_with_a_cap() { + let pki = TestPki::new(); + let server = server( + &pki, + vec![ + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + ], + ) + .await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + for delay in [20, 40, 80] { + assert_eq!( + wait_for(&mut status, |status| { + matches!(status, HeartbeatStatus::BackingOff { delay: observed } if *observed == Duration::from_millis(delay)) + }) + .await, + HeartbeatStatus::BackingOff { + delay: Duration::from_millis(delay) + } + ); + } + runtime.shutdown().await; +} + +#[tokio::test] +async fn revoked_credential_stops_and_exposes_local_status() { + let pki = TestPki::new(); + let mut reply = Reply::error(StatusCode::UNAUTHORIZED); + reply.body = json!({"details": [{"reason": "CREDENTIAL_REVOKED"}]}); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await, + HeartbeatStatus::AuthenticationStopped { + status: 401, + reason: Some("CREDENTIAL_REVOKED".to_owned()) + } + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(server.seen.lock().expect("seen lock").len(), 1); + runtime.shutdown().await; +} + +#[tokio::test] +async fn shutdown_cancels_an_in_flight_request() { + let pki = TestPki::new(); + let mut reply = Reply::ok("2026-08-22T01:02:03Z"); + reply.delay = Duration::from_secs(5); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + tokio::time::timeout(Duration::from_secs(3), async { + while server.seen.lock().expect("seen lock").is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("request reached server"); + tokio::time::timeout(Duration::from_millis(250), runtime.shutdown()) + .await + .expect("cancellable shutdown"); +} + +#[test] +fn consumes_the_frozen_heartbeat_fixtures() { + let registry: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/fixture-sets.json")).expect("fixture registry"); + let heartbeat = registry["sets"] + .as_array() + .expect("fixture sets") + .iter() + .find(|set| set["name"] == "heartbeat") + .expect("heartbeat fixture set"); + assert_eq!(heartbeat["status"], "populated"); + let valid: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/valid.json")).expect("valid fixture"); + assert_eq!(valid["request"]["protocolVersion"], "v1"); + let overflow: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/overflow.json")).expect("overflow fixture"); + assert_eq!(overflow["expected"]["httpStatus"], 422); +} + +#[cfg(unix)] +fn private_mode(path: &Path) { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("private mode"); +} + +#[cfg(not(unix))] +fn private_mode(_path: &Path) {} From 3ddf1a81ac6760474646a3965f12ed5e2af457c5 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 00:35:57 +0800 Subject: [PATCH 13/32] test(kms): replace 33 hard-coded startup sleeps with readiness probe (#6349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(e2e/kms): replace fixed startup sleeps with KMS readiness probe Replace 33 hard-coded sleep(3s) / sleep(2s) startup waits in KMS e2e tests with an active readiness probe (wait_for_kms_ready) that polls the KMS status endpoint with exponential backoff (200ms→1s, 5s budget). This cuts per-test startup latency from a fixed 3s to ~200-500ms while remaining robust against slow CI machines. Non-startup sleeps (ILM polling loops, fault-recovery detection delays, test-runner inter-test pauses) are left untouched. * style: cargo fmt * fix(kms): use .expect() instead of ? in test functions that return () 7 call sites of wait_for_kms_ready() used ? in async test functions that return () instead of Result. Changed to .expect("KMS ready"). * fix(kms): enforce readiness probe deadline * fix(kms): validate readiness backend status --- .../src/kms/bucket_default_encryption_test.rs | 10 +- crates/e2e_test/src/kms/common.rs | 127 +++++++++++++++--- .../src/kms/copy_object_self_copy_sse_test.rs | 6 +- .../copy_object_version_restore_sse_test.rs | 2 +- .../src/kms/encryption_metadata_test.rs | 6 +- .../src/kms/kms_comprehensive_test.rs | 11 +- .../e2e_test/src/kms/kms_edge_cases_test.rs | 12 +- .../src/kms/kms_fault_recovery_test.rs | 8 +- crates/e2e_test/src/kms/kms_local_test.rs | 8 +- crates/e2e_test/src/kms/kms_vault_test.rs | 5 +- .../src/kms/multipart_encryption_test.rs | 10 +- 11 files changed, 145 insertions(+), 60 deletions(-) diff --git a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs index fecba2b89..c0f0f0181 100644 --- a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs +++ b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs @@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - let total_deadline = Duration::from_secs(5); + wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await +} + +async fn wait_for_kms_ready_with_timeout( + base_url: &str, + access_key: &str, + secret_key: &str, + total_deadline: Duration, +) -> Result<(), Box> { let start = tokio::time::Instant::now(); + let deadline = start + total_deadline; let mut backoff = Duration::from_millis(200); let max_backoff = Duration::from_secs(1); - let mut first_attempt = true; loop { - if !first_attempt { - if start.elapsed() >= total_deadline { - return Err("KMS failed to become ready within 5 seconds".into()); - } - sleep(backoff).await; - backoff = (backoff * 2).min(max_backoff); - } - first_attempt = false; - - match get_kms_status(base_url, access_key, secret_key).await { - Ok(status) => { - info!("KMS is ready (status: {})", status); - return Ok(()); - } - Err(e) => { - if start.elapsed() >= total_deadline { - return Err(format!("KMS did not become ready within 5 s: last error: {e}").into()); + match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await { + Ok(Ok(status)) => { + let backend_status = serde_json::from_str::(&status) + .ok() + .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned)); + if backend_status.as_deref() == Some("healthy") { + info!("KMS is ready (status: {})", status); + return Ok(()); } - warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…"); + warn!( + backend_status = backend_status.as_deref().unwrap_or("missing"), + elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX), + "KMS not ready yet, retrying…" + ); } + Ok(Err(e)) => { + let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX); + warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…"); + } + Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()), } + + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()); + } + sleep((now + backoff).min(deadline) - now).await; + backoff = (backoff * 2).min(max_backoff); + } +} + +#[cfg(test)] +mod readiness_tests { + use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + #[tokio::test] + async fn kms_readiness_retries_http_success_until_backend_is_healthy() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server"); + let address = listener.local_addr().expect("read readiness test server address"); + let requests = Arc::new(AtomicUsize::new(0)); + let server_requests = Arc::clone(&requests); + let server = tokio::spawn(async move { + for backend_status in ["error", "healthy"] { + let (mut socket, _) = listener.accept().await.expect("accept readiness request"); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut chunk).await.expect("read readiness request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + server_requests.fetch_add(1, Ordering::SeqCst); + + let body = format!(r#"{{"backend_status":"{backend_status}"}}"#); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.expect("write readiness response"); + } + }); + + wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key") + .await + .expect("KMS should become ready after the healthy response"); + + let observed_requests = requests.load(Ordering::SeqCst); + server.abort(); + assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried"); + } + + #[tokio::test] + async fn kms_readiness_deadline_covers_a_stalled_status_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server"); + let address = listener.local_addr().expect("read readiness test server address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept readiness request"); + let mut request = [0_u8; 1024]; + let _ = socket.read(&mut request).await.expect("read readiness request"); + std::future::pending::<()>().await; + }); + + let result = tokio::time::timeout( + Duration::from_secs(1), + wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)), + ) + .await + .expect("readiness helper must enforce its own deadline"); + + server.abort(); + assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline"); } } diff --git a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs index 1cf19a565..85a013ab3 100644 --- a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs @@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service @@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below @@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-self-copy-bucket-default-sse-test"; diff --git a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs index 3241a217d..c7a572e93 100644 --- a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs @@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-version-restore-sse-test"; diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index a316668f6..59501430b 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), let mut kms_env = LocalKMSTestEnvironment::new().await?; let default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Bo let mut kms_env = LocalKMSTestEnvironment::new().await?; let _default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box Result<(), Box Result<(), Box Date: Sun, 23 Aug 2026 00:40:38 +0800 Subject: [PATCH 14/32] fix(ecstore): fence deletes against decommission commits (#6363) * fix(ecstore): fence deletes against decommission commits * fix(ecstore): preserve decommission target write locks * fix(ecstore): preserve delete markers during source cleanup * fix(ecstore): route batch delete markers to active pools * fix(ecstore): preserve batch delete pool errors * fix(ecstore): retain source-set lock during cleanup * test(ecstore): exercise decommission delete fences * test(ecstore): finish decommission delete fence scenario * fix(ecstore): reuse fixed fence for reverse decommission * fix(ecstore): fence decommission commit loss * fix(ecstore): annotate batch delete fallback * test(ecstore): fix decommission fence fixtures * fix(ecstore): unblock decommission delete fences * fix(ecstore): preserve distributed decommission set locks * fix(ecstore): match decommission lock backend domain * test(ecstore): align decommission fence barriers * fix(ecstore): satisfy delete fence lint checks * fix(rebalance): preserve access-denied delete errors --- crates/ecstore/src/core/pools.rs | 31 +- crates/ecstore/src/data_movement/mod.rs | 203 ++- crates/ecstore/src/disk/local.rs | 79 +- crates/ecstore/src/object_api/types.rs | 30 +- .../ecstore/src/services/rebalance/entry.rs | 1 + crates/ecstore/src/set_disk/mod.rs | 27 +- crates/ecstore/src/set_disk/ops/multipart.rs | 95 ++ crates/ecstore/src/set_disk/ops/object.rs | 33 +- crates/ecstore/src/store/init.rs | 1354 ++++++++++++++++- crates/ecstore/src/store/mod.rs | 2 +- crates/ecstore/src/store/multipart.rs | 29 +- crates/ecstore/src/store/object.rs | 825 +++++++++- crates/ecstore/src/store/rebalance/support.rs | 31 +- 13 files changed, 2606 insertions(+), 134 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 66fe2bbe9..90ad40464 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -3673,6 +3673,9 @@ impl ECStore { ) .await?; + let source_cleanup_mutation_fence = self + .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref()) + .await?; let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( set.clone(), bucket.as_str(), @@ -3684,6 +3687,7 @@ impl ECStore { lifecycle_guard: bucket_incarnation_fence .as_ref() .and_then(|guard| guard.namespace_lock_guard()), + object_mutation_fence: Some(&source_cleanup_mutation_fence), }, "decommission", ) @@ -3785,6 +3789,22 @@ impl ECStore { Ok(()) } + #[cfg(test)] + pub(crate) async fn decommission_entry_for_test( + self: &Arc, + idx: usize, + entry: MetaCacheEntry, + bucket: String, + set: Arc, + ) -> Result<()> { + let worker_permit = Arc::new(Semaphore::new(1)) + .acquire_owned() + .await + .map_err(|err| Error::other(format!("decommission test worker permit acquire failed: {err}")))?; + self.decommission_entry(CancellationToken::new(), idx, entry, bucket, set, worker_permit, None, None, None, None) + .await + } + #[tracing::instrument(skip(self, rx))] async fn decommission_pool( self: &Arc, @@ -4821,15 +4841,20 @@ impl ECStore { ) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_name = rd.object_info.name.clone(); - let result = data_movement::migrate_object( + let mut migration = tokio::task::JoinSet::new(); + migration.spawn(data_movement::migrate_decommission_object( self, pool_idx, bucket.clone(), rd, expected_bucket_incarnation_id, "decommission_object", - ) - .await; + )); + let result = migration + .join_next() + .await + .ok_or_else(|| Error::other("decommission migration task was not started"))? + .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?; if result.is_ok() { warn!("decommission_object: migrated {} {}", &bucket, &object_name); } diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 1f4daa703..4fdda942e 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -26,7 +26,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{HTTPPreconditions, ObjectOperations as _}, }; -use crate::store::ECStore; +use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence}; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; @@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { is_unversioned_data_movement_object(source) && is_unversioned_data_movement_object(target) - && !target.delete_marker && source .mod_time .zip(target.mod_time) @@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError { pub(crate) struct SourceCleanupBucketFence<'a> { pub(crate) expected_incarnation_id: Option, pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>, + pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>, } fn ensure_source_cleanup_versions_match( @@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged( struct SourceCleanupDeleteBarrierState { bucket: String, object: String, + fence_pending: tokio::sync::Notify, arrived: tokio::sync::Notify, + is_paused: AtomicBool, release: tokio::sync::Notify, } @@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier { } #[cfg(test)] -static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock>>> = +static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock>>> = std::sync::OnceLock::new(); #[cfg(test)] @@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier { let state = Arc::new(SourceCleanupDeleteBarrierState { bucket: bucket.to_string(), object: object.to_string(), + fence_pending: tokio::sync::Notify::new(), arrived: tokio::sync::Notify::new(), + is_paused: AtomicBool::new(false), release: tokio::sync::Notify::new(), }); - let mut slot = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison"); - assert!(slot.is_none(), "source cleanup delete barrier must be unique"); - *slot = Some(Arc::clone(&state)); + assert!( + !barriers + .iter() + .any(|barrier| barrier.bucket == bucket && barrier.object == object), + "source cleanup delete barrier must be unique per object" + ); + barriers.push(Arc::clone(&state)); Self { state } } @@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier { .expect("source cleanup should reach the pre-delete barrier"); } + pub(crate) async fn wait_until_fence_pending(&self) { + tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified()) + .await + .expect("source cleanup should attempt the fixed mutation fence"); + } + + pub(crate) fn is_paused(&self) -> bool { + self.state.is_paused.load(Ordering::Acquire) + } + pub(crate) fn release(&self) { self.state.release.notify_one(); } } +#[cfg(test)] +pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) { + let barrier = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("source cleanup delete barrier mutex should not poison") + .iter() + .find(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.fence_pending.notify_one(); + } +} + #[cfg(test)] impl Drop for SourceCleanupDeleteBarrier { fn drop(&mut self) { self.state.release.notify_one(); - let mut slot = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison"); - if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { - *slot = None; - } + barriers.retain(|state| !Arc::ptr_eq(state, &self.state)); } } #[cfg(test)] async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) { - let barrier = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let barrier = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison") - .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .iter() + .find(|barrier| barrier.bucket == bucket && barrier.object == object) .cloned(); if let Some(barrier) = barrier { + barrier.is_paused.store(true, Ordering::Release); barrier.arrived.notify_one(); barrier.release.notified().await; } @@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( op_label: &str, ) -> std::result::Result { let cleanup_key = encode_dir_object(object); - let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?; - let _guard = ns_lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(Error::from)?; + let source_guard = if bucket_fence + .object_mutation_fence + .is_some_and(SourceCleanupMutationFence::source_lock_covered) + { + None + } else { + let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?; + Some( + ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(Error::from)?, + ) + }; if bucket_fence .lifecycle_guard @@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( "{op_label}: bucket incarnation fence was lost before source cleanup" )))); } + if bucket_fence + .object_mutation_fence + .is_some_and(SourceCleanupMutationFence::is_lock_lost) + { + return Err(SourceCleanupError::Storage(Error::other(format!( + "{op_label}: object mutation fence was lost before source cleanup" + )))); + } ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?; @@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id, ..Default::default() }; - opts.add_namespace_lock_guard(&_guard); + if let Some(source_guard) = source_guard.as_ref() { + opts.add_namespace_lock_guard(source_guard); + } + if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence { + object_mutation_fence.add_namespace_lock_fence(&mut opts); + } if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard { opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard); } @@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str { } } +pub(crate) async fn migrate_decommission_object( + store: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + source_bucket_incarnation_id: Option, + op_label: &str, +) -> Result<()> { + let source = rd.object_info.clone(); + let _mutation_fence = store + .acquire_decommission_object_mutation_fence(&bucket, &source.name) + .await?; + let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source) + .await? + .ok_or(Error::FileNotFound)?; + if !is_equivalent_data_movement_object_identity(&source, ¤t, true, false) { + return Err(Error::FileNotFound); + } + + migrate_object_inner( + store, + pool_idx, + bucket, + rd, + source_bucket_incarnation_id, + op_label, + Some(&_mutation_fence), + ) + .await +} + pub(crate) async fn migrate_object( store: Arc, pool_idx: usize, @@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object( rd: GetObjectReader, source_bucket_incarnation_id: Option, op_label: &str, +) -> Result<()> { + migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await +} + +async fn migrate_object_inner( + store: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + source_bucket_incarnation_id: Option, + op_label: &str, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<()> { let object_info = rd.object_info.clone(); let has_part_checksums = object_info @@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object( let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx); new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (res, target_pool_idx, expected_bucket_incarnation_id) = match store - .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts) + .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence) .await { Ok(res) => res, @@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object( if let Err(err) = store .clone() .complete_multipart_upload_for_data_movement( - target_pool_idx, + (target_pool_idx, mutation_fence), &bucket, &object_info.name, &res.upload_id, @@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object( let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx); put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (target_pool_idx, put_result) = store - .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts) + .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence) .await .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?; if let Err(err) = put_result { @@ -3541,25 +3638,47 @@ mod tests { } #[test] - fn test_precondition_conflict_rejects_newer_delete_marker() { - let source = ObjectInfo { - size: 128, - etag: Some("etag-source".to_string()), - mod_time: Some(OffsetDateTime::UNIX_EPOCH), - ..Default::default() - }; - let target = ObjectInfo { - delete_marker: true, - etag: None, - mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), - ..source.clone() - }; + fn test_precondition_conflict_accepts_only_newer_null_delete_marker() { + for version_id in [None, Some(Uuid::nil())] { + let source = ObjectInfo { + version_id, + size: 128, + etag: Some("etag-source".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }; + let target = ObjectInfo { + delete_marker: true, + etag: None, + mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), + ..source.clone() + }; - let should_resume = - resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) - .expect("delete marker conflict should be evaluated"); + assert!( + resolve_data_movement_overwrite_resume_result( + &Error::PreconditionFailed, + Ok(Some(target.clone())), + &source, + 0, + 1, + ) + .expect("newer null delete marker should be evaluated") + ); - assert!(!should_resume); + let mut same_time = target.clone(); + same_time.mod_time = source.mod_time; + assert!( + !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,) + .expect("same-generation null delete marker should be rejected") + ); + + let mut versioned = target; + versioned.version_id = Some(Uuid::new_v4()); + assert!( + !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,) + .expect("a UUID delete marker must not erase a null source version") + ); + } } #[test] diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2d7270cff..9994f8612 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback #[cfg(target_os = "linux")] const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off"; const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed"; +const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed"; const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts"; const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed"; const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed"; @@ -6106,6 +6107,43 @@ impl LocalDisk { Ok((bytes, modtime)) } + async fn write_missing_delete_marker( + &self, + volume: &str, + path: &str, + fi: FileInfo, + object_dir: &Path, + xl_path: &Path, + rollback_dir: Option, + ) -> Result<()> { + if let Some(rollback_dir) = rollback_dir { + let rollback_path = object_dir.join(rollback_dir.to_string()); + fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?; + fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), []) + .await + .map_err(to_file_error)?; + } + if let Err(err) = self.write_metadata("", volume, path, fi).await { + if let Some(rollback_dir) = rollback_dir + && let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await + { + warn!( + event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + result = "failed", + volume, + path, + rollback_dir = %rollback_dir, + error = ?restore_err, + "Disk local delete rollback failed" + ); + } + return Err(err); + } + Ok(()) + } + async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; @@ -6123,7 +6161,20 @@ impl LocalDisk { return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await; } - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?; + let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await { + Ok(data) => data, + Err(DiskError::FileNotFound) => { + // `deleted` alone can be an explicit marker purge; only + // `mark_deleted` may create metadata that was not present. + let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else { + return Err(DiskError::FileNotFound); + }; + return self + .write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir) + .await; + } + Err(err) => return Err(err), + }; if data.is_empty() { return Err(DiskError::FileNotFound); @@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk { } if fi.deleted && force_del_marker { - if let Some(rollback_dir) = rollback_dir { - let rollback_path = file_path.join(rollback_dir.to_string()); - fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?; - fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), []) - .await - .map_err(to_file_error)?; - } - if let Err(err) = self.write_metadata("", volume, path, fi).await { - if let Some(rollback_dir) = rollback_dir - && let Err(restore_err) = - restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await - { - warn!( - volume, - path, - rollback_dir = %rollback_dir, - error = ?restore_err, - "failed to restore metadata after delete marker commit error" - ); - } - return Err(err); - } - return Ok(()); + return self + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir) + .await; } return if fi.version_id.is_some() { diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 0194667e9..fc1513584 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -24,7 +24,7 @@ use crate::storage_api_contracts::{ pub struct NamespaceLockFence { signals: Arc>>, #[cfg(test)] - forced_lost: Arc, + forced_lost: Arc>>, } impl Debug for NamespaceLockFence { @@ -40,13 +40,17 @@ impl NamespaceLockFence { Self { signals: Arc::default(), #[cfg(test)] - forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)), + forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]), } } pub(crate) fn is_lock_lost(&self) -> bool { #[cfg(test)] - if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) { + if self + .forced_lost + .iter() + .any(|lost| lost.load(std::sync::atomic::Ordering::Acquire)) + { return true; } self.signals.iter().any(|signal| signal.is_lost()) @@ -57,27 +61,26 @@ impl NamespaceLockFence { } fn extend(&mut self, other: &Self) { - if Arc::ptr_eq(&self.signals, &other.signals) { - return; + if !Arc::ptr_eq(&self.signals, &other.signals) { + Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); } - Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); #[cfg(test)] - if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) { - self.forced_lost.store(true, std::sync::atomic::Ordering::Release); + if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) { + Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned()); } } #[cfg(test)] pub(crate) fn lost_for_test() -> Self { let fence = Self::new(); - fence.forced_lost.store(true, std::sync::atomic::Ordering::Release); + fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release); fence } #[cfg(test)] pub(crate) fn loss_handle_for_test() -> (Self, Arc) { let fence = Self::new(); - (fence.clone(), Arc::clone(&fence.forced_lost)) + (fence.clone(), Arc::clone(&fence.forced_lost[0])) } } @@ -411,6 +414,13 @@ impl ObjectOptions { self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new); } + #[cfg(test)] + pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) { + self.namespace_lock_fence + .get_or_insert_with(NamespaceLockFence::new) + .extend(fence); + } + pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) { self.lifecycle_delete_all_journal .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default()))); diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index 764a68500..e9e1e2343 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -334,6 +334,7 @@ impl ECStore { lifecycle_guard: bucket_incarnation_fence .as_ref() .and_then(|guard| guard.namespace_lock_guard()), + ..Default::default() }, "rebalance", ), diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index e64e1704f..0422ba94e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters; mod ctx; mod metadata; mod ops; +#[cfg(test)] +pub(crate) use ops::multipart::NewMultipartUploadCommitObservation; #[cfg(any(test, feature = "test-util"))] pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; +#[cfg(test)] +pub(crate) use ops::object::DeleteObjectCommitBarrier; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; @@ -3025,6 +3029,16 @@ pub struct SetDisks { storage_class_config_override: Arc>>>, } +// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones +// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain. +pub(crate) fn same_distributed_lock_domain(left: &[Arc], right: &[Arc]) -> bool { + left.iter() + .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client))) + && right + .iter() + .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client))) +} + const ERASURE_CACHE_MAX_ENTRIES: usize = 32; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -3600,6 +3614,15 @@ impl SetDisks { &self.ctx } + /// Whether both sets' namespace-lock implementations cover the same object key. + pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { + match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { + (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager), + (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers), + _ => false, + } + } + /// The lock manager this set actually uses (test-only; Phase 5 Slice 3). #[cfg(test)] pub(crate) fn local_lock_manager_for_test(&self) -> &Arc { @@ -4584,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool { } fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool { - opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement) + opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement) } fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) { - let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none()); + let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none()); let mut delete_marker = opts.versioned; if opts.version_id.is_some() { diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 6af71db17..5aa59b158 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; use std::future::Future; +#[cfg(test)] +use std::sync::atomic::AtomicBool; #[cfg(any(test, feature = "test-util"))] use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard { #[cfg(any(test, feature = "test-util"))] #[derive(Clone, Copy, PartialEq, Eq)] pub enum MultipartCommitPause { + NewUploadBeforeLockLost, PutPartBeforeLockAcquire, PutPartBeforeLockLost, PutPartAfterRename, @@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier { } } +#[cfg(test)] +struct NewMultipartUploadCommitObservationState { + bucket: String, + object: String, + committed: AtomicBool, +} + +#[cfg(test)] +pub(crate) struct NewMultipartUploadCommitObservation { + state: Arc, +} + +#[cfg(test)] +static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl NewMultipartUploadCommitObservation { + pub(crate) fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(NewMultipartUploadCommitObservationState { + bucket: bucket.to_string(), + object: object.to_string(), + committed: AtomicBool::new(false), + }); + let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison"); + assert!(slot.is_none(), "new multipart upload commit observation must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn committed(&self) -> bool { + self.state.committed.load(Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for NewMultipartUploadCommitObservation { + fn drop(&mut self) { + let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn observe_new_multipart_upload_commit(bucket: &str, object: &str) { + let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.object == object) + .cloned(); + if let Some(state) = state { + state.committed.store(true, Ordering::Release); + } +} + #[cfg(any(test, feature = "test-util"))] async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { let barrier = { @@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement); + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await; + if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "new_multipart_upload_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "new_multipart_upload_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Self::write_unique_file_info( &shuffle_disks, @@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + #[cfg(test)] + observe_new_multipart_upload_commit(bucket, object); // evalDisks diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index f5a55a6b4..0d429e3b3 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -2497,6 +2497,7 @@ impl SetDisks { }) .await?, ); + notify_put_object_commit_namespace_acquired(bucket, object); } #[cfg(not(any(test, feature = "test-util")))] { @@ -4644,6 +4645,7 @@ struct PutObjectCommitBarrierState { arrived: tokio::sync::Notify, release: tokio::sync::Notify, namespace_pending: tokio::sync::Notify, + namespace_acquired: std::sync::atomic::AtomicBool, } #[cfg(any(test, feature = "test-util"))] @@ -4665,6 +4667,7 @@ impl PutObjectCommitBarrier { arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), namespace_pending: tokio::sync::Notify::new(), + namespace_acquired: std::sync::atomic::AtomicBool::new(false), }); let mut slot = PUT_OBJECT_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(Vec::new())) @@ -4699,6 +4702,10 @@ impl PutObjectCommitBarrier { .await .expect("put object should wait for the namespace lock after leaving the commit barrier"); } + + pub fn namespace_acquired(&self) -> bool { + self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire) + } } #[cfg(any(test, feature = "test-util"))] @@ -4755,6 +4762,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) { } } +#[cfg(any(test, feature = "test-util"))] +fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) { + let barrier = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison") + .iter() + .find(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace + }) + .cloned(); + if let Some(barrier) = barrier { + barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release); + } +} + #[cfg(test)] struct DeleteObjectCommitBarrierState { bucket: String, @@ -4764,7 +4787,7 @@ struct DeleteObjectCommitBarrierState { } #[cfg(test)] -struct DeleteObjectCommitBarrier { +pub(crate) struct DeleteObjectCommitBarrier { state: Arc, } @@ -4774,7 +4797,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock Self { + pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(DeleteObjectCommitBarrierState { bucket: bucket.to_string(), object: object.to_string(), @@ -4790,13 +4813,13 @@ impl DeleteObjectCommitBarrier { Self { state } } - async fn wait_until_paused(&self) { + pub(crate) async fn wait_until_paused(&self) { tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) .await .expect("delete object should reach the deterministic commit barrier"); } - fn release(&self) { + pub(crate) fn release(&self) { self.state.release.notify_one(); } } @@ -5918,6 +5941,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if dobj.version_id.is_none() && (version_suspended || versioned) { vr.mod_time = Some(OffsetDateTime::now_utc()); vr.deleted = true; + vr.mark_deleted = true; if versioned { vr.version_id = Some(Uuid::new_v4()); } @@ -11804,6 +11828,7 @@ mod transition_upload_integrity_tests { crate::data_movement::SourceCleanupBucketFence { expected_incarnation_id: None, lifecycle_guard: Some(&bucket_guard), + ..Default::default() }, "test_data_movement", ) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index e0452c5bd..451e42242 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -604,15 +604,15 @@ mod tests { storage_api_contracts::{ bucket::{BucketOperations as _, MakeBucketOptions}, multipart::MultipartOperations as _, - object::{ObjectIO, ObjectOperations as _}, + object::{ObjectIO, ObjectOperations as _, ObjectToDelete}, range::HTTPRangeSpec, }, }; use http::HeaderMap; use rustfs_config::server_config::KVS; - use rustfs_filemeta::ObjectPartInfo; #[cfg(feature = "test-util")] use rustfs_filemeta::{FileInfo, FileMeta}; + use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry, ObjectPartInfo}; #[cfg(feature = "test-util")] use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; use rustfs_rio::{Checksum, ChecksumType}; @@ -1226,6 +1226,212 @@ mod tests { shutdown.cancel(); } + async fn migrate_versioned_decommission_test_object( + store: &Arc, + bucket: &str, + object: &str, + payload: &[u8], + op_label: &'static str, + ) -> (uuid::Uuid, FileInfoVersions) { + let mut source = PutObjReader::from_vec(payload.to_vec()); + let source_info = store.pools[0] + .put_object( + bucket, + object, + &mut source, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write versioned source to the pool being decommissioned"); + let source_version = source_info.version_id.expect("versioned source must have a version ID"); + let expected_source_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("source versions should be readable before migration") + .expect("source versions should exist before migration"); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let migration_store = Arc::clone(store); + let migration_bucket = bucket.to_string(); + let migration_object = object.to_string(); + let migration = tokio::spawn(async move { + let source_reader = migration_store.pools[0] + .get_object_reader( + &migration_bucket, + &migration_object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await?; + crate::data_movement::migrate_decommission_object(migration_store, 0, migration_bucket, source_reader, None, op_label) + .await + }); + barrier.wait_until_paused().await; + barrier.release(); + migration + .await + .expect("versioned decommission migration task should join") + .expect("versioned decommission migration should commit"); + + (source_version, expected_source_versions) + } + + async fn mark_test_pool_decommissioning(store: &Arc, pool_idx: usize) { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + + async fn write_decommission_test_multipart_source( + store: &Arc, + pool_idx: usize, + bucket: &str, + object: &str, + ) { + let pool = &store.pools[pool_idx]; + let upload = pool + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("create decommission multipart source upload"); + let first_part = vec![b'm'; 5 * 1024 * 1024]; + let second_part = b"decommission multipart tail".to_vec(); + let mut completed_parts = Vec::with_capacity(2); + for (part_number, body) in [(1, first_part), (2, second_part)] { + let mut reader = PutObjReader::from_vec(body); + let part = pool + .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default()) + .await + .expect("write decommission multipart source part"); + completed_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + pool.clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .expect("complete decommission multipart source object"); + } + + async fn assert_pool_object_present(pool: &Arc, bucket: &str, object: &str) { + pool.get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("expected object generation must remain present"); + } + + async fn assert_pool_object_absent(pool: &Arc, bucket: &str, object: &str) { + let err = pool + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect_err("fenced decommission target must remain absent"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)), + "unexpected fenced target result: {err:?}" + ); + } + + async fn write_suspended_decommission_source(store: &Arc, bucket: &str, object: &str) { + let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec()); + let source = store.pools[0] + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + version_suspended: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND), + ..Default::default() + }, + ) + .await + .expect("write suspended null source version"); + assert!( + source.version_id.is_none_or(|version_id| version_id.is_nil()), + "suspended source must use the null version identity" + ); + } + + async fn assert_suspended_null_source_present(store: &Arc, bucket: &str, object: &str) { + let versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("suspended source versions should be readable") + .expect("suspended source must exist before worker convergence"); + assert!( + versions + .versions + .iter() + .any(|version| !version.deleted && version.version_id.is_none_or(|version_id| version_id.is_nil())), + "the source pool must retain its null data version while DELETE owns the fixed fence" + ); + } + + async fn assert_suspended_decommission_converged(store: &Arc, bucket: &str, object: &str) { + let source_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("source versions should remain readable after suspended convergence"); + assert!( + source_versions.is_none_or(|versions| versions.versions.is_empty()), + "worker convergence must remove only the decommissioned source null version" + ); + + let target_versions = store.pools[1] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("active target versions should be readable") + .expect("active target must retain the suspended DELETE marker"); + assert!( + matches!(target_versions.versions.as_slice(), [marker] if marker.deleted && marker.version_id.is_none_or(|version_id| version_id.is_nil())), + "active target must contain only its null delete marker: {target_versions:?}" + ); + + let err = store + .get_object_info( + bucket, + object, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect_err("the active null delete marker must hide the migrated source generation"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected suspended latest-object result: {err:?}" + ); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn tag_updates_skip_active_rebalance_source_pool() { @@ -2752,6 +2958,1150 @@ mod tests { .expect_err("suspended delete must remove the requested UUID version"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() { + let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "decommission-delete-fence", + &[(2, 4), (1, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4()); + let object = (0..128) + .map(|index| format!("object-{index}.bin")) + .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1) + .expect("the deterministic object search should select source set 1"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission delete-fence bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, &object, &mut source, &ObjectOptions::default()) + .await + .expect("write source object to the pool being decommissioned"); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source"); + + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object); + let source_set = store.pools[0].get_disks_by_key(&object); + assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock"); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker_object = object.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: worker_object, + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + barrier.wait_until_paused().await; + + let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete_object = object.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object(&delete_bucket, &delete_object, ObjectOptions::default()) + .await + }); + delete_barrier.wait_until_paused().await; + delete_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !delete_barrier.namespace_acquired() && !delete.is_finished(), + "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence" + ); + + barrier.release(); + cleanup_barrier.wait_until_paused().await; + drop(barrier); + + let fixed_set = Arc::clone(&store.pools[0].disk_set[0]); + let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::BeforeNamespace, + ); + let mutation_bucket = bucket.clone(); + let mutation_object = object.clone(); + let fixed_mutation = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec()); + fixed_set + .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default()) + .await + }); + fixed_mutation_barrier.wait_until_paused().await; + fixed_mutation_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(), + "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace" + ); + fixed_mutation.abort(); + assert!( + fixed_mutation + .await + .expect_err("the fixed-domain mutation should be canceled") + .is_cancelled(), + "the competing fixed-domain mutation must remain cancelable while blocked" + ); + drop(fixed_mutation_barrier); + + cleanup_barrier.release(); + worker + .await + .expect("decommission entry worker should join") + .expect("decommission entry should migrate and clean its source"); + delete + .await + .expect("DELETE task should join") + .expect("DELETE should remove the source and migrated target generations"); + + for pool in &store.pools { + let err = pool + .get_object_info(&bucket, &object, &ObjectOptions::default()) + .await + .expect_err("DELETE must remove the source and migrated target copies"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected post-delete pool result: {err:?}" + ); + } + let err = store + .get_object_info(&bucket, &object, &ObjectOptions::default()) + .await + .expect_err("the deleted generation must not become visible again"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected post-delete store result: {err:?}" + ); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_target_put_commit() { + let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4()); + let object = "ordinary.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission PUT fence-loss bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write decommission PUT source"); + mark_test_pool_decommissioning(&store, 0).await; + + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::Migration, + ); + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::BeforeQuotaRename, + ); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + worker + .await + .expect("decommission PUT fence-loss worker should join") + .expect("a fenced migration failure should remain retryable at entry scope"); + + assert_pool_object_absent(&store.pools[1], &bucket, object).await; + assert_pool_object_present(&store.pools[0], &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_multipart_commits() { + let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-fence-loss", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission multipart fence-loss bucket"); + for object in ["new-upload.bin", "complete.bin"] { + write_decommission_test_multipart_source(&store, 0, &bucket, object).await; + } + mark_test_pool_decommissioning(&store, 0).await; + + for (object, pause) in [ + ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost), + ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost), + ] { + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::Migration, + ); + let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost) + .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object)); + let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + worker + .await + .expect("decommission multipart fence-loss worker should join") + .expect("a fenced multipart migration failure should remain retryable at entry scope"); + + if let Some(commit_observation) = commit_observation { + assert!( + !commit_observation.committed(), + "new multipart upload metadata must not commit after the outer fence is lost" + ); + } + assert_pool_object_absent(&store.pools[1], &bucket, object).await; + assert_pool_object_present(&store.pools[0], &bucket, object).await; + let uploads = store.pools[1] + .list_multipart_uploads(&bucket, object, None, None, None, 100) + .await + .expect("list target multipart uploads after fenced migration"); + assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging"); + } + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() { + let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4()); + let object = "cleanup.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission cleanup fence-loss bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write decommission cleanup source"); + mark_test_pool_decommissioning(&store, 0).await; + + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup, + ); + let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + let err = worker + .await + .expect("decommission cleanup fence-loss worker should join") + .expect_err("source cleanup must fail after its outer fence is lost"); + assert!( + err.to_string().contains("delete_object_commit"), + "cleanup failure must come from the delete commit fence: {err:?}" + ); + + assert_pool_object_present(&store.pools[0], &bucket, object).await; + assert_pool_object_present(&store.pools[1], &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() { + let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "reverse-decommission-fixed-target", + &[(1, 4), (1, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4()); + let object = "ordinary.bin"; + let object_body = b"reverse ordinary generation".to_vec(); + let multipart_object = "multipart.bin"; + let first_part = vec![b'm'; 5 * 1024 * 1024]; + let second_part = b"reverse multipart tail".to_vec(); + let mut multipart_body = first_part.clone(); + multipart_body.extend_from_slice(&second_part); + + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create reverse decommission bucket"); + let mut source = PutObjReader::from_vec(object_body.clone()); + store.pools[1] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write ordinary source object to pool 1"); + + let upload = store.pools[1] + .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect("create source multipart upload in pool 1"); + let mut completed_parts = Vec::with_capacity(2); + for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] { + let mut reader = PutObjReader::from_vec(bytes.to_vec()); + let part = store.pools[1] + .put_object_part( + &bucket, + multipart_object, + &upload.upload_id, + part_number, + &mut reader, + &ObjectOptions::default(), + ) + .await + .expect("write source multipart part"); + completed_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + store.pools[1] + .clone() + .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .expect("complete source multipart object in pool 1"); + + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[1].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source"); + + let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let source_set = store.pools[1].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 1, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + commit_barrier.wait_until_paused().await; + + let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object(&delete_bucket, object, ObjectOptions::default()) + .await + }); + delete_barrier.wait_until_paused().await; + delete_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !delete_barrier.namespace_acquired() && !delete.is_finished(), + "the reverse target commit must keep DELETE behind the fixed read fence" + ); + delete.abort(); + assert!( + delete + .await + .expect_err("the blocked DELETE should be canceled") + .is_cancelled(), + "canceling the blocked DELETE must not mutate either pool" + ); + drop(delete_barrier); + + commit_barrier.release(); + drop(commit_barrier); + tokio::time::timeout(Duration::from_secs(60), worker) + .await + .expect("reverse ordinary decommission must not self-deadlock on the fixed target set") + .expect("reverse ordinary decommission worker should join") + .expect("reverse ordinary decommission should complete"); + + let mut ordinary_reader = store.pools[0] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read the ordinary object from the fixed target set"); + let mut ordinary_target_body = Vec::new(); + ordinary_reader + .stream + .read_to_end(&mut ordinary_target_body) + .await + .expect("drain the ordinary target body"); + assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body"); + let ordinary_source_err = store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("ordinary source generation must be cleaned after migration"); + assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _))); + + let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object); + let multipart_store = Arc::clone(&store); + let multipart_bucket = bucket.clone(); + let multipart_worker = tokio::spawn(async move { + multipart_store + .decommission_entry_for_test( + 1, + MetaCacheEntry { + name: multipart_object.to_string(), + ..Default::default() + }, + multipart_bucket, + multipart_source_set, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(60), multipart_worker) + .await + .expect("reverse multipart decommission must not self-deadlock on new or complete") + .expect("reverse multipart decommission worker should join") + .expect("reverse multipart decommission should complete"); + + let target_info = store.pools[0] + .get_object_info(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect("read migrated multipart metadata from the fixed target set"); + assert!(target_info.is_multipart(), "migration must retain multipart identity"); + let mut multipart_reader = store.pools[0] + .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read migrated multipart object from the fixed target set"); + let mut multipart_target_body = Vec::new(); + multipart_reader + .stream + .read_to_end(&mut multipart_target_body) + .await + .expect("drain the multipart target body"); + assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body"); + let multipart_source_err = store.pools[1] + .get_object_info(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect_err("multipart source generation must be cleaned after migration"); + assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _))); + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() { + let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + for source_pool_idx in [0, 1] { + { + let mut pool_meta = store.pool_meta.write().await; + for pool in &mut pool_meta.pools { + pool.decommission = None; + } + } + + let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4()); + let object_names = vec![ + format!("third-{source_pool_idx}.bin"), + format!("first-{source_pool_idx}.bin"), + format!("second-{source_pool_idx}.bin"), + ]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create batch delete pool-error bucket"); + for pool in &store.pools { + for object_name in &object_names { + let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes()); + pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default()) + .await + .expect("seed each object in both the source and active pools"); + } + } + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!( + store.is_suspended(source_pool_idx).await, + "the injected error pool must be the decommission source" + ); + + let expected_errors = [ + StorageError::ErasureWriteQuorum, + StorageError::NamespaceLockQuorumUnavailable { + mode: "delete_objects_commit", + bucket: bucket.clone(), + object: object_names[1].clone(), + required: 3, + achieved: 2, + }, + StorageError::ErasureWriteQuorum, + ]; + let injection = crate::store::object::BatchDeletePoolErrorInjection::install( + &bucket, + source_pool_idx, + object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(), + ); + let requests = object_names + .iter() + .map(|object_name| ObjectToDelete { + object_name: object_name.clone(), + ..Default::default() + }) + .collect(); + + let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await; + + assert_eq!( + injection.observed(), + object_names.len(), + "the source pool must first complete every real delete" + ); + assert_eq!( + errors, + expected_errors.iter().cloned().map(Some).collect::>(), + "a successful pool must not clear a source pool failure at any request index" + ); + assert_eq!( + deleted.iter().map(|object| object.object_name.as_str()).collect::>(), + object_names.iter().map(String::as_str).collect::>(), + "DeleteObjects must preserve request index mapping while aggregating pool failures" + ); + assert!( + deleted.iter().all(|object| object.found), + "the injected source results must retain real delete success data" + ); + + for pool in &store.pools { + for object_name in &object_names { + let error = pool + .get_object_info(&bucket, object_name, &ObjectOptions::default()) + .await + .expect_err("both the active and source pool delete calls must execute"); + assert!( + matches!(error, StorageError::ObjectNotFound(_, _)), + "unexpected residual object: {error:?}" + ); + } + } + drop(injection); + } + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() { + let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "multi-set-decommission-source-cleanup", + &[(2, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4()); + let object = (0..128) + .map(|index| format!("object-{index}.bin")) + .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1) + .expect("the deterministic object search should select source set 1"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create multi-set decommission cleanup bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, &object, &mut source, &ObjectOptions::default()) + .await + .expect("write the source generation to set 1"); + let source_set = store.pools[0].get_disks_by_key(&object); + assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace"); + let expected_source_versions = source_set + .load_file_info_versions_exact(&bucket, &object) + .await + .expect("source versions should be readable") + .expect("the source generation should exist"); + + let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object); + let cleanup_store = Arc::clone(&store); + let cleanup_bucket = bucket.clone(); + let cleanup_object = object.clone(); + let cleanup = tokio::spawn(async move { + let mutation_fence = cleanup_store + .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref()) + .await?; + crate::data_movement::cleanup_source_entry_if_unchanged( + source_set, + &cleanup_bucket, + &cleanup_object, + &expected_source_versions, + &[], + crate::data_movement::SourceCleanupBucketFence { + object_mutation_fence: Some(&mutation_fence), + ..Default::default() + }, + "test_multi_set_decommission_source_cleanup", + ) + .await + }); + cleanup_barrier.wait_until_paused().await; + + let put_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::BeforeNamespace, + ); + let mutation_pool = Arc::clone(&store.pools[0]); + let mutation_bucket = bucket.clone(); + let mutation_object = object.clone(); + let replacement = b"replacement generation".to_vec(); + let expected_replacement = replacement.clone(); + let mutation = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(replacement); + mutation_pool + .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default()) + .await + }); + put_barrier.wait_until_paused().await; + put_barrier.release_and_wait_until_namespace_pending().await; + assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock"); + + cleanup_barrier.release(); + cleanup + .await + .expect("source cleanup task should join") + .expect("source cleanup should remove only the preflight generation"); + mutation + .await + .expect("source mutation task should join") + .expect("source mutation should commit after cleanup releases the set lock"); + + let mut reader = store.pools[0] + .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("the replacement generation must remain readable"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("read the replacement generation"); + assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn versioned_delete_marker_survives_decommission_source_cleanup() { + let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create versioned decommission delete-fence bucket"); + let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object( + &store, + &bucket, + object, + b"source generation", + "test_versioned_decommission_delete_fence", + ) + .await; + + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + &delete_bucket, + object, + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + let cleanup_set = store.pools[0].get_disks_by_key(object); + crate::data_movement::ensure_source_cleanup_versions_unchanged( + Arc::clone(&cleanup_set), + &bucket, + object, + &expected_source_versions, + &[], + "test_versioned_decommission_delete_fence", + ) + .await + .expect("the committed delete marker must not be published to the suspended source pool"); + + let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object); + let cleanup_store = Arc::clone(&store); + let cleanup_bucket = bucket.clone(); + let cleanup = tokio::spawn(async move { + let mutation_fence = cleanup_store + .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref()) + .await?; + crate::data_movement::cleanup_source_entry_if_unchanged( + cleanup_set, + &cleanup_bucket, + object, + &expected_source_versions, + &[], + crate::data_movement::SourceCleanupBucketFence { + object_mutation_fence: Some(&mutation_fence), + ..Default::default() + }, + "test_versioned_decommission_delete_fence", + ) + .await + }); + cleanup_delete_barrier.wait_until_fence_pending().await; + assert!( + !cleanup_delete_barrier.is_paused(), + "source cleanup must wait for the versioned DELETE mutation fence" + ); + + delete_barrier.release(); + let marker = delete + .await + .expect("versioned DELETE task should join") + .expect("versioned DELETE should publish a delete marker after migration"); + assert!(marker.delete_marker, "versioned DELETE must publish a delete marker"); + assert!( + marker.version_id.is_some_and(|version_id| !version_id.is_nil()), + "the delete marker must have a non-nil version ID" + ); + + cleanup_delete_barrier.wait_until_paused().await; + cleanup_delete_barrier.release(); + cleanup + .await + .expect("source cleanup task should join") + .expect("source cleanup should preserve the active-pool delete marker"); + + let err = store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("the post-migration delete marker must hide the migrated version"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected latest-version result: {err:?}" + ); + store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + ..Default::default() + }, + ) + .await + .expect("the migrated source version must remain addressable below the delete marker"); + store.pools[0] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + ..Default::default() + }, + ) + .await + .expect_err("source cleanup must remove the decommissioned source versions"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn versioned_batch_delete_marker_skips_decommission_source() { + let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "versioned-batch-decommission-delete-fence", + &[4, 4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "batch-object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create versioned batch decommission bucket"); + let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object( + &store, + &bucket, + object, + b"batch source generation", + "test_versioned_batch_decommission_delete_fence", + ) + .await; + + let delete_config_snapshot = + Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)), + ..Default::default() + }, + None, + )); + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_objects( + &delete_bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions { + delete_replication_config_snapshot: Some(delete_config_snapshot), + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + + let source_set = store.pools[0].get_disks_by_key(object); + crate::data_movement::ensure_source_cleanup_versions_unchanged( + source_set, + &bucket, + object, + &expected_source_versions, + &[], + "test_versioned_batch_decommission_delete_fence", + ) + .await + .expect("batch DELETE must not publish a marker to the suspended source"); + + delete_barrier.release(); + let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join"); + assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}"); + assert_eq!(deleted.len(), 1); + assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker"); + assert!( + deleted[0] + .delete_marker_version_id + .is_some_and(|version_id| !version_id.is_nil()), + "versioned batch DELETE marker must have a non-nil version ID" + ); + + let mut active_marker_count = 0; + for pool in store.pools.iter().skip(1) { + let Some(versions) = pool + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("active-pool versions should be readable") + else { + continue; + }; + active_marker_count += versions.versions.iter().filter(|version| version.deleted).count(); + } + assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn suspended_delete_marker_then_decommission_worker_converges_null_source() { + let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "suspended-decommission-delete-convergence", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "single.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create suspended decommission DELETE bucket"); + write_suspended_decommission_source(&store, &bucket, object).await; + mark_test_pool_decommissioning(&store, 0).await; + + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + &delete_bucket, + object, + ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + assert_suspended_null_source_present(&store, &bucket, object).await; + + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + delete_barrier.release(); + let marker = delete + .await + .expect("suspended DELETE task should join") + .expect("suspended DELETE should commit its active-pool marker"); + drop(delete_barrier); + assert!(marker.delete_marker, "suspended DELETE must create a marker"); + assert!( + marker.version_id.is_none_or(|version_id| version_id.is_nil()), + "suspended DELETE marker must keep the null version identity" + ); + worker + .await + .expect("suspended decommission worker should join") + .expect("worker must treat the newer active null marker as a completed migration"); + + assert_suspended_decommission_converged(&store, &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() { + let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "suspended-batch-decommission-delete-convergence", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "batch.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create suspended batch decommission DELETE bucket"); + write_suspended_decommission_source(&store, &bucket, object).await; + mark_test_pool_decommissioning(&store, 0).await; + + let delete_config_snapshot = + Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)), + ..Default::default() + }, + None, + )); + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_objects( + &delete_bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions { + delete_replication_config_snapshot: Some(delete_config_snapshot), + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + assert_suspended_null_source_present(&store, &bucket, object).await; + + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + delete_barrier.release(); + let (deleted, errors) = delete.await.expect("suspended batch DELETE task should join"); + drop(delete_barrier); + assert!(errors.iter().all(Option::is_none), "suspended batch DELETE should succeed: {errors:?}"); + assert!( + matches!(deleted.as_slice(), [marker] if marker.delete_marker && marker.delete_marker_version_id.is_none_or(|version_id| version_id.is_nil())), + "suspended batch DELETE must create one null marker: {deleted:?}" + ); + worker + .await + .expect("suspended batch decommission worker should join") + .expect("worker must treat the newer batch null marker as a completed migration"); + + assert_suspended_decommission_converged(&store, &bucket, object).await; + shutdown.cancel(); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 757be0c53..a40edd929 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -151,7 +151,7 @@ pub(crate) mod init_format; pub(crate) mod list_objects; mod multipart; mod object; -pub(crate) use object::ObjectLockDiagGuard; +pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence}; pub use object::{ PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError, SnapshotConsistencyError, diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 2c0b18b2a..8d72287d4 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -400,7 +400,7 @@ impl ECStore { object: &str, opts: &ObjectOptions, ) -> Result { - self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts) + self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None) .await .map(|(res, _, _)| res) } @@ -410,20 +410,22 @@ impl ECStore { bucket: &str, object: &str, opts: &ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<(MultipartUploadResult, usize, Option)> { check_new_multipart_args(bucket, object)?; - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; - let opts = &opts; + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; if self.single_pool() { + self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence) + .await; return self.pools[0] - .new_multipart_upload(bucket, object, opts) + .new_multipart_upload(bucket, object, &opts) .await .map(|res| (res, 0, opts.expected_bucket_incarnation_id)); } if opts.data_movement && opts.version_id.is_some() { - let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?; + let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?; if idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( bucket.to_owned(), @@ -431,7 +433,9 @@ impl ECStore { opts.version_id.clone().unwrap_or_default(), )); } - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; return Ok((res, idx, opts.expected_bucket_incarnation_id)); } @@ -454,7 +458,9 @@ impl ECStore { .await?; if !res.uploads.is_empty() { - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; return Ok((res, idx, opts.expected_bucket_incarnation_id)); } } @@ -467,7 +473,9 @@ impl ECStore { )); } - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; Ok((res, idx, opts.expected_bucket_incarnation_id)) } @@ -704,13 +712,14 @@ impl ECStore { pub(crate) async fn complete_multipart_upload_for_data_movement( self: Arc, - target_pool_idx: usize, + target: (usize, Option<&ObjectLockDiagGuard>), bucket: &str, object: &str, upload_id: &str, uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result { + let (target_pool_idx, mutation_fence) = target; check_complete_multipart_args(bucket, object, upload_id)?; if !opts.data_movement { return Err(Error::other("targeted multipart completion requires data_movement options")); @@ -739,6 +748,8 @@ impl ECStore { snapshot.add_lock_fences(&mut opts); opts.object_lock_config_snapshot = Some(snapshot); } + self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence) + .await; #[cfg(test)] pause_data_movement_multipart_before_selected_completion(bucket).await; let pool = self diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index e5f2f0465..79a8232f7 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -32,12 +32,13 @@ use crate::bucket::metadata_sys::{ use crate::bucket::object_lock::objectlock_sys::{ check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state, }; -use crate::bucket::replication::ReplicationObjectBridge; +use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge}; +use crate::bucket::versioning::VersioningApi; use crate::disk::OldCurrentSize; use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot}; use crate::set_disk::{ - get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, - is_lock_optimization_enabled, is_object_lock_diag_enabled, + SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, + is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain, }; use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, @@ -352,6 +353,8 @@ impl fmt::Display for ObjectLockDiagMode { pub(crate) struct ObjectLockDiagGuard { guard: rustfs_lock::NamespaceLockGuard, + #[cfg(test)] + test_namespace_lock_fence: Option, enabled: bool, op: &'static str, bucket: Option, @@ -373,6 +376,8 @@ impl ObjectLockDiagGuard { ) -> Self { Self { guard, + #[cfg(test)] + test_namespace_lock_fence: None, enabled, op, bucket, @@ -393,6 +398,115 @@ impl ObjectLockDiagGuard { pub(crate) fn is_lock_lost(&self) -> bool { self.guard.is_lock_lost() } + + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + opts.ensure_namespace_lock_fence(); + if let Some(signal) = self.lock_lost_signal() { + opts.add_namespace_lock_lost_signal(signal); + } + #[cfg(test)] + if let Some(fence) = self.test_namespace_lock_fence.as_ref() { + opts.add_namespace_lock_fence_for_test(fence); + } + } +} + +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DecommissionMutationFenceTestPhase { + Migration, + SourceCleanup, +} + +#[cfg(test)] +struct DecommissionMutationFenceLossState { + bucket: String, + object: String, + phase: DecommissionMutationFenceTestPhase, + fence: NamespaceLockFence, + loss_handle: Arc, +} + +#[cfg(test)] +pub(crate) struct DecommissionMutationFenceLossHook { + state: Arc, +} + +#[cfg(test)] +static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DecommissionMutationFenceLossHook { + pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self { + let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test(); + let state = Arc::new(DecommissionMutationFenceLossState { + bucket: bucket.to_string(), + object: object.to_string(), + phase, + fence, + loss_handle, + }); + let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison"); + assert!(slot.is_none(), "decommission mutation fence loss hook must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn mark_lost(&self) { + self.state.loss_handle.store(true, Ordering::Release); + } +} + +#[cfg(test)] +impl Drop for DecommissionMutationFenceLossHook { + fn drop(&mut self) { + let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison"); + if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn decommission_mutation_fence_for_test( + bucket: &str, + object: &str, + phase: DecommissionMutationFenceTestPhase, +) -> Option { + DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison") + .as_ref() + .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase) + .map(|hook| hook.fence.clone()) +} + +pub(crate) struct SourceCleanupMutationFence { + guard: ObjectLockDiagGuard, + source_lock_covered: bool, +} + +impl SourceCleanupMutationFence { + pub(crate) fn source_lock_covered(&self) -> bool { + self.source_lock_covered + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.guard.is_lock_lost() + } + + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + self.guard.add_namespace_lock_fence(opts); + } } /// Opaque write-lock guard for the RestoreObject accept path; see @@ -410,10 +524,7 @@ impl RestoreAcceptGuard { } pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { - opts.ensure_namespace_lock_fence(); - if let Some(signal) = self.0.lock_lost_signal() { - opts.add_namespace_lock_lost_signal(signal); - } + self.0.add_namespace_lock_fence(opts); } } @@ -690,16 +801,6 @@ impl SelectObjectSnapshotLockLossWake { } } -// LockRegistry clones its canonical client Arc for each endpoint host, so an -// exact Arc set identifies one distributed namespace-lock quorum domain. -fn same_distributed_lock_domain(left: &[Arc], right: &[Arc]) -> bool { - left.iter() - .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client))) - && right - .iter() - .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client))) -} - impl AsyncRead for SelectObjectSnapshotReader { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() { @@ -805,7 +906,7 @@ fn resolve_latest_object_access( } fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool { - opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement + (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement } #[cfg(test)] @@ -813,6 +914,8 @@ struct DeleteAfterObjectLockSnapshotBarrierState { bucket: String, arrived: tokio::sync::Notify, release: tokio::sync::Notify, + namespace_pending: tokio::sync::Notify, + namespace_acquired: AtomicBool, } #[cfg(test)] @@ -832,6 +935,8 @@ impl DeleteAfterObjectLockSnapshotBarrier { bucket: bucket.to_string(), arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), + namespace_pending: tokio::sync::Notify::new(), + namespace_acquired: AtomicBool::new(false), }); let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -849,6 +954,18 @@ impl DeleteAfterObjectLockSnapshotBarrier { pub(crate) fn release(&self) { self.state.release.notify_one(); } + + pub(crate) async fn release_and_wait_until_namespace_pending(&self) { + let namespace_pending = self.state.namespace_pending.notified(); + self.release(); + tokio::time::timeout(Duration::from_secs(5), namespace_pending) + .await + .expect("delete should proceed to its namespace lock after leaving the snapshot barrier"); + } + + pub(crate) fn namespace_acquired(&self) -> bool { + self.state.namespace_acquired.load(Ordering::Acquire) + } } #[cfg(test)] @@ -873,6 +990,97 @@ async fn pause_delete_after_object_lock_snapshot(bucket: &str) { .as_ref() .filter(|state| state.bucket == bucket) .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + state.namespace_pending.notify_one(); + } +} + +#[cfg(test)] +fn notify_delete_namespace_acquired(bucket: &str) { + let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete snapshot barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket) + .cloned(); + if let Some(state) = state { + state.namespace_acquired.store(true, Ordering::Release); + } +} + +#[cfg(test)] +struct VersionedDeleteMarkerCommitBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct VersionedDeleteMarkerCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl VersionedDeleteMarkerCommitBarrier { + pub(crate) fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(VersionedDeleteMarkerCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison"); + assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("versioned DELETE should reach the post-marker-commit barrier"); + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for VersionedDeleteMarkerCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) { + let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.object == object) + .cloned(); if let Some(state) = state { state.arrived.notify_one(); state.release.notified().await; @@ -913,6 +1121,160 @@ fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions lookup_opts } +fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { + let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock); + lookup_opts.skip_decommissioned = opts.data_movement; + lookup_opts +} + +fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool { + pool_count > 0 && (!opts.versioned && !opts.version_suspended || opts.version_id.is_some()) +} + +fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool { + if object.version_id.is_some() { + return false; + } + + let object_name = decode_dir_object(&object.object_name); + let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name); + versioned || version_suspended +} + +fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option, pool_idx: usize) -> bool { + !creates_latest_marker || marker_target_pool_idx == Some(pool_idx) +} + +#[cfg(test)] +struct BatchDeletePoolErrorInjectionState { + bucket: String, + pool_idx: usize, + errors: std::collections::HashMap, + observed: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +pub(crate) struct BatchDeletePoolErrorInjection { + state: Arc, +} + +#[cfg(test)] +static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl BatchDeletePoolErrorInjection { + pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self { + let state = Arc::new(BatchDeletePoolErrorInjectionState { + bucket: bucket.to_string(), + pool_idx, + errors: errors.into_iter().collect(), + observed: std::sync::atomic::AtomicUsize::new(0), + }); + let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison"); + assert!(slot.is_none(), "batch delete pool error injection must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn observed(&self) -> usize { + self.state.observed.load(Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for BatchDeletePoolErrorInjection { + fn drop(&mut self) { + let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn inject_batch_delete_pool_errors( + bucket: &str, + pool_idx: usize, + object_names: &[String], + result: &mut (Vec, Vec>), +) { + let state = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx) + .cloned(); + let Some(state) = state else { + return; + }; + + for (idx, object_name) in object_names.iter().enumerate() { + let Some(error) = state.errors.get(object_name) else { + continue; + }; + if result.1[idx].is_none() && result.0[idx].found { + result.1[idx] = Some(error.clone()); + state.observed.fetch_add(1, Ordering::AcqRel); + } + } +} + +fn resolve_batch_delete_pool_results<'a>( + initial_error: Option, + pool_results: impl IntoIterator)>, +) -> (Option, Option, bool) { + let mut failure = initial_error.map(|err| (None, err)); + let mut deleted = None; + let mut fallback: Option<(DeletedObject, Option)> = None; + let mut attempted = false; + + for (pool_delete, pool_error) in pool_results { + attempted = true; + match pool_error { + Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => { + if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) { + fallback = Some(((*pool_delete).clone(), Some(err.clone()))); + } + } + Some(err) => { + if failure.is_none() { + failure = Some((Some((*pool_delete).clone()), err.clone())); + } + } + None if pool_delete.found => { + if deleted.is_none() { + deleted = Some((*pool_delete).clone()); + } + } + None => { + if fallback.is_none() { + fallback = Some(((*pool_delete).clone(), None)); + } + } + } + } + + if let Some((failed_delete, err)) = failure { + return (failed_delete, Some(err), attempted); + } + if let Some(deleted) = deleted { + return (Some(deleted), None, attempted); + } + if let Some((deleted, err)) = fallback { + return (Some(deleted), err, attempted); + } + + (None, None, attempted) +} + fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions { let mut lookup_opts = opts.clone(); lookup_opts.skip_decommissioned = true; @@ -1541,6 +1903,89 @@ impl ECStore { ))) } + pub(crate) async fn acquire_decommission_object_mutation_fence( + &self, + bucket: &str, + object: &str, + ) -> Result { + if self.ctx.lock_manager().is_disabled() { + return Err(Error::other("decommission object migration requires namespace locking")); + } + + #[cfg(test)] + let test_namespace_lock_fence = + decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration); + let object = encode_dir_object(object); + let mut opts = ObjectOptions::default(); + let guard = self + .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts) + .await? + .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?; + #[cfg(test)] + let guard = { + let mut guard = guard; + guard.test_namespace_lock_fence = test_namespace_lock_fence; + guard + }; + Ok(guard) + } + + pub(super) async fn apply_decommission_target_mutation_fence( + &self, + target_pool_idx: usize, + object: &str, + opts: &mut ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, + ) { + let Some(mutation_fence) = mutation_fence else { + return; + }; + + mutation_fence.add_namespace_lock_fence(opts); + let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()); + let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object)); + opts.no_lock = match (fixed_set, target_set) { + (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await, + _ => false, + }; + } + + pub(crate) async fn acquire_decommission_source_cleanup_fence( + &self, + bucket: &str, + object: &str, + source_set: &SetDisks, + ) -> Result { + if self.ctx.lock_manager().is_disabled() { + return Err(Error::other("decommission source cleanup requires namespace locking")); + } + + #[cfg(test)] + crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object); + #[cfg(test)] + let test_namespace_lock_fence = + decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup); + let object = encode_dir_object(object); + let fixed_set = Arc::clone(&self.pools[0].disk_set[0]); + let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await; + // Lock order: fixed store mutation domain first; source cleanup takes its + // hashed source-domain lock second only when this guard does not cover it. + let guard = self + .acquire_object_write_lock("decommission_source_cleanup", bucket, &object) + .await?; + #[cfg(test)] + let guard = { + let mut guard = guard; + guard.test_namespace_lock_fence = test_namespace_lock_fence; + guard + }; + + Ok(SourceCleanupMutationFence { + guard, + source_lock_covered, + }) + } + pub(crate) async fn acquire_all_object_read_locks( &self, op: &'static str, @@ -1994,14 +2439,17 @@ impl ECStore { object: &str, data: &mut PutObjReader, opts: &ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<(usize, Result)> { if !opts.data_movement { return Err(Error::other("data movement PUT requires data_movement options")); } - let (object, opts) = self.prepare_put_object(bucket, object, opts).await?; + let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?; let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; + self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence) + .await; let result = self.pools[idx] .put_object_with_old_current_size(bucket, &object, data, &opts) .await @@ -2470,6 +2918,10 @@ impl ECStore { } else { None }; + #[cfg(test)] + if _object_lock_guard.is_some() { + notify_delete_namespace_acquired(bucket); + } if let Some(trigger) = opts.lifecycle_delete_all.as_ref() { let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?; let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; @@ -2503,7 +2955,7 @@ impl ECStore { return Ok(ObjectInfo::default()); } - let gopts = writer_pool_lookup_opts(&opts, true); + let gopts = delete_pool_lookup_opts(&opts, true); if opts.data_movement { let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await; @@ -2608,6 +3060,8 @@ impl ECStore { Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => { let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?; let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + #[cfg(test)] + pause_versioned_delete_marker_after_commit(bucket, object).await; obj.name = decode_dir_object(object); return Ok(obj); } @@ -2646,7 +3100,7 @@ impl ECStore { None }; - if !errs.is_empty() && !opts.versioned && !opts.version_suspended { + if should_delete_from_all_pools(&opts, errs.len()) { let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await { Ok(obj) => obj, Err(err) => { @@ -2670,6 +3124,8 @@ impl ECStore { match pool.delete_object(bucket, object, opts.clone()).await { Ok(res) => { + #[cfg(test)] + pause_versioned_delete_marker_after_commit(bucket, object).await; if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { commit_prepared_tier_delete_journal_entry(api, je).await; } @@ -2817,32 +3273,104 @@ impl ECStore { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), }; + #[cfg(test)] + if !_object_lock_guards.is_empty() { + notify_delete_namespace_acquired(bucket); + } + + let delete_config_snapshot = opts + .delete_replication_config_snapshot + .as_deref() + .expect("batch delete replication config snapshot should be loaded"); + let latest_marker_objects = objects + .iter() + .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot)) + .collect::>(); + let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map( + |(object, creates_marker)| async move { + if *creates_marker { + Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await) + } else { + None + } + }, + )) + .await; + let mut marker_target_pool_indices = Vec::with_capacity(objects.len()); + for (idx, target_result) in marker_target_results.into_iter().enumerate() { + match target_result { + Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)), + Some(Err(err)) => { + del_errs[idx] = Some(err); + marker_target_pool_indices.push(None); + } + None => marker_target_pool_indices.push(None), + } + } let mut futures = Vec::with_capacity(self.pools.len()); - for pool in self.pools.iter() { if self.is_pool_rebalancing(pool.pool_idx).await { continue; } - futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone())); + + let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects + .iter() + .enumerate() + .filter(|(idx, _)| { + batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx) + }) + .map(|(idx, object)| (idx, object.clone())) + .unzip(); + if pool_objects.is_empty() { + continue; + } + + let pool_opts = opts.clone(); + futures.push(async move { + #[cfg(test)] + let pool_object_names = pool_objects + .iter() + .map(|object| object.object_name.clone()) + .collect::>(); + let result = pool.delete_objects(bucket, pool_objects, pool_opts).await; + #[cfg(test)] + let result = { + let mut result = result; + inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result); + result + }; + (object_indices, result) + }); } let results = join_all(futures).await; for idx in 0..del_objects.len() { - for (dels, errs, pool_accounting) in results.iter() { - if errs[idx].is_none() && dels[idx].found { - del_errs[idx] = None; - del_objects[idx] = dels[idx].clone(); - accounting[idx] = pool_accounting[idx].clone(); - break; - } + let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| { + let pool_object_idx = object_indices.binary_search(&idx).ok()?; + Some((&dels[pool_object_idx], &errs[pool_object_idx])) + }); + let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results); + if let Some(deleted) = deleted { + del_objects[idx] = deleted; + } + del_errs[idx] = error; - if del_errs[idx].is_none() { - del_errs[idx] = errs[idx].clone(); - del_objects[idx] = dels[idx].clone(); - accounting[idx] = pool_accounting[idx].clone(); - } + if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] { + del_objects[idx] = DeletedObject { + object_name: objects[idx].object_name.clone(), + version_id: objects[idx].version_id, + ..Default::default() + }; + del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone())); + } + } + + #[cfg(test)] + for (idx, object) in objects.iter().enumerate() { + if del_errs[idx].is_none() && del_objects[idx].delete_marker { + pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await; } } @@ -3417,6 +3945,80 @@ mod tests { assert!(!same_distributed_lock_domain(&[first, second], &[other])); } + #[tokio::test] + async fn decommission_fence_covers_dist_sets_with_same_clients_despite_different_namespaces() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await; + let mut second_set = (*original_sets.disk_set[1]).clone(); + second_set.lockers = original_sets.disk_set[0].lockers.clone(); + let mut sets = (*original_sets).clone(); + sets.disk_set[1] = Arc::new(second_set); + let sets = Arc::new(sets); + ctx.update_erasure_type(SetupType::DistErasure).await; + + assert!( + sets.disk_set[0] + .lockers + .iter() + .zip(&sets.disk_set[1].lockers) + .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)), + "the regression requires identical distributed lock clients" + ); + assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index); + + let pool_config = sets.endpoints.clone(); + let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx); + let object = (0..1_000) + .map(|index| format!("decommission-dist-domain-{index}.bin")) + .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1])) + .expect("a key should hash to the second set namespace"); + let mutation_fence = store + .acquire_decommission_object_mutation_fence("bucket", &object) + .await + .expect("the fixed distributed mutation fence should be acquired"); + let target_lock = sets.disk_set[1] + .new_ns_lock("bucket", &object) + .await + .expect("the hashed-set namespace lock should be created"); + let target_err = target_lock + .get_write_lock(Duration::from_millis(50)) + .await + .expect_err("the fixed read fence must conflict through the shared clients"); + assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. })); + + let mut put_opts = ObjectOptions::default(); + store + .apply_decommission_target_mutation_fence(0, &object, &mut put_opts, Some(&mutation_fence)) + .await; + assert!(put_opts.no_lock, "migration target PUT must reuse the covering fixed fence"); + + let mut multipart_opts = ObjectOptions::default(); + store + .apply_decommission_target_mutation_fence(0, &object, &mut multipart_opts, Some(&mutation_fence)) + .await; + assert!(multipart_opts.no_lock, "migration target multipart must reuse the covering fixed fence"); + drop(mutation_fence); + + let cleanup_object = (0..1_000) + .map(|index| format!("decommission-dist-cleanup-{index}.bin")) + .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1])) + .expect("a cleanup key should hash to the second set namespace"); + let source_fence = store + .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref()) + .await + .expect("the fixed distributed cleanup fence should be acquired"); + assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence"); + let source_lock = sets.disk_set[1] + .new_ns_lock("bucket", &cleanup_object) + .await + .expect("the source-set namespace lock should be created"); + let source_err = source_lock + .get_read_lock(Duration::from_millis(50)) + .await + .expect_err("the fixed write fence must conflict through the shared clients"); + assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. })); + } + #[test] fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() { let nil = Uuid::nil(); @@ -4476,6 +5078,159 @@ mod tests { assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1")); } + #[test] + fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() { + let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true); + + assert!(lookup_opts.no_lock); + assert!(!lookup_opts.skip_decommissioned); + assert!(lookup_opts.skip_rebalancing); + + let explicit_version = delete_pool_lookup_opts( + &ObjectOptions { + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + true, + ); + assert!(!explicit_version.skip_decommissioned); + } + + #[test] + fn delete_fans_out_for_unversioned_and_explicit_version_mutations() { + assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1)); + assert!(should_delete_from_all_pools( + &ObjectOptions { + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + 2, + )); + assert!(!should_delete_from_all_pools( + &ObjectOptions { + versioned: true, + ..Default::default() + }, + 1, + )); + assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0)); + } + + #[test] + fn batch_delete_identifies_only_latest_versioned_markers() { + let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)), + ..Default::default() + }, + None, + ); + let latest = ObjectToDelete { + object_name: "latest".to_string(), + ..Default::default() + }; + assert!(batch_delete_creates_latest_marker(&latest, &versioned)); + assert!(!batch_delete_targets_pool(true, Some(1), 0)); + assert!(batch_delete_targets_pool(true, Some(1), 1)); + assert!(!batch_delete_targets_pool(true, Some(1), 2)); + + let explicit = ObjectToDelete { + object_name: "explicit".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + ..Default::default() + }; + assert!(!batch_delete_creates_latest_marker(&explicit, &versioned)); + assert!(batch_delete_targets_pool(false, Some(1), 0)); + + let unversioned = DeleteReplicationConfigSnapshot::default(); + assert!(!batch_delete_creates_latest_marker(&latest, &unversioned)); + assert!(batch_delete_targets_pool(false, None, 0)); + } + + #[test] + fn batch_delete_pool_failures_override_success_in_any_pool_order() { + let success = DeletedObject { + object_name: "object".to_string(), + found: true, + ..Default::default() + }; + let source_errors = [ + StorageError::ErasureWriteQuorum, + StorageError::NamespaceLockQuorumUnavailable { + mode: "delete_objects_commit", + bucket: "bucket".to_string(), + object: "object".to_string(), + required: 1, + achieved: 0, + }, + ]; + + for source_error in source_errors { + for source_first in [true, false] { + let failed = (DeletedObject::default(), Some(source_error.clone())); + let succeeded = (success.clone(), None); + let pool_results = if source_first { + vec![failed, succeeded] + } else { + vec![succeeded, failed] + }; + + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error))); + + assert!(attempted); + assert_eq!(error, Some(source_error.clone())); + } + } + } + + #[test] + fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() { + let success = DeletedObject { + object_name: "object".to_string(), + found: true, + ..Default::default() + }; + let missing_errors = [ + StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()), + StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()), + ]; + + for missing_error in missing_errors { + let missing = (DeletedObject::default(), Some(missing_error.clone())); + for missing_first in [true, false] { + let succeeded = (success.clone(), None); + let pool_results = if missing_first { + vec![missing.clone(), succeeded] + } else { + vec![succeeded, missing.clone()] + }; + let (deleted, error, attempted) = + resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error))); + + assert!(attempted); + let deleted = deleted.expect("successful pool result should be retained"); + assert!(deleted.found); + assert_eq!(deleted.object_name, success.object_name.as_str()); + assert!(error.is_none()); + } + + let missing_only = [missing]; + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error))); + assert!(attempted); + assert_eq!(error, Some(missing_error)); + } + + let silent_missing = [(DeletedObject::default(), None)]; + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error))); + assert!(attempted); + assert!(error.is_none()); + } + #[test] fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() { let lookup_opts = data_movement_pool_lookup_opts( diff --git a/crates/ecstore/src/store/rebalance/support.rs b/crates/ecstore/src/store/rebalance/support.rs index 6035e58d8..e37fc97bc 100644 --- a/crates/ecstore/src/store/rebalance/support.rs +++ b/crates/ecstore/src/store/rebalance/support.rs @@ -73,7 +73,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result( object: &str, ) -> Result { result.map_err(|err| { - if err == Error::PreconditionFailed { + if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) { err } else { Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")) @@ -86,7 +86,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool { } fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error { - if err == Error::PreconditionFailed { + if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) { err } else { Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}")) @@ -191,6 +191,18 @@ mod tests { assert_eq!(err, Error::PreconditionFailed); } + #[test] + fn rebalance_delete_result_preserves_prefix_access_denied() { + let err = resolve_rebalance_delete_from_all_pools_result( + Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())), + "bucket", + "object", + ) + .expect_err("prefix access denial should remain structured"); + + assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())); + } + #[test] fn rebalance_delete_pool_result_preserves_precondition_failed() { let err = resolve_rebalance_delete_from_all_pools_results( @@ -205,4 +217,19 @@ mod tests { assert_eq!(err, Error::PreconditionFailed); } + + #[test] + fn rebalance_delete_pool_result_preserves_prefix_access_denied() { + let err = resolve_rebalance_delete_from_all_pools_results( + vec![RebalanceDeletePoolResult { + pool_idx: 0, + result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())), + }], + "bucket", + "object", + ) + .expect_err("prefix access denial should remain structured"); + + assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())); + } } From f1b92af4a37267e01b944129ab1f41e2154073bf Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 01:15:38 +0800 Subject: [PATCH 15/32] feat(ecstore): coalesce GET ReadVersion RPCs (#6395) --- crates/ecstore/src/cluster/rpc/remote_disk.rs | 97 ++- .../src/cluster/rpc/runtime_sources.rs | 60 +- crates/ecstore/src/disk/mod.rs | 94 ++- crates/ecstore/src/lib.rs | 8 + crates/ecstore/src/runtime/global.rs | 15 +- .../src/set_disk/core/io_primitives.rs | 572 +++++++++++++++++- crates/ecstore/src/set_disk/ops/object.rs | 2 +- crates/ecstore/src/set_disk/read.rs | 80 ++- crates/io-metrics/src/internode_metrics.rs | 7 + rustfs/src/server/readiness.rs | 5 + rustfs/src/storage/rpc/node_service/disk.rs | 114 +++- rustfs/src/storage/storage_api.rs | 4 + rustfs/src/storage_api.rs | 3 +- 13 files changed, 987 insertions(+), 74 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 32bf3ae57..a47533a81 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -42,8 +42,9 @@ use futures::lock::Mutex; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_io_metrics::internode_metrics::{ - INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, - INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, + INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, + INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, }; use rustfs_protos::ChannelClass; use rustfs_protos::evict_failed_connection; @@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50); const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ"; const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; const BATCH_METADATA_RPC_OFF: &str = "off"; const BATCH_METADATA_RPC_AUTO: &str = "auto"; const BATCH_METADATA_RPC_ON: &str = "on"; @@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode { } fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode { - rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ) + rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) + .or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)) .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC)) .as_deref() .map(parse_batch_metadata_rpc_mode) @@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option) { } } +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed()); + } +} + /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload /// histogram / large-payload alerting (grpc-optimization P0 instrumentation). @@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items( Ok(batch_read_version_resps) } +fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize { + req.items + .iter() + .fold(req_json.len().saturating_add(req_bin.len()), |total, item| { + total + .saturating_add(item.org_volume.len()) + .saturating_add(item.volume.len()) + .saturating_add(item.path.len()) + .saturating_add(item.version_id.len()) + }) +} + +fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize { + response + .batch_read_version_resps + .iter() + .map(String::len) + .sum::() + .saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::()) +} + fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> { file_info.validate_for_metadata_read().map_err(Into::into) } @@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk { state = "started", "Remote disk RPC started" ); + let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled); let batch_read_version_req = compat_json(&req)?; let batch_read_version_req_bin = encode_msgpack(&req)?; - + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started); + let request_payload_bytes = batch_read_version_attribution_enabled + .then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin)); let batch_result = self .execute_with_timeout_for_op( "batch_read_version", move || async move { let disk = self.disk_ref().await; + let disk_len = disk.len(); let mut client = self .get_bulk_client() .await @@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk { batch_read_version_req_bin: batch_read_version_req_bin.into(), }); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request(); + if let Some(request_payload_bytes) = request_payload_bytes { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes( + request_payload_bytes.saturating_add(disk_len), + ); + } + let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled); let response = match client.batch_read_version(request).await { - Ok(response) => response.into_inner(), + Ok(response) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + response.into_inner() + } Err(status) if status.code() == Code::Unimplemented => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); if mode.should_fallback_on_unimplemented() { record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED); warn!( @@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk { } record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); warn!( event = EVENT_REMOTE_DISK_RPC, component = LOG_COMPONENT_ECSTORE, @@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk { ); return Err(Error::from(status)); } - Err(status) => return Err(Error::from(status)), + Err(status) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); + return Err(Error::from(status)); + } }; if !response.success { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); return Err(response.error.unwrap_or_default().into()); } - decode_batch_read_version_response_items(response, &self.endpoint).map(Some) + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes( + batch_read_version_response_payload_len(&response), + ); + let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled); + match decode_batch_read_version_response_items(response, &self.endpoint) { + Ok(batch_read_version_resps) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started); + Ok(Some(batch_read_version_resps)) + } + Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); + Err(err) + } + } }, get_max_timeout_duration(), ) @@ -4621,6 +4687,7 @@ mod tests { } else { "file version not found".to_string() }, + error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() }, } } @@ -4740,6 +4807,7 @@ mod tests { fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() { temp_env::with_vars( [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>), (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")), (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), ], @@ -4749,10 +4817,25 @@ mod tests { ); } + #[test] + fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")), + (ENV_RUSTFS_METADATA_BATCH_READ, Some("off")), + (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")), + ], + || { + assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On); + }, + ); + } + #[test] fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() { temp_env::with_vars( [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>), (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>), (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), ], diff --git a/crates/ecstore/src/cluster/rpc/runtime_sources.rs b/crates/ecstore/src/cluster/rpc/runtime_sources.rs index 0c8393a2e..ab9a34fe9 100644 --- a/crates/ecstore/src/cluster/rpc/runtime_sources.rs +++ b/crates/ecstore/src/cluster/rpc/runtime_sources.rs @@ -14,9 +14,10 @@ use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION, - INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, - INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, + INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, + INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + global_internode_metrics, }; use std::time::Duration; @@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() { ); } +pub(crate) fn record_remote_disk_grpc_batch_read_version_request() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_outgoing_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + duration, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_error() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics() + .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); + record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes); +} + pub(crate) fn record_remote_disk_grpc_read_version_error() { if !rustfs_io_metrics::get_stage_metrics_enabled() { return; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index ecd0179e0..fb8c8de5a 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback"; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_DISK: &str = "disk"; const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified"; +const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM"; +const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4; pub fn part_transaction_path(part_path: &str) -> String { match part_path.rsplit_once('/') { @@ -62,6 +64,7 @@ use bytes::Bytes; use endpoint::Endpoint; use error::DiskError; use error::{Error, Result}; +use futures::stream::{self, StreamExt}; use local::LocalDisk; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_madmin::info_commands::DiskMetrics; @@ -417,6 +420,14 @@ impl DiskAPI for Disk { } } + #[tracing::instrument(level = "trace", skip_all)] + async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result> { + match self { + Disk::Local(local_disk) => local_disk.batch_read_version(req).await, + Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await, + } + } + #[tracing::instrument(level = "trace", skip_all)] async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { match self { @@ -1028,36 +1039,47 @@ where D: DiskAPI + ?Sized, { validate_batch_read_version_item_count(req.items.len())?; + let parallelism = batch_read_version_server_parallelism(); - let mut responses = Vec::with_capacity(req.items.len()); - for (index, item) in req.items.iter().enumerate() { - let response = match disk - .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts) - .await - { - Ok(file_info) => BatchReadVersionResp { - index, - path: item.path.clone(), - version_id: item.version_id.clone(), - success: true, - file_info, - error: String::new(), - }, - Err(err) => BatchReadVersionResp { - index, - path: item.path.clone(), - version_id: item.version_id.clone(), - success: false, - file_info: FileInfo::default(), - error: err.to_string(), - }, - }; - responses.push(response); - } + let mut responses = stream::iter(req.items.into_iter().enumerate()) + .map(|(index, item)| async move { + match disk + .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts) + .await + { + Ok(file_info) => BatchReadVersionResp { + index, + path: item.path, + version_id: item.version_id, + success: true, + file_info, + error: String::new(), + error_code: 0, + }, + Err(err) => BatchReadVersionResp { + index, + path: item.path, + version_id: item.version_id, + success: false, + file_info: FileInfo::default(), + error: err.to_string(), + error_code: err.to_u32(), + }, + } + }) + .buffer_unordered(parallelism) + .collect::>() + .await; + responses.sort_unstable_by_key(|response| response.index); Ok(responses) } +fn batch_read_version_server_parallelism() -> usize { + rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM) + .clamp(1, BATCH_READ_VERSION_MAX_ITEMS) +} + #[derive(Debug, Default, Serialize, Deserialize)] pub struct CheckPartsResp { pub results: Vec, @@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp { pub success: bool, pub file_info: FileInfo, pub error: String, + #[serde(default)] + pub error_code: u32, } pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> { @@ -1417,6 +1441,26 @@ mod tests { assert!(!partial_valid_location.valid()); } + #[test] + fn batch_read_version_server_parallelism_defaults_to_conservative_four() { + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || { + assert_eq!(batch_read_version_server_parallelism(), 4); + }); + } + + #[test] + fn batch_read_version_server_parallelism_honors_env_with_bounds() { + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || { + assert_eq!(batch_read_version_server_parallelism(), 8); + }); + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || { + assert_eq!(batch_read_version_server_parallelism(), 1); + }); + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || { + assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS); + }); + } + /// Test FileInfoVersions find_version_index #[test] fn test_file_info_versions_find_version_index() { diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 2ed643ead..da4abbd39 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() { cluster::rpc::shutdown_background_monitors(); } +/// Publish that the process is ready to serve user-object GET traffic. +/// +/// Experimental metadata coalescing is allowed to run only after this point so +/// startup and internal metadata reads keep the original per-disk path. +pub fn mark_get_metadata_read_version_coalescing_service_ready() { + runtime::global::mark_get_metadata_read_version_coalescing_service_ready(); +} + #[cfg(test)] mod rio_tests { #[test] diff --git a/crates/ecstore/src/runtime/global.rs b/crates/ecstore/src/runtime/global.rs index fcc4411f0..56ba22a0a 100644 --- a/crates/ecstore/src/runtime/global.rs +++ b/crates/ecstore/src/runtime/global.rs @@ -25,7 +25,10 @@ use lazy_static::lazy_static; use rustfs_lock::client::LockClient; use std::{ collections::HashMap, - sync::{Arc, OnceLock}, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, Ordering}, + }, time::SystemTime, }; use tokio::sync::{OnceCell, RwLock}; @@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; +static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false); + +pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() { + GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release); +} + +pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool { + GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire) +} + // Global singletons for backward compatibility with MinIO port. // These should be migrated to AppContext over time. // See issue #730 for migration plan. diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index d17c1c751..1a77ba267 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -53,11 +53,12 @@ use crate::diagnostics::get::{ GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; -use crate::disk::disk_store::DiskStoreRenameDataExt; +use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout}; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::disk::{ - DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, - PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, + BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk, + OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, }; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; @@ -75,7 +76,7 @@ use std::{ future::Future, pin::Pin, sync::{ - OnceLock, + Arc, OnceLock, atomic::{AtomicUsize, Ordering}, }, task::{Context, Poll}, @@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String { [bucket, object].join("/") } +fn read_version_coalescing_enabled() -> bool { + let enabled = || { + rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) + .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on")) + }; + + #[cfg(test)] + { + enabled() + } + + #[cfg(not(test))] + { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(enabled) + } +} + +fn read_version_coalescing_delay() -> Duration { + #[cfg(test)] + { + let micros = rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + ); + Duration::from_micros(micros) + } + + #[cfg(not(test))] + { + static DELAY: OnceLock = OnceLock::new(); + *DELAY.get_or_init(|| { + Duration::from_micros(rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + )) + }) + } +} + +struct CoalescedReadVersionRequest { + item: BatchReadVersionItem, + tx: oneshot::Sender>, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct ReadVersionCoalescerKey { + disk: usize, + incl_free_versions: bool, + read_data: bool, + healing: bool, +} + +impl ReadVersionCoalescerKey { + fn new(disk: &DiskStore, opts: &ReadOptions) -> Self { + Self { + disk: Arc::as_ptr(disk) as usize, + incl_free_versions: opts.incl_free_versions, + read_data: opts.read_data, + healing: opts.healing, + } + } +} + +#[derive(Default)] +struct ReadVersionCoalescer { + lanes: HashMap>, +} + +fn read_version_coalescer() -> &'static Mutex { + static COALESCER: OnceLock> = OnceLock::new(); + COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default())) +} + +fn record_read_version_coalescer_event(event: &'static str, item_count: usize) { + counter!( + METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL, + "event" => event, + "item_count" => item_count.to_string() + ) + .increment(1); +} + +async fn read_version_via_coalescer( + disk: DiskStore, + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + opts: &ReadOptions, + allow_coalescing: bool, +) -> disk::error::Result { + if !allow_coalescing || !read_version_coalescing_enabled() { + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + if !matches!(disk.as_ref(), Disk::Remote(_)) { + record_read_version_coalescer_event("bypass_non_remote", 1); + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + + let (tx, rx) = oneshot::channel(); + let item = BatchReadVersionItem { + org_volume: org_bucket.to_string(), + volume: bucket.to_string(), + path: object.to_string(), + version_id: version_id.to_string(), + }; + let lane_key = ReadVersionCoalescerKey::new(&disk, opts); + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + let lane = coalescer.lanes.entry(lane_key).or_default(); + let schedule_delayed_flush = lane.is_empty(); + lane.push(CoalescedReadVersionRequest { item, tx }); + if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS { + coalescer.lanes.remove(&lane_key) + } else if schedule_delayed_flush { + let disk = disk.clone(); + let task_opts = *opts; + tokio::spawn(async move { + tokio::time::sleep(read_version_coalescing_delay()).await; + flush_read_version_coalescer_lane(lane_key, disk, task_opts).await; + }); + None + } else { + None + } + }; + + if let Some(pending) = pending { + flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await; + } + + rx.await + .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))) +} + +async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) { + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + coalescer.lanes.remove(&lane_key).unwrap_or_default() + }; + flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await; +} + +async fn flush_read_version_coalescer_pending( + lane_key: ReadVersionCoalescerKey, + disk: DiskStore, + opts: ReadOptions, + pending: Vec, +) { + if pending.is_empty() { + return; + } + + #[cfg(test)] + { + let mut observed_paths = HashSet::new(); + for request in &pending { + if observed_paths.insert(request.item.path.as_str()) { + disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk); + } + } + } + + let mut senders = Vec::with_capacity(pending.len()); + let mut items = Vec::with_capacity(pending.len()); + for request in pending { + senders.push(request.tx); + items.push(request.item); + } + + let expected_items = items.clone(); + record_read_version_coalescer_event("attempted_batch", items.len()); + let result = + match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts })) + .await + { + Ok(result) => result, + Err(_) => Err(DiskError::Timeout), + }; + match result { + Ok(responses) => { + let results = map_batch_read_version_responses(&expected_items, responses); + for (tx, result) in senders.into_iter().zip(results) { + let _ = tx.send(result); + } + } + Err(err) => { + let message = err.to_string(); + for tx in senders { + let _ = tx.send(Err(DiskError::other(message.clone()))); + } + } + } +} + +fn map_batch_read_version_responses( + expected_items: &[BatchReadVersionItem], + responses: Vec, +) -> Vec> { + let mut results = (0..expected_items.len()) + .map(|_| Err(DiskError::other("coalesced read_version response missing"))) + .collect::>(); + let mut seen = vec![false; expected_items.len()]; + for response in responses { + let Some(expected) = expected_items.get(response.index) else { + continue; + }; + let Some(slot) = results.get_mut(response.index) else { + continue; + }; + if seen[response.index] { + *slot = Err(DiskError::other("coalesced read_version response duplicate index")); + continue; + } + seen[response.index] = true; + if response.path != expected.path || response.version_id != expected.version_id { + *slot = Err(DiskError::other("coalesced read_version response identity mismatch")); + } else { + *slot = if response.success { + Ok(response.file_info) + } else { + Err(batch_read_version_response_error(response.error_code, response.error)) + }; + } + } + results +} + +fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError { + match DiskError::from_u32(error_code) { + Some(DiskError::Io(_)) | None => DiskError::other(error), + Some(error) => error, + } +} + pub(in crate::set_disk) fn bounded_metadata_fanout_order( bucket: &str, object: &str, @@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order( order } use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock, oneshot}; use tokio::task::JoinSet; pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS"; +const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200; +const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total"; pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"; /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215, /// #1159, #923). @@ -2356,6 +2597,7 @@ impl SetDisks { false, true, 0, + false, ) .await?; Ok((ress, errors)) @@ -2386,6 +2628,36 @@ impl SetDisks { true, caller_allows_early_stop, default_parity_count, + false, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + incl_free_versions: bool, + caller_allows_early_stop: bool, + default_parity_count: usize, + ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { + Self::read_all_fileinfo_inner( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + false, + incl_free_versions, + true, + caller_allows_early_stop, + default_parity_count, + true, ) .await } @@ -2408,6 +2680,7 @@ impl SetDisks { // subset would fail write quorum (backlog#872 regression). caller_allows_early_stop: bool, default_parity_count: usize, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let early_stop_enabled = caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled()); @@ -2424,6 +2697,7 @@ impl SetDisks { healing, incl_free_versions, default_parity_count, + allow_coalescing, ) .await; } @@ -2446,6 +2720,7 @@ impl SetDisks { healing, incl_free_versions, observe, + allow_coalescing, ) .await } @@ -2461,6 +2736,7 @@ impl SetDisks { healing: bool, incl_free_versions: bool, observe: bool, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let fanout_start = observe.then(Instant::now); let mut ress = Vec::with_capacity(disks.len()); @@ -2492,7 +2768,7 @@ impl SetDisks { if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) { tokio::time::sleep(delay).await; } - disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) .await } else { Err(DiskError::DiskNotFound) @@ -2559,6 +2835,7 @@ impl SetDisks { healing: bool, incl_free_versions: bool, default_parity_count: usize, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let fanout_start = Instant::now(); let mut ress = vec![FileInfo::default(); disks.len()]; @@ -2607,7 +2884,7 @@ impl SetDisks { if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) { tokio::time::sleep(delay).await; } - disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) .await } else { Err(DiskError::DiskNotFound) @@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters { /// Kind label for the per-disk `read_version` metadata RPC. pub const KIND_READ_VERSION: &str = "read_version"; + pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version"; /// Registry key: (object, kind, disk_index). type CountKey = (String, String, usize); @@ -6460,6 +6738,286 @@ mod tests { drop(dirs); } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn metadata_read_version_coalescer_bypasses_local_disks() { + const DISKS: usize = 4; + let bucket = "coalesced-read-version-local-bypass-bucket"; + let object_a = "coalesced-local-object-a"; + let object_b = "coalesced-local-object-b"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await; + install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await; + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")), + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")), + ], + async { + let calls = disk_call_counters::observe(object_a); + let disks_a = disks.clone(); + let disks_b = disks.clone(); + let read_a = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed_for_get_object( + &disks_a, "", bucket, object_a, "", false, false, false, 2, + ) + .await + .map(|(file_infos, errors, _)| (file_infos, errors)) + }); + tokio::task::yield_now().await; + let read_b = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed_for_get_object( + &disks_b, "", bucket, object_b, "", false, false, false, 2, + ) + .await + .map(|(file_infos, errors, _)| (file_infos, errors)) + }); + + let (metadata_a, errs_a) = read_a + .await + .expect("first read task should not panic") + .expect("first coalesced read should resolve"); + let (metadata_b, errs_b) = read_b + .await + .expect("second read task should not panic") + .expect("second coalesced read should resolve"); + + assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS); + assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS); + assert!(errs_a.iter().all(Option::is_none)); + assert!(errs_b.iter().all(Option::is_none)); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "local disks still execute the ordinary per-disk read_version path" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_BATCH_READ_VERSION), + 0, + "GET coalescing targets internode RPC count only and must not batch local disk reads" + ); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn metadata_read_version_coalescer_requires_get_object_intent() { + const DISKS: usize = 4; + let bucket = "coalesced-read-version-default-bypass-bucket"; + let object = "default-bypass-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object, None).await; + + temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async { + let calls = disk_call_counters::observe(object); + let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false) + .await + .expect("default metadata read should resolve"); + + assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64); + assert_eq!( + calls.total(disk_call_counters::KIND_BATCH_READ_VERSION), + 0, + "non-GET metadata paths must bypass coalescer even when the env gate is enabled" + ); + }) + .await; + + drop(dirs); + } + + #[test] + fn batch_read_version_response_mapping_preserves_index_and_errors() { + let expected_items = vec![ + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-b".to_string(), + version_id: "v-b".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-c".to_string(), + version_id: "v-c".to_string(), + }, + ]; + let ok_file_info = FileInfo { + name: "object-a".to_string(), + ..Default::default() + }; + let responses = vec![ + BatchReadVersionResp { + index: 2, + path: "object-c".to_string(), + version_id: "v-c".to_string(), + success: false, + file_info: FileInfo::default(), + error: "disk read failed".to_string(), + error_code: 0, + }, + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: ok_file_info, + error: String::new(), + error_code: 0, + }, + ]; + + let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter(); + let first = results + .next() + .expect("slot 0 should exist") + .expect("slot 0 should map the success response by index"); + assert_eq!(first.name, "object-a"); + + let missing = results + .next() + .expect("slot 1 should exist") + .expect_err("slot 1 should stay missing"); + assert!( + missing.to_string().contains("response missing"), + "unexpected missing response error: {missing}" + ); + + let failed = results + .next() + .expect("slot 2 should exist") + .expect_err("slot 2 should map the response error"); + assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}"); + assert!(results.next().is_none()); + } + + #[test] + fn batch_read_version_response_mapping_preserves_typed_not_found_errors() { + let expected_items = vec![ + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-b".to_string(), + version_id: "v-b".to_string(), + }, + ]; + let results = map_batch_read_version_responses( + &expected_items, + vec![ + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: false, + file_info: FileInfo::default(), + error: DiskError::FileNotFound.to_string(), + error_code: DiskError::FileNotFound.to_u32(), + }, + BatchReadVersionResp { + index: 1, + path: "object-b".to_string(), + version_id: "v-b".to_string(), + success: false, + file_info: FileInfo::default(), + error: DiskError::FileVersionNotFound.to_string(), + error_code: DiskError::FileVersionNotFound.to_u32(), + }, + ], + ); + + assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound))); + assert!(matches!( + results.get(1).expect("slot 1 should exist"), + Err(DiskError::FileVersionNotFound) + )); + } + + #[test] + fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() { + let expected_items = vec![BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }]; + let mismatched = map_batch_read_version_responses( + &expected_items, + vec![BatchReadVersionResp { + index: 0, + path: "object-b".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-b".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }], + ) + .pop() + .expect("slot 0 should exist") + .expect_err("identity mismatch should fail closed"); + assert!( + mismatched.to_string().contains("identity mismatch"), + "unexpected mismatch error: {mismatched}" + ); + + let duplicate = map_batch_read_version_responses( + &expected_items, + vec![ + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-a".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }, + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-a".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }, + ], + ) + .pop() + .expect("slot 0 should exist") + .expect_err("duplicate response index should fail closed"); + assert!( + duplicate.to_string().contains("duplicate index"), + "unexpected duplicate error: {duplicate}" + ); + } + /// Isolation guard: unobserved objects record nothing (so parallel tests do /// not inflate one another), and a scope clears its own counts on drop. #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 0d429e3b3..8f07a7ff7 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -1294,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { (prepared.snapshot, prepared.object_info) } else { match self - .get_object_fileinfo( + .get_object_fileinfo_for_get_object_reader( bucket, object, opts, diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 2b556b143..15e7e4e2a 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -259,10 +259,33 @@ impl SetDisks { read_data: bool, caller_allows_early_stop: bool, ) -> Result { - self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop) + self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false) .await } + #[tracing::instrument(level = "debug", skip(self))] + #[hotpath::measure(impl_type = "SetDisks")] + pub(super) async fn get_object_fileinfo_for_get_object_reader( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + read_data: bool, + caller_allows_early_stop: bool, + ) -> Result { + let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket) + && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); + self.get_object_fileinfo_gated_inner( + bucket, + object, + opts, + read_data, + caller_allows_early_stop, + allow_read_version_coalescing, + ) + .await + } + /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full /// quorum fanout. Read-before-write callers (object tagging) must use this: /// the returned online-disk set is the write target, and the early-stop @@ -275,6 +298,20 @@ impl SetDisks { opts: &ObjectOptions, read_data: bool, allow_early_stop: bool, + ) -> Result { + self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn get_object_fileinfo_gated_inner( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + read_data: bool, + allow_early_stop: bool, + allow_read_version_coalescing: bool, ) -> Result { let vid = opts.version_id.clone().unwrap_or_default(); let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); @@ -337,19 +374,34 @@ impl SetDisks { // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in // core/io_primitives.rs); unsafe requests and callers that opt out // (allow_early_stop=false) fall back to full-wait. - let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed( - &disks, - "", - bucket, - object, - vid.as_str(), - read_data, - false, - opts.incl_free_versions, - allow_early_stop, - self.default_parity_count, - ) - .await?; + let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing { + Self::read_all_fileinfo_observed_for_get_object( + &disks, + "", + bucket, + object, + vid.as_str(), + read_data, + opts.incl_free_versions, + allow_early_stop, + self.default_parity_count, + ) + .await? + } else { + Self::read_all_fileinfo_observed( + &disks, + "", + bucket, + object, + vid.as_str(), + read_data, + false, + opts.incl_free_versions, + allow_early_stop, + self.default_parity_count, + ) + .await? + }; let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { GET_OBJECT_PATH_INTERNAL_META } else { diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 65f93c2a1..6b1ea382c 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode"; pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip"; pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index d3954b7a7..8a934e06e 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -20,6 +20,7 @@ use crate::storage_api::server::readiness::contract::admin::StorageAdminApi; use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure}; #[cfg(test)] use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints}; +use crate::storage_api::startup::shutdown::mark_get_metadata_read_version_coalescing_service_ready; use bytes::Bytes; use http::HeaderValue; use http::{Request as HttpRequest, Response, StatusCode}; @@ -212,6 +213,9 @@ where if readiness_gate_blocks_path(path, &readiness) { return Ok(service_not_ready_response(readiness.current_stage())); } + if !is_probe_path(path) && readiness.is_ready() { + mark_get_metadata_read_version_coalescing_service_ready(); + } let resp = inner.call(req).await?; // System is ready, forward to the actual S3/RPC handlers // Transparently converts any response body into a BoxBody, and then Trace/Cors/Compression continues to work @@ -232,6 +236,7 @@ pub async fn publish_ready_when_runtime_ready( collect_node_readiness, |dependency_readiness| { readiness.mark_stage(rustfs_common::SystemStage::FullReady); + mark_get_metadata_read_version_coalescing_service_ready(); if let Some(state_manager) = state_manager { state_manager.update(ServiceState::Ready); } diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index de80a10f0..617d0b80d 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -23,10 +23,12 @@ use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, - INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, - INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, - INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, + INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE, + INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_STAGE_READ_VERSION_DISK_READ, + INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, }; use rustfs_protos::proto_gen::node_service::*; use serde::de::DeserializeOwned; @@ -242,24 +244,42 @@ fn record_read_version_stage(stage: &'static str, started_at: Option) { } } +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + started_at.elapsed(), + ); + } +} + fn encode_batch_read_version_response_payloads( batch_read_version_resps: &[BatchReadVersionResp], request_decoded_from_msgpack: bool, ) -> std::result::Result<(Vec, Vec), DiskError> { + let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let mut batch_read_version_resps_json = Vec::with_capacity(batch_read_version_resps.len()); - let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len()); - + let json_encode_started = internode_stage_timer(attribution_enabled); for batch_read_version_resp in batch_read_version_resps { batch_read_version_resps_json.push( compat_response_json(batch_read_version_resp, request_decoded_from_msgpack) .map_err(|err| DiskError::other(format!("encode BatchReadVersionResp json failed: {err}")))?, ); + } + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE, json_encode_started); + + let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len()); + let msgpack_encode_started = internode_stage_timer(attribution_enabled); + for batch_read_version_resp in batch_read_version_resps { batch_read_version_resps_bin.push(Bytes::from(encode_msgpack_with_capacity( batch_read_version_resp, "BatchReadVersionResp", FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT, )?)); } + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, msgpack_encode_started); Ok((batch_read_version_resps_json, batch_read_version_resps_bin)) } @@ -485,15 +505,37 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { + let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let request = request.into_inner(); + if attribution_enabled { + let metrics = global_internode_metrics(); + metrics.record_incoming_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + request + .disk + .len() + .saturating_add(request.batch_read_version_req.len()) + .saturating_add(request.batch_read_version_req_bin.len()), + ); + } if let Some(disk) = self.find_disk(&request.disk).await { + let decode_started = internode_stage_timer(attribution_enabled); let decoded_batch_read_version_req: DecodedRpcPayload = match decode_msgpack_or_json_with_source( &request.batch_read_version_req_bin, &request.batch_read_version_req, "BatchReadVersionReq", ) { - Ok(batch_read_version_req) => batch_read_version_req, + Ok(batch_read_version_req) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started); + batch_read_version_req + } Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started); return Ok(Response::new(BatchReadVersionResponse { success: false, batch_read_version_resps: Vec::new(), @@ -514,8 +556,10 @@ impl NodeService { })); } + let disk_read_started = internode_stage_timer(attribution_enabled); match disk.batch_read_version(batch_read_version_req).await { Ok(batch_read_version_resps) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started); let (batch_read_version_resps, batch_read_version_resps_bin) = match encode_batch_read_version_response_payloads(&batch_read_version_resps, request_decoded_from_msgpack) { @@ -537,12 +581,15 @@ impl NodeService { error: None, })) } - Err(err) => Ok(Response::new(BatchReadVersionResponse { - success: false, - batch_read_version_resps: Vec::new(), - batch_read_version_resps_bin: Vec::new(), - error: Some(err.into()), - })), + Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started); + Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(err.into()), + })) + } } } else { Ok(Response::new(BatchReadVersionResponse { @@ -722,9 +769,9 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { - let request = request.into_inner(); let metrics = global_internode_metrics(); let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let request = request.into_inner(); if read_version_attribution_enabled { metrics.record_incoming_request_for_operation_and_backend( INTERNODE_OPERATION_GRPC_READ_VERSION, @@ -1635,6 +1682,7 @@ mod tests { encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; + use crate::storage::DiskError; use crate::storage::rpc::node_service::make_server; use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::RenameDataResp; @@ -2028,7 +2076,8 @@ mod tests { path: "object-a".to_string(), version_id: "version-a".to_string(), success: false, - error: "file version not found".to_string(), + error: DiskError::FileVersionNotFound.to_string(), + error_code: DiskError::FileVersionNotFound.to_u32(), ..Default::default() }]; @@ -2044,8 +2093,43 @@ mod tests { .expect("msgpack batch read version response should decode"); assert_eq!(json_decoded.index, responses[0].index); + assert_eq!(json_decoded.error_code, responses[0].error_code); assert_eq!(msgpack_decoded.path, responses[0].path); assert_eq!(msgpack_decoded.error, responses[0].error); + assert_eq!(msgpack_decoded.error_code, responses[0].error_code); + } + + #[test] + fn batch_read_version_response_decode_accepts_legacy_payload_without_error_code() { + #[derive(Serialize)] + struct LegacyBatchReadVersionResp { + index: usize, + path: String, + version_id: String, + success: bool, + file_info: FileInfo, + error: String, + } + + let legacy = LegacyBatchReadVersionResp { + index: 2, + path: "object-legacy".to_string(), + version_id: "version-legacy".to_string(), + success: false, + file_info: FileInfo::default(), + error: "legacy error".to_string(), + }; + let legacy_json = serde_json::to_string(&legacy).expect("legacy json should encode"); + let legacy_msgpack = encode_msgpack(&legacy, "LegacyBatchReadVersionResp").expect("legacy msgpack should encode"); + + let json_decoded: BatchReadVersionResp = + decode_msgpack_or_json(&[], &legacy_json, "BatchReadVersionResp").expect("legacy json should decode"); + let msgpack_decoded: BatchReadVersionResp = + decode_msgpack_or_json(&legacy_msgpack, "", "BatchReadVersionResp").expect("legacy msgpack should decode"); + + assert_eq!(json_decoded.error_code, 0); + assert_eq!(msgpack_decoded.error_code, 0); + assert_eq!(msgpack_decoded.error, legacy.error); } #[test] diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e5277c0dd..512e255ee 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1100,6 +1100,10 @@ pub(crate) fn shutdown_background_monitors() { rustfs_ecstore::shutdown_background_monitors(); } +pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() { + rustfs_ecstore::mark_get_metadata_read_version_coalescing_service_ready(); +} + pub(crate) fn set_global_rustfs_port(value: u16) { ecstore_global::set_global_rustfs_port(value); } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index e90c1f528..dea2923ac 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -284,7 +284,8 @@ pub(crate) mod startup { pub(crate) mod shutdown { pub(crate) use crate::storage::storage_api::{ - shutdown_background_monitors, shutdown_background_services, store_compression_total_in_backend, + mark_get_metadata_read_version_coalescing_service_ready, shutdown_background_monitors, shutdown_background_services, + store_compression_total_in_backend, }; } From 62c465eceff14a83988d2998558275f77595b4c6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 01:39:06 +0800 Subject: [PATCH 16/32] fix(heal): supervise scheduler task panics (#6351) From 87235ffd285d859cf7403d6d94d34998e1cac7c0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 01:40:03 +0800 Subject: [PATCH 17/32] perf(ecstore): bound decommission entry workers (#6360) --- crates/ecstore/src/core/pools.rs | 1270 +++++++++++++++++++----- crates/ecstore/src/runtime/instance.rs | 9 + 2 files changed, 1019 insertions(+), 260 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 90ad40464..9d8e0d01a 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -75,7 +75,7 @@ use std::sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, }; use time::{Duration, OffsetDateTime}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -93,6 +93,10 @@ const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000; const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1); const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY"; const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4; +const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY"; +const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; +const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; +const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); @@ -356,6 +360,19 @@ fn decommission_bucket_concurrency_limit() -> usize { rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1) } +fn default_decommission_entry_concurrency(cpu_count: usize) -> usize { + cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP) +} + +fn clamp_decommission_entry_concurrency(limit: usize) -> usize { + limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP) +} + +fn decommission_entry_concurrency_limit() -> usize { + let default_limit = default_decommission_entry_concurrency(num_cpus::get()); + clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit)) +} + fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool { bucket.name == RUSTFS_META_BUCKET } @@ -521,6 +538,7 @@ fn spawn_decommission_index_cancelers( store: Arc, rx: CancellationToken, index_cancelers: Vec<(usize, DecommissionCancelerGuard)>, + entry_budget: Arc, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut stop_queue = false; @@ -536,7 +554,8 @@ fn spawn_decommission_index_cancelers( let worker = tokio::spawn({ let store = store.clone(); let canceler = canceler.clone(); - async move { store.do_decommission_in_routine(canceler, idx).await } + let entry_budget = entry_budget.clone(); + async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await } }); if let Err(err) = await_decommission_worker(idx, worker).await { error!( @@ -770,6 +789,47 @@ fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: Ok(()) } +fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> { + let Some(pool) = meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx)); + }; + let Some(info) = pool.decommission.as_ref() else { + return Err(decommission_metadata_not_initialized_error("check decommission generation")); + }; + + if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) { + Ok(()) + } else { + Err(Error::OperationCanceled) + } +} + +async fn run_decommission_side_effect( + rx: &CancellationToken, + operation_gate: &Arc>, + operation: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + let _operation_guard = tokio::select! { + biased; + _ = rx.cancelled() => return Err(Error::OperationCanceled), + guard = operation_gate.read() => guard, + }; + + if rx.is_cancelled() { + return Err(Error::OperationCanceled); + } + + let result = operation().await; + if rx.is_cancelled() { + return Err(Error::OperationCanceled); + } + result +} + fn track_decommission_current_object_stage( meta: &mut PoolMeta, idx: usize, @@ -962,22 +1022,6 @@ fn resolve_decommission_partial_listing_entry( )) } -async fn record_decommission_entry_error( - entry_error: &Arc>>, - rx: &CancellationToken, - err: Error, -) { - if rx.is_cancelled() { - return; - } - - let mut first_err = entry_error.lock().await; - if first_err.is_none() && !rx.is_cancelled() { - *first_err = Some(err); - rx.cancel(); - } -} - fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> { result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}"))) } @@ -1110,6 +1154,7 @@ async fn wait_decommission_listing_retry(rx: &CancellationToken, delay: std::tim } } +#[cfg(test)] async fn run_decommission_listing_with_retry( rx: CancellationToken, bucket: String, @@ -1117,11 +1162,31 @@ async fn run_decommission_listing_with_retry( pool_idx: usize, set_idx: usize, max_attempts: usize, - mut list: List, + list: List, ) -> Result<()> where List: FnMut(ListCallback) -> ListFuture, ListFuture: std::future::Future>, +{ + run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await +} + +#[allow(clippy::too_many_arguments)] +async fn run_decommission_listing_with_retry_and_drain( + rx: CancellationToken, + bucket: String, + cb: ListCallback, + pool_idx: usize, + set_idx: usize, + max_attempts: usize, + mut list: List, + mut drain: Drain, +) -> Result<()> +where + List: FnMut(ListCallback) -> ListFuture, + ListFuture: std::future::Future>, + Drain: FnMut() -> DrainFuture, + DrainFuture: std::future::Future, { let max_attempts = max_attempts.max(1); @@ -1153,7 +1218,12 @@ where "Decommission listing started" ); - match list(cb.clone()).await { + let list_result = list(cb.clone()).await; + if drain().await { + return Ok(()); + } + + match list_result { Ok(()) => { debug!( event = EVENT_DECOMMISSION_BUCKET, @@ -1385,6 +1455,7 @@ where Ok(()) } +#[cfg(test)] async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> { let permits = u32::try_from(limit) .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?; @@ -2058,7 +2129,7 @@ impl PoolMeta { pub fn decommission_failed(&mut self, idx: usize) -> bool { if let Some(stats) = self.pools.get_mut(idx) { if let Some(d) = &stats.decommission { - if !d.failed { + if is_decommission_active(d.complete, d.failed, d.canceled) { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); @@ -2106,7 +2177,7 @@ impl PoolMeta { pub fn decommission_complete(&mut self, idx: usize) -> bool { if let Some(stats) = self.pools.get_mut(idx) { if let Some(d) = &stats.decommission { - if !d.complete { + if is_decommission_active(d.complete, d.failed, d.canceled) { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); @@ -2759,12 +2830,13 @@ impl ECStore { snapshot.save(self.pools.clone()).await } - async fn save_decommission_progress_checkpoint(&self, idx: usize) -> Result { + async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result { // Lock order: save gate, then the short pool metadata read/write sections. Peer // reloads are intentionally performed by the caller after both locks are released. let _save_guard = self.pool_meta_save_gate.lock().await; let (snapshot, checkpoint) = { let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; let Some(checkpoint) = pool_meta.decommission_progress_checkpoint( idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL, @@ -3078,6 +3150,8 @@ impl ECStore { ); } + self.wait_for_decommission_side_effects().await; + if should_save_pool_meta && let Err(err) = self.save_current_pool_meta().await { if let Some(previous_pool_meta) = previous_pool_meta { let mut pool_meta = self.pool_meta.write().await; @@ -3103,6 +3177,22 @@ impl ECStore { ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?; let _start_guard = self.start_gate.lock().await; + { + let pool_meta = self.pool_meta.read().await; + let pool_count = pool_meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + let Some(pool) = pool_meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let (decommission_present, complete, failed, canceled) = pool + .decommission + .as_ref() + .map(|info| (info.has_decommission_state(), info.complete, info.failed, info.canceled)) + .unwrap_or((false, false, false, false)); + ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; + } + self.cancel_decommission_routines_and_wait(&[idx]).await; + let (should_reload_pool_meta, previous_pool_meta) = { let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); @@ -3118,11 +3208,6 @@ impl ECStore { return Err(err); } - { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); - } - if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("clear_decommission for pool {idx}"); resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; @@ -3131,21 +3216,51 @@ impl ECStore { Ok(()) } - async fn promote_queued_decommission(&self, idx: usize) -> Result<()> { - let promoted = { + async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { + // Serialize promotion and generation capture with clear/restart transitions. + let (promoted, generation, save_error) = { + let _start_guard = self.start_gate.lock().await; let mut pool_meta = self.pool_meta.write().await; - pool_meta.promote_queued_decommission(idx) + if pool_meta.pools.get(idx).is_none() { + return Err(Error::other("failed to start decommission: target pool was not found")); + } + let promoted = pool_meta.promote_queued_decommission(idx); + drop(pool_meta); + + let save_error = if promoted { + self.save_current_pool_meta().await.err() + } else { + None + }; + + let generation = self.active_decommission_generation(idx).await?; + (promoted, generation, save_error) }; - if promoted { - self.save_current_pool_meta().await?; - if let Some(notification_sys) = runtime_sources::notification_sys() { - let stage = format!("promote_queued_decommission for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + if let Some(err) = save_error { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, owner).await, + idx, + &err, + )?; + return Err(err); + } + + if promoted && let Some(notification_sys) = runtime_sources::notification_sys() { + let stage = format!("promote_queued_decommission for pool {idx}"); + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, owner).await, + idx, + &err, + )?; + return Err(err); } } - Ok(()) + Ok(generation) } async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { @@ -3189,6 +3304,21 @@ impl ECStore { is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) } + async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) { + { + let mut cancelers = self.decommission_cancelers.write().await; + for idx in indices { + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx); + } + } + self.wait_for_decommission_side_effects().await; + } + + async fn wait_for_decommission_side_effects(&self) { + let operation_gate = self.ctx.decommission_operation_gate(); + let _operation_guard = operation_gate.write().await; + } + async fn reserve_decommission_routines( &self, rx: &CancellationToken, @@ -3233,7 +3363,12 @@ impl ECStore { ) -> Result<()> { let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; if !index_cancelers.is_empty() { - std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + store, + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); } Ok(()) @@ -3255,7 +3390,12 @@ impl ECStore { return Ok(()); } - std::mem::drop(spawn_decommission_index_cancelers(self.clone(), rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + self.clone(), + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); Ok(()) } @@ -3280,20 +3420,344 @@ impl ECStore { let index_cancelers = self .start_decommission_with_routines(indices, &rx, local_indices.as_slice()) .await?; - std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + store, + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); Ok(()) } + async fn active_decommission_generation(&self, idx: usize) -> Result { + let pool_meta = self.pool_meta.read().await; + let Some(pool) = pool_meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx)); + }; + let Some(info) = pool.decommission.as_ref() else { + return Err(decommission_metadata_not_initialized_error("load decommission generation")); + }; + let Some(generation) = info.start_time else { + return Err(Error::OperationCanceled); + }; + ensure_decommission_generation(&pool_meta, idx, generation)?; + Ok(generation) + } + + async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> { + let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation) + } + + #[allow(clippy::too_many_arguments)] + async fn decommission_entry_worker( + self: Arc, + rx: CancellationToken, + idx: usize, + set_idx: usize, + generation: OffsetDateTime, + bucket: String, + set: Arc, + lifecycle_config: Option, + object_lock_config: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + expected_bucket_incarnation_id: Option, + entry_budget: Arc, + queue: Arc>>, + entry_error: Arc>>, + ) { + loop { + let queued = tokio::select! { + biased; + _ = rx.cancelled() => return, + item = async { + let mut queue = queue.lock().await; + queue.recv().await + } => item, + }; + let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else { + return; + }; + let object_name = entry.name.clone(); + + if entry_error.lock().await.is_some() { + drop(queue_permit); + continue; + } + + if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await { + if matches!(err, Error::OperationCanceled) { + rx.cancel(); + } else { + record_decommission_entry_error(&entry_error, &rx, err).await; + } + return; + } + + if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await + { + if matches!(err, Error::OperationCanceled) { + return; + } + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_admission_failed", + error = %err, + "Decommission entry admission failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + + let entry_budget_permit = match tokio::select! { + biased; + _ = rx.cancelled() => return, + permit = entry_budget.clone().acquire_owned() => permit, + } { + Ok(permit) => permit, + Err(err) => { + let err = Error::other(format!("decommission entry budget permit acquire failed: {err}")); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_budget_acquire_failed", + error = %err, + "Decommission entry budget permit acquire failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + }; + + let result = self + .decommission_entry( + rx.clone(), + idx, + generation, + entry, + bucket.clone(), + set.clone(), + lifecycle_config.clone(), + object_lock_config.clone(), + replication_config.clone(), + expected_bucket_incarnation_id, + ) + .await; + drop(entry_budget_permit); + drop(queue_permit); + + if let Err(err) = result { + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_failed", + error = %err, + "Decommission entry failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn decommission_set( + self: Arc, + rx: CancellationToken, + idx: usize, + set_idx: usize, + generation: OffsetDateTime, + set: Arc, + bi: DecomBucketInfo, + lifecycle_config: Option, + object_lock_config: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + expected_bucket_incarnation_id: Option, + entry_budget: Arc, + entry_error: Arc>>, + ) -> Result<()> { + let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET; + let queue_capacity = decommission_entry_queue_capacity(worker_count); + let outstanding_capacity = queue_capacity.saturating_add(worker_count); + let outstanding = Arc::new(Semaphore::new(outstanding_capacity)); + let (tx, rx_queue) = mpsc::channel(queue_capacity); + let queue = Arc::new(tokio::sync::Mutex::new(rx_queue)); + + let mut entry_workers = tokio::task::JoinSet::new(); + for _ in 0..worker_count { + let this = self.clone(); + let rx = rx.clone(); + let bucket = bi.name.clone(); + let set = set.clone(); + let lifecycle_config = lifecycle_config.clone(); + let object_lock_config = object_lock_config.clone(); + let replication_config = replication_config.clone(); + let queue = queue.clone(); + let entry_budget = entry_budget.clone(); + let entry_error = entry_error.clone(); + entry_workers.spawn(async move { + this.decommission_entry_worker( + rx, + idx, + set_idx, + generation, + bucket, + set, + lifecycle_config, + object_lock_config, + replication_config, + expected_bucket_incarnation_id, + entry_budget, + queue, + entry_error, + ) + .await; + }); + } + + let callback: ListCallback = Arc::new({ + let tx = tx.clone(); + let outstanding = outstanding.clone(); + let callback_rx = rx.clone(); + let entry_error = entry_error.clone(); + let bucket = bi.name.clone(); + move |entry: MetaCacheEntry| { + let tx = tx.clone(); + let outstanding = outstanding.clone(); + let callback_rx = callback_rx.clone(); + let entry_error = entry_error.clone(); + let bucket = bucket.clone(); + Box::pin(async move { + if callback_rx.is_cancelled() || entry_error.lock().await.is_some() { + return; + } + + if matches!( + enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await, + DecommissionEntryEnqueueResult::Closed + ) { + let err = Error::other("decommission entry queue closed"); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + state = "entry_queue_closed", + error = %err, + "Decommission entry queue closed" + ); + record_decommission_entry_error(&entry_error, &callback_rx, err).await; + } + }) + } + }); + + let list_set = set.clone(); + let list_rx = rx.clone(); + let list_rx_for_list = list_rx.clone(); + let list_rx_for_drain = list_rx.clone(); + let list_bi = bi.clone(); + let list_outstanding = outstanding.clone(); + let list_entry_error = entry_error.clone(); + let mut listing = tokio::spawn(async move { + run_decommission_listing_with_retry_and_drain( + list_rx.clone(), + list_bi.name.clone(), + callback, + idx, + set_idx, + DECOMMISSION_LISTING_MAX_ATTEMPTS, + move |callback| { + let set = list_set.clone(); + let rx = list_rx_for_list.clone(); + let bucket = list_bi.clone(); + let entry_error = list_entry_error.clone(); + async move { + set.list_objects_to_decommission(rx, bucket, callback, entry_error, idx, set_idx) + .await + } + }, + move || { + let rx = list_rx_for_drain.clone(); + let outstanding = list_outstanding.clone(); + async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await } + }, + ) + .await + }); + + let mut listing_result = None; + let mut workers_left = worker_count; + let mut sender = Some(tx); + while listing_result.is_none() || workers_left > 0 { + tokio::select! { + biased; + result = &mut listing, if listing_result.is_none() => { + let result = resolve_decommission_listing_worker_result(set_idx, result); + if result.is_err() { + rx.cancel(); + } + listing_result = Some(result); + drop(sender.take()); + } + worker_result = entry_workers.join_next(), if workers_left > 0 => { + workers_left -= 1; + if let Some(Err(err)) = worker_result { + let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}")); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bi.name, + state = "entry_worker_join_failed", + error = %err, + "Decommission entry worker task failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + } + } + } + } + + let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete"))); + if let Some(err) = entry_error.lock().await.clone() { + return Err(err); + } + listing_result + } + async fn track_decommission_entry_progress_stage( &self, idx: usize, + generation: OffsetDateTime, bucket: &str, object: &str, stage: &'static str, ) -> Result<()> { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage) .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?; } @@ -3302,15 +3766,15 @@ impl ECStore { } #[allow(unused_assignments, clippy::too_many_arguments)] - #[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, object_lock_config, replication_config))] + #[tracing::instrument(skip(self, set, lifecycle_config, object_lock_config, replication_config))] async fn decommission_entry( self: &Arc, rx: CancellationToken, idx: usize, + generation: OffsetDateTime, entry: MetaCacheEntry, bucket: String, set: Arc, - _worker_permit: OwnedSemaphorePermit, lifecycle_config: Option, object_lock_config: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, @@ -3343,6 +3807,8 @@ impl ECStore { rx.cancel(); } decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; + let operation_gate = self.ctx.decommission_operation_gate(); let bucket_incarnation_fence = match expected_bucket_incarnation_id { Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), @@ -3364,15 +3830,18 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; - if should_skip_lifecycle_for_data_movement( - self.clone(), - &bucket, - version, - lifecycle_config.as_ref(), - object_lock_config.as_ref(), - true, - &LcEventSrc::Decom, - ) + if run_decommission_side_effect(&rx, &operation_gate, || async { + should_skip_lifecycle_for_data_movement( + self.clone(), + &bucket, + version, + lifecycle_config.as_ref(), + object_lock_config.as_ref(), + true, + &LcEventSrc::Decom, + ) + .await + }) .await .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))? { @@ -3405,13 +3874,15 @@ impl ECStore { let mut failure = false; let mut error = None; if version.deleted { - if let Err(err) = self - .delete_object( + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.delete_object( bucket.as_str(), &version.name, decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { warn!( @@ -3463,6 +3934,7 @@ impl ECStore { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) { return Err(with_decommission_entry_context( "count_decommission_item", @@ -3494,14 +3966,16 @@ impl ECStore { for _i in 0..3 { if version.is_remote() { - if let Err(err) = self - .decommission_tiered_object( + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.decommission_tiered_object( bucket.as_str(), &version.name, version, &decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; @@ -3565,16 +4039,19 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket_name.as_str(), object_name.as_str(), DECOMMISSION_STAGE_MIGRATE_OBJECT, ) .await?; - if let Err(err) = self - .clone() - .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) - .await + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.clone() + .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) + .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; @@ -3632,6 +4109,7 @@ impl ECStore { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) { return Err(with_decommission_entry_context( "count_decommission_item", @@ -3656,9 +4134,11 @@ impl ECStore { return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); } decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_CLEANUP_PREFLIGHT, @@ -3667,6 +4147,7 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_SOURCE_CLEANUP, @@ -3676,29 +4157,32 @@ impl ECStore { let source_cleanup_mutation_fence = self .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref()) .await?; - let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( - set.clone(), - bucket.as_str(), - entry.name.as_str(), - &fivs, - &cleanup_preflight_allowed_missing, - data_movement::SourceCleanupBucketFence { - expected_incarnation_id: expected_bucket_incarnation_id, - lifecycle_guard: bucket_incarnation_fence - .as_ref() - .and_then(|guard| guard.namespace_lock_guard()), - object_mutation_fence: Some(&source_cleanup_mutation_fence), - }, - "decommission", - ) - .await - .map_err(|err| match err { - data_movement::SourceCleanupError::SourceChanged => Error::other(format!( - "decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started", - bucket, entry.name - )), - data_movement::SourceCleanupError::Storage(err) => err, - }); + let cleanup_result = run_decommission_side_effect(&rx, &operation_gate, || async { + data_movement::cleanup_source_entry_if_unchanged( + set.clone(), + bucket.as_str(), + entry.name.as_str(), + &fivs, + &cleanup_preflight_allowed_missing, + data_movement::SourceCleanupBucketFence { + expected_incarnation_id: expected_bucket_incarnation_id, + lifecycle_guard: bucket_incarnation_fence + .as_ref() + .and_then(|guard| guard.namespace_lock_guard()), + object_mutation_fence: Some(&source_cleanup_mutation_fence), + }, + "decommission", + ) + .await + .map_err(|err| match err { + data_movement::SourceCleanupError::SourceChanged => Error::other(format!( + "decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started", + bucket, entry.name + )), + data_movement::SourceCleanupError::Storage(err) => err, + }) + }) + .await; resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? } else if decommissioned != fivs.versions.len() || expired > 0 { warn!( @@ -3718,6 +4202,7 @@ impl ECStore { let should_save_progress = { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) { return Err(with_decommission_entry_context( @@ -3738,6 +4223,7 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_ENTRY_FINISHED, @@ -3745,7 +4231,7 @@ impl ECStore { .await?; if should_save_progress { - match self.save_decommission_progress_checkpoint(idx).await { + match self.save_decommission_progress_checkpoint(idx, generation).await { Ok(true) => { if let Some(notification_sys) = runtime_sources::notification_sys() && let Err(err) = resolve_decommission_entry_reload_result( @@ -3797,12 +4283,19 @@ impl ECStore { bucket: String, set: Arc, ) -> Result<()> { - let worker_permit = Arc::new(Semaphore::new(1)) - .acquire_owned() - .await - .map_err(|err| Error::other(format!("decommission test worker permit acquire failed: {err}")))?; - self.decommission_entry(CancellationToken::new(), idx, entry, bucket, set, worker_permit, None, None, None, None) - .await + self.decommission_entry( + CancellationToken::new(), + idx, + OffsetDateTime::now_utc(), + entry, + bucket, + set, + None, + None, + None, + None, + ) + .await } #[tracing::instrument(skip(self, rx))] @@ -3812,13 +4305,10 @@ impl ECStore { idx: usize, pool: Arc, bi: DecomBucketInfo, + entry_budget: Arc, ) -> Result<()> { - let worker_limit = pool.disk_set.len() * 2; - if worker_limit == 0 { - return Err(Error::other("decommission worker limit must be greater than zero")); - } - let workers = Arc::new(Semaphore::new(worker_limit)); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let generation = self.active_decommission_generation(idx).await?; let mut listing_workers = Vec::with_capacity(pool.disk_set.len()); let mut lifecycle_config = None; @@ -3847,12 +4337,6 @@ impl ECStore { } for (set_idx, set) in pool.disk_set.iter().enumerate() { - let listing_permit = workers - .clone() - .acquire_owned() - .await - .map_err(|err| Error::other(format!("decommission listing worker permit acquire failed: {err}")))?; - debug!( event = EVENT_DECOMMISSION_BUCKET, component = LOG_COMPONENT_ECSTORE, @@ -3864,130 +4348,34 @@ impl ECStore { "Decommission listing worker started" ); - let decommission_entry: ListCallback = Arc::new({ - let this = Arc::clone(self); - let bucket = bi.name.clone(); - let workers = workers.clone(); - let set = set.clone(); - let lifecycle_config = lifecycle_config.clone(); - let object_lock_config = object_lock_config.clone(); - let replication_config = replication_config.clone(); - let entry_error = entry_error.clone(); - let callback_rx = rx.clone(); - move |entry: MetaCacheEntry| { - let this = this.clone(); - let bucket = bucket.clone(); - let workers = workers.clone(); - let set = set.clone(); - let lifecycle_config = lifecycle_config.clone(); - let object_lock_config = object_lock_config.clone(); - let replication_config = replication_config.clone(); - let expected_bucket_incarnation_id = expected_bucket_incarnation_id; - let entry_error = entry_error.clone(); - let callback_rx = callback_rx.clone(); - - Box::pin(async move { - if callback_rx.is_cancelled() { - return; - } - if entry_error.lock().await.is_some() { - return; - } - - if let Err(err) = - backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &callback_rx) - .await - { - if matches!(err, Error::OperationCanceled) { - return; - } - error!("decommission_pool: data movement admission failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - return; - } - - if entry_error.lock().await.is_some() { - return; - } - - let worker_permit = match tokio::select! { - _ = callback_rx.cancelled() => return, - permit = workers.clone().acquire_owned() => permit, - } { - Ok(permit) => permit, - Err(err) => { - let err = Error::other(format!("decommission entry worker permit acquire failed: {err}")); - error!("decommission_pool: decommission_entry failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - return; - } - }; - if entry_error.lock().await.is_some() { - return; - } - let entry_rx = callback_rx.clone(); - if let Err(err) = this - .decommission_entry( - entry_rx, - idx, - entry, - bucket, - set, - worker_permit, - lifecycle_config, - object_lock_config, - replication_config, - expected_bucket_incarnation_id, - ) - .await - { - error!("decommission_pool: decommission_entry failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - } - }) - } - }); - let set = set.clone(); + let store = Arc::clone(self); let rx_clone = rx.clone(); - let bi = bi.clone(); - let set_id = set_idx; - let listing_entry_error = entry_error.clone(); + let bi_clone = bi.clone(); + let lifecycle_config = lifecycle_config.clone(); + let object_lock_config = object_lock_config.clone(); + let replication_config = replication_config.clone(); + let entry_budget = entry_budget.clone(); + let entry_error = entry_error.clone(); let worker = tokio::spawn(async move { - let _listing_permit = listing_permit; - run_decommission_listing_with_retry( - rx_clone.clone(), - bi.name.clone(), - decommission_entry.clone(), - idx, - set_id, - DECOMMISSION_LISTING_MAX_ATTEMPTS, - |callback| { - let set = set.clone(); - let rx = rx_clone.clone(); - let bucket = bi.clone(); - let entry_error = listing_entry_error.clone(); - async move { - set.list_objects_to_decommission(rx, bucket, callback, entry_error.clone(), idx, set_id) - .await - } - }, - ) - .await + store + .decommission_set( + rx_clone, + idx, + set_idx, + generation, + set, + bi_clone, + lifecycle_config, + object_lock_config, + replication_config, + expected_bucket_incarnation_id, + entry_budget, + entry_error, + ) + .await }); - listing_workers.push((set_id, worker)); + listing_workers.push((set_idx, worker)); } debug!( @@ -4010,9 +4398,11 @@ impl ECStore { } } - wait_decommission_worker_drain(&workers, worker_limit).await?; + if let Some(err) = listing_worker_error { + return Err(err); + } - if let Some(err) = resolve_decommission_listing_error(listing_worker_error, entry_error.lock().await.clone()) { + if let Some(err) = entry_error.lock().await.clone() { return Err(err); } @@ -4044,9 +4434,14 @@ impl ECStore { } #[tracing::instrument(skip(self, canceler))] - pub async fn do_decommission_in_routine(self: &Arc, canceler: DecommissionCanceler, idx: usize) -> Result<()> { + pub async fn do_decommission_in_routine( + self: &Arc, + canceler: DecommissionCanceler, + idx: usize, + entry_budget: Arc, + ) -> Result<()> { let rx = canceler.token().clone(); - self.run_decommission_in_routine(rx, idx, &canceler).await + self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await } async fn run_decommission_in_routine( @@ -4054,15 +4449,20 @@ impl ECStore { rx: CancellationToken, idx: usize, canceler: &DecommissionCanceler, + entry_budget: Arc, ) -> Result<()> { - if let Err(err) = self.promote_queued_decommission(idx).await { - resolve_decommission_terminal_mark_after_error_result( - self.decommission_failed_for_operation(idx, canceler).await, - idx, - &err, - )?; - return Err(err); - } + let generation = match self.promote_queued_decommission(idx, canceler).await { + Ok(generation) => generation, + Err(Error::OperationCanceled) => return Ok(()), + Err(err) => { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, canceler).await, + idx, + &err, + )?; + return Err(err); + } + }; if rx.is_cancelled() { let already_canceled = { let pool_meta = self.pool_meta.read().await; @@ -4089,7 +4489,7 @@ impl ECStore { } return Ok(()); } - let result = self.decommission_in_background(rx.clone(), idx).await; + let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await; let (final_state, canceled, cmd_line) = { let pool_meta = self.pool_meta.read().await; @@ -4204,6 +4604,12 @@ impl ECStore { ))); } + if self.decommission_cancel_requested(idx, &rx).await { + rx.cancel(); + } + decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; + info!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -4440,6 +4846,7 @@ impl ECStore { idx: usize, pool: Arc, bucket: DecomBucketInfo, + entry_budget: Arc, ) -> Result<()> { let is_decommissioned = { let pool_meta = self.pool_meta.read().await; @@ -4465,7 +4872,10 @@ impl ECStore { warn!("decommission: currently on bucket {}", &bucket.name); - if let Err(err) = self.decommission_pool(rx.clone(), idx, pool, bucket.clone()).await { + if let Err(err) = self + .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget) + .await + { error!("decommission: decommission_pool err {:?}", &err); return Err(err); } else { @@ -4499,40 +4909,53 @@ impl ECStore { pool: Arc, buckets: Vec, limit: usize, + entry_budget: Arc, ) -> Result<()> { let store = Arc::clone(self); run_decommission_buckets_bounded(rx, buckets, limit, move |bucket, rx| { let store = Arc::clone(&store); let pool = pool.clone(); - Box::pin(async move { store.decommission_pending_bucket(rx, idx, pool, bucket).await }) + let entry_budget = entry_budget.clone(); + Box::pin(async move { store.decommission_pending_bucket(rx, idx, pool, bucket, entry_budget).await }) }) .await } #[tracing::instrument(skip(self, rx))] - async fn decommission_in_background(self: &Arc, rx: CancellationToken, idx: usize) -> Result<()> { + async fn decommission_in_background( + self: &Arc, + rx: CancellationToken, + idx: usize, + entry_budget: Arc, + ) -> Result<()> { let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone(); let pending = { let pool_meta = self.pool_meta.read().await; pool_meta.pending_buckets(idx) }; - let bucket_concurrency = decommission_bucket_concurrency_limit(); if bucket_concurrency <= 1 { for bucket in pending { - self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket) + self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket, entry_budget.clone()) .await?; } return Ok(()); } let (regular_buckets, meta_buckets) = split_decommission_buckets(pending); - self.decommission_buckets_concurrently(rx.clone(), idx, pool.clone(), regular_buckets, bucket_concurrency) - .await?; + self.decommission_buckets_concurrently( + rx.clone(), + idx, + pool.clone(), + regular_buckets, + bucket_concurrency, + entry_budget.clone(), + ) + .await?; for bucket in meta_buckets { - self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket) + self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket, entry_budget.clone()) .await?; } @@ -4600,6 +5023,8 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?; + self.cancel_decommission_routines_and_wait(&indices).await; + let index_cancelers = if let Some((rx, local_indices)) = reservation { // Lock order matches terminal transitions: decommission_cancelers // before pool_meta while start_gate excludes another start. @@ -5169,6 +5594,14 @@ mod tests { let mut pool_meta = build_pool_meta(); assert!(pool_meta.decommission_cancel(0)); assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None); + + let mut pool_meta = build_pool_meta(); + assert!(pool_meta.decommission_cancel(0)); + assert!(!pool_meta.decommission_complete(0)); + + let mut pool_meta = build_pool_meta(); + assert!(pool_meta.decommission_failed(0)); + assert!(!pool_meta.decommission_complete(0)); } #[test] @@ -5552,6 +5985,79 @@ mod tests { pub type ListCallback = Arc BoxFuture<'static, ()> + Send + Sync + 'static>; +const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256; + +struct QueuedDecommissionEntry { + entry: MetaCacheEntry, + queue_permit: OwnedSemaphorePermit, +} + +enum DecommissionEntryEnqueueResult { + Enqueued, + Canceled, + Closed, +} + +fn decommission_entry_queue_capacity(worker_limit: usize) -> usize { + worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP) +} + +async fn enqueue_decommission_entry( + rx: &CancellationToken, + outstanding: &Arc, + tx: &mpsc::Sender, + entry: MetaCacheEntry, +) -> DecommissionEntryEnqueueResult { + let queue_permit = match tokio::select! { + biased; + _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled, + permit = outstanding.clone().acquire_owned() => permit, + } { + Ok(permit) => permit, + Err(_) => return DecommissionEntryEnqueueResult::Closed, + }; + + let queued = QueuedDecommissionEntry { entry, queue_permit }; + tokio::select! { + biased; + _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled, + result = tx.send(queued) => { + if result.is_ok() { + DecommissionEntryEnqueueResult::Enqueued + } else { + DecommissionEntryEnqueueResult::Closed + } + } + } +} + +async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc, capacity: usize) -> bool { + let Ok(permits) = u32::try_from(capacity) else { + return true; + }; + + tokio::select! { + _ = rx.cancelled() => true, + result = outstanding.acquire_many(permits) => result.is_err(), + } +} + +async fn record_decommission_entry_error( + entry_error: &Arc>>, + rx: &CancellationToken, + err: Error, +) { + if rx.is_cancelled() { + return; + } + + let mut first_err = entry_error.lock().await; + if first_err.is_none() && !rx.is_cancelled() { + *first_err = Some(err); + rx.cancel(); + } +} + impl SetDisks { #[tracing::instrument(skip(self, rx, cb_func, entry_error))] async fn list_objects_to_decommission( @@ -5843,23 +6349,26 @@ mod pools_tests { use super::record_decommission_entry_error; use super::resolve_decommission_listing_error; use super::{ + DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, - DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, - PoolStatus, apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, - bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options, - decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, - ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available, - ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, - ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, - ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, - ensure_decommission_terminal_operation_supported, ensure_local_decommission_pool_leaders, - ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, get_by_index, - guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, - load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, - merge_pool_status_refresh, missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, - pool_meta_has_active_decommission, require_decommission_store, reserve_decommission_start_cancelers, - resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, + DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, + PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, apply_decommission_status_space_info, + await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, + cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, + count_decommission_item, decommission_cancel_signal_result, decommission_entry_queue_capacity, decommission_item_size, + decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, + default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, + ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation, + ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, + ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, + ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed, + ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, + ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, + get_by_index, guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, + is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, + mark_decommission_bucket_done, merge_pool_status_refresh, missing_decommission_worker_prefix, + observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, require_decommission_store, + reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, @@ -5868,7 +6377,8 @@ mod pools_tests { resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, - should_cleanup_decommission_source_entry, should_continue_decommission_queue, should_count_decommission_version_complete, + run_decommission_listing_with_retry_and_drain, run_decommission_side_effect, should_cleanup_decommission_source_entry, + should_continue_decommission_queue, should_count_decommission_version_complete, should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler, @@ -6130,6 +6640,25 @@ mod pools_tests { assert_eq!(default_decommission_bucket_concurrency(8), 4); } + #[test] + fn test_default_decommission_entry_concurrency_is_conservative() { + assert_eq!(default_decommission_entry_concurrency(0), 1); + assert_eq!(default_decommission_entry_concurrency(1), 1); + assert_eq!(default_decommission_entry_concurrency(4), 4); + assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP); + } + + #[test] + fn test_decommission_entry_concurrency_clamps_operator_configuration() { + assert_eq!(clamp_decommission_entry_concurrency(0), 1); + assert_eq!(clamp_decommission_entry_concurrency(1), 1); + assert_eq!( + clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP), + DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP + ); + assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP); + } + #[test] fn test_split_decommission_buckets_keeps_meta_buckets_last() { let (regular, meta) = split_decommission_buckets(vec![ @@ -6301,6 +6830,190 @@ mod pools_tests { assert!(result.is_ok()); } + #[test] + fn test_decommission_entry_queue_capacity_is_bounded() { + assert_eq!(decommission_entry_queue_capacity(0), 1); + assert_eq!(decommission_entry_queue_capacity(1), 2); + assert_eq!( + decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP), + DECOMMISSION_ENTRY_QUEUE_HARD_CAP + ); + assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP); + } + + #[tokio::test] + async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() { + let outstanding = Arc::new(Semaphore::new(1)); + let held = outstanding + .clone() + .acquire_owned() + .await + .expect("test outstanding permit should acquire"); + let rx = CancellationToken::new(); + let drain = tokio::spawn({ + let outstanding = outstanding.clone(); + let rx = rx.clone(); + async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await } + }); + + tokio::task::yield_now().await; + assert!(!drain.is_finished(), "queue drain must wait for active entry work"); + drop(held); + + let drained = tokio::time::timeout(StdDuration::from_secs(1), drain) + .await + .expect("queue drain should finish after entry completion") + .expect("queue drain task should not panic"); + assert!(!drained); + } + + #[tokio::test] + async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() { + let outstanding = Arc::new(Semaphore::new(2)); + let (tx, mut queue) = tokio::sync::mpsc::channel(1); + let held = outstanding + .clone() + .acquire_owned() + .await + .expect("first queue permit should acquire"); + tx.send(QueuedDecommissionEntry { + entry: MetaCacheEntry::default(), + queue_permit: held, + }) + .await + .expect("first entry should fill the queue"); + + let rx = CancellationToken::new(); + let enqueue = tokio::spawn({ + let rx = rx.clone(); + let outstanding = outstanding.clone(); + let tx = tx.clone(); + async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await } + }); + + tokio::task::yield_now().await; + rx.cancel(); + let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue) + .await + .expect("full queue enqueue should observe cancellation") + .expect("enqueue task should not panic"); + assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled)); + drop(queue.recv().await); + } + + #[tokio::test] + async fn test_decommission_side_effect_gate_quiesces_before_transition() { + let operation_gate = Arc::new(tokio::sync::RwLock::new(())); + let rx = CancellationToken::new(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let operation = tokio::spawn({ + let operation_gate = operation_gate.clone(); + let rx = rx.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + run_decommission_side_effect(&rx, &operation_gate, || async { + started.notify_one(); + release.notified().await; + Ok::<_, Error>(()) + }) + .await + } + }); + + started.notified().await; + rx.cancel(); + let transition = tokio::spawn({ + let operation_gate = operation_gate.clone(); + async move { + let _guard = operation_gate.write().await; + } + }); + tokio::task::yield_now().await; + assert!(!transition.is_finished(), "transition must wait for the in-flight side effect"); + + release.notify_one(); + let operation_result = operation.await.expect("operation task should not panic"); + assert!(matches!(operation_result, Err(Error::OperationCanceled))); + transition.await.expect("transition task should not panic"); + + let called = Arc::new(AtomicBool::new(false)); + let result = run_decommission_side_effect(&rx, &operation_gate, { + let called = called.clone(); + move || async move { + called.store(true, Ordering::SeqCst); + Ok::<_, Error>(()) + } + }) + .await; + assert!(matches!(result, Err(Error::OperationCanceled))); + assert!(!called.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_decommission_transition_waits_without_registered_canceler() { + let store = decommission_worker_test_store(PoolMeta::default(), vec![None]); + let operation_gate = store.ctx.decommission_operation_gate(); + let operation_guard = operation_gate.read().await; + let transition = tokio::spawn({ + let store = store.clone(); + async move { store.cancel_decommission_routines_and_wait(&[0]).await } + }); + + tokio::task::yield_now().await; + assert!( + !transition.is_finished(), + "a transition must wait for an in-flight side effect even after its canceler slot is gone" + ); + + drop(operation_guard); + tokio::time::timeout(StdDuration::from_secs(1), transition) + .await + .expect("transition should finish after the side effect") + .expect("transition task should not panic"); + } + + #[tokio::test(start_paused = true)] + async fn test_run_decommission_listing_with_retry_drains_before_each_retry() { + let attempts = Arc::new(AtomicUsize::new(0)); + let drains = Arc::new(AtomicUsize::new(0)); + let err = run_decommission_listing_with_retry_and_drain( + CancellationToken::new(), + "bucket-a".to_string(), + noop_decommission_list_callback(), + 1, + 2, + 2, + { + let attempts = attempts.clone(); + move |_| { + let attempts = attempts.clone(); + async move { + attempts.fetch_add(1, Ordering::SeqCst); + Err(Error::SlowDown) + } + } + }, + { + let drains = drains.clone(); + move || { + let drains = drains.clone(); + async move { + drains.fetch_add(1, Ordering::SeqCst); + false + } + } + }, + ) + .await + .expect_err("permanent listing failure must be returned"); + + assert!(err.to_string().contains("attempt 2/2")); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert_eq!(drains.load(Ordering::SeqCst), 2); + } + #[test] fn test_get_by_index_returns_value_when_in_range() { let values = vec!["a", "b", "c"]; @@ -7528,6 +8241,43 @@ mod pools_tests { assert!(!is_decommission_active(false, false, true)); } + #[test] + fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() { + let generation = OffsetDateTime::UNIX_EPOCH; + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: generation, + decommission: Some(PoolDecommissionInfo { + start_time: Some(generation), + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!(ensure_decommission_generation(&meta, 0, generation).is_ok()); + assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err()); + + meta.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist") + .queued = true; + assert!(ensure_decommission_generation(&meta, 0, generation).is_err()); + + let replacement_generation = generation + Duration::seconds(2); + let info = meta.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist"); + info.queued = false; + info.start_time = Some(replacement_generation); + assert!(ensure_decommission_generation(&meta, 0, generation).is_err()); + assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok()); + } + #[test] fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() { let active_meta = PoolMeta { @@ -8869,7 +9619,7 @@ mod pools_tests { canceler.cancel(); let err = store - .do_decommission_in_routine(canceler.clone(), 0) + .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1))) .await .expect_err("missing worker metadata should fail the routine"); @@ -8885,7 +9635,7 @@ mod pools_tests { let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]); let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]); - spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards) + spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))) .await .expect("decommission supervisor should finish after queued cleanup"); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 71a1898cf..9453ed02b 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -160,6 +160,10 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, + /// Serializes decommission data-movement operations with cancellation and + /// a subsequent restart. Readers are held across one object side effect; + /// the transition path takes the writer after cancelling the routine. + decommission_operation_gate: Arc>, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, @@ -200,6 +204,7 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), + decommission_operation_gate: Arc::new(RwLock::new(())), object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), @@ -218,6 +223,10 @@ impl InstanceContext { self.lock_manager.clone() } + pub(crate) fn decommission_operation_gate(&self) -> Arc> { + Arc::clone(&self.decommission_operation_gate) + } + /// Install the application-owned object-encryption resolver once. pub fn set_object_encryption_resolver( &self, From ddc4120c82bf82627e711deee1f1ce302e141635 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:40:28 +0800 Subject: [PATCH 18/32] ci: detect incomplete and stale scheduled validations (#6357) --- .config/make/tests.mak | 1 + .../actions/schedule-failure-issue/action.yml | 65 +- .github/scheduled-validations.json | 14 + .github/workflows/audit.yml | 2 +- .github/workflows/build.yml | 22 +- .github/workflows/ci-docs-only.yml | 5 +- .github/workflows/ci.yml | 41 +- .github/workflows/coverage.yml | 2 +- .github/workflows/e2e-replication-nightly.yml | 2 +- .github/workflows/e2e-s3tests.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/minio-interop.yml | 18 + .github/workflows/mint.yml | 2 +- .github/workflows/nightly-gnu.yml | 22 +- .github/workflows/performance-ab.yml | 2 +- .github/workflows/runner-hygiene.yml | 2 +- .../scheduled-validation-freshness.yml | 57 ++ .../scheduled-validation-watchdog.yml | 63 ++ .../check_scheduled_validation_freshness.py | 263 ++++++++ scripts/check_test_wiring.py | 562 +++++++++++++++++- 20 files changed, 1123 insertions(+), 26 deletions(-) create mode 100644 .github/scheduled-validations.json create mode 100644 .github/workflows/scheduled-validation-freshness.yml create mode 100644 .github/workflows/scheduled-validation-watchdog.yml create mode 100644 scripts/check_scheduled_validation_freshness.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 1dec7db3e..626df9b84 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -36,6 +36,7 @@ script-tests: ## Run shell script tests ./scripts/test_manual_transition_runbooks.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/actions/schedule-failure-issue/action.yml b/.github/actions/schedule-failure-issue/action.yml index 60e938690..d505387e4 100644 --- a/.github/actions/schedule-failure-issue/action.yml +++ b/.github/actions/schedule-failure-issue/action.yml @@ -14,9 +14,10 @@ name: "Schedule Failure Issue" description: >- - Open (or update) a tracking issue when a scheduled workflow run fails. + Open (or update) a tracking issue when a scheduled workflow run fails or + does not complete normally. Dedupes by workflow name: if an open issue titled - "[scheduled-failure] " already exists, the failure is + "[scheduled-failure] " already exists, the result is appended as a comment; otherwise a new issue is created. This is the single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8). @@ -38,6 +39,30 @@ inputs: Set to an empty string to skip labeling. required: false default: "infrastructure" + source-run-id: + description: "Run ID to report. Defaults to the current workflow run." + required: false + default: ${{ github.run_id }} + source-run-attempt: + description: "Run attempt to report. Defaults to the current attempt." + required: false + default: ${{ github.run_attempt }} + source-event: + description: "Trigger event of the run being reported." + required: false + default: ${{ github.event_name }} + source-ref-name: + description: "Ref name of the run being reported." + required: false + default: ${{ github.ref_name }} + source-sha: + description: "Commit SHA of the run being reported." + required: false + default: ${{ github.sha }} + details-file: + description: "Optional Markdown file appended to the issue body." + required: false + default: "" runs: using: "composite" @@ -48,17 +73,22 @@ runs: GH_TOKEN: ${{ inputs.github-token }} WORKFLOW_NAME: ${{ inputs.workflow-name }} ISSUE_LABEL: ${{ inputs.label }} + SOURCE_RUN_ID: ${{ inputs.source-run-id }} + SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }} + SOURCE_EVENT: ${{ inputs.source-event }} + SOURCE_REF_NAME: ${{ inputs.source-ref-name }} + SOURCE_SHA: ${{ inputs.source-sha }} + DETAILS_FILE: ${{ inputs.details-file }} run: | set -euo pipefail title="[scheduled-failure] ${WORKFLOW_NAME}" - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}" - # Failed job names for this run attempt. The alert job runs while the - # run as a whole is still in progress, so inspect the jobs that have - # already completed with a non-success conclusion. + # Inspect the reported run attempt. It can be the current in-workflow + # failure or a completed run observed by the external watchdog. failed_jobs="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \ --paginate \ --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled") @@ -67,15 +97,26 @@ runs: failed_jobs="- (failed job not recorded yet — see the run page)" fi + details="" + if [ -n "${DETAILS_FILE}" ]; then + if [ -f "${DETAILS_FILE}" ]; then + details="$(cat "${DETAILS_FILE}")" + else + details="Details file was not available: \`${DETAILS_FILE}\`" + fi + fi + body="$(cat <- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 85ce9b32f..de78e7f7f 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -126,7 +126,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a06e4667..ae74ec40a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ on: merge_group: types: [ checks_requested ] schedule: - - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC + - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC workflow_dispatch: permissions: @@ -161,7 +161,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh @@ -1032,3 +1035,37 @@ jobs: path: artifacts/s3tests-single/** if-no-files-found: ignore retention-days: 3 + + alert-on-failure: + name: Alert on scheduled failure + needs: + - typos + - quick-checks + - test-and-lint + - test-ilm-integration-serial + - test-and-lint-rio-v2 + - test-and-lint-protocols + - build-rustfs-debug-binary + - build-rustfs-debug-binary-rio-v2 + - uring-integration + - e2e-tests + - e2e-full + - e2e-tests-rio-v2 + - s3-implemented-tests + - s3-lifecycle-behavior-tests + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index baa3c87a7..ece00c2eb 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,7 +37,7 @@ on: # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17), # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. - - cron: "0 7 * * 0" + - cron: "43 7 * * 0" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 837d9f79f..ef312d180 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -40,7 +40,7 @@ on: schedule: # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00), # stale (01:30) and performance-ab (06:00). - - cron: "0 4 * * *" + - cron: "29 4 * * *" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 1e3ad90b6..693fa8450 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -93,7 +93,7 @@ on: schedule: # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the # single-node and the 4-node distributed topologies (matrix below). - - cron: "0 2 * * 0" + - cron: "19 2 * * 0" env: # main user diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index d41a1107f..6aa780fc6 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -30,7 +30,7 @@ on: - "Cargo.lock" - ".github/workflows/fuzz.yml" schedule: - - cron: "0 2 * * *" + - cron: "17 2 * * *" workflow_dispatch: inputs: profile: diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml index 3ee33e9bf..5f105ac41 100644 --- a/.github/workflows/minio-interop.yml +++ b/.github/workflows/minio-interop.yml @@ -121,3 +121,21 @@ jobs: cargo nextest run --run-ignored ignored-only --no-tests=fail \ -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \ -E "$INTEROP_FILTER" + + alert-on-failure: + name: Alert on scheduled failure + needs: [minio-interop] + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml index dd9acc6f8..b9a7e354a 100644 --- a/.github/workflows/mint.yml +++ b/.github/workflows/mint.yml @@ -76,7 +76,7 @@ on: schedule: # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to # 3h) has finished, so the two never contend for the same runner pool. - - cron: "0 6 * * 0" + - cron: "41 6 * * 0" env: S3_ACCESS_KEY: rustfsadmin-ci diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 1f7c2d488..946761e54 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -16,7 +16,7 @@ name: Nightly GNU Build on: schedule: - - cron: "0 0 * * *" + - cron: "7 0 * * *" timezone: "Asia/Shanghai" workflow_dispatch: @@ -194,3 +194,23 @@ jobs: - name: Run HA leader failover live checks (three-node Raft cluster in Docker) run: bash scripts/test/vault_ha_kms_live.sh + + alert-on-failure: + name: Alert on scheduled failure + needs: [build, kms-vault-lane, kms-vault-ha-failover] + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index ac8e8d7c5..a7ecf1d4b 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -33,7 +33,7 @@ name: Performance A/B on: schedule: - - cron: "0 6 * * *" # 06:00 UTC nightly, against main + - cron: "31 6 * * *" # 06:31 UTC nightly, against main workflow_dispatch: inputs: duration: diff --git a/.github/workflows/runner-hygiene.yml b/.github/workflows/runner-hygiene.yml index c231b5706..bdebf2c77 100644 --- a/.github/workflows/runner-hygiene.yml +++ b/.github/workflows/runner-hygiene.yml @@ -30,7 +30,7 @@ name: Runner Hygiene on: schedule: - - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron) + - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC workflow_dispatch: permissions: diff --git a/.github/workflows/scheduled-validation-freshness.yml b/.github/workflows/scheduled-validation-freshness.yml new file mode 100644 index 000000000..eef340869 --- /dev/null +++ b/.github/workflows/scheduled-validation-freshness.yml @@ -0,0 +1,57 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Freshness + +on: + schedule: + - cron: "47 23 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scheduled-validation-freshness + cancel-in-progress: false + +jobs: + check-freshness: + name: Check scheduled validation freshness + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Check latest scheduled runs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + python3 scripts/check_scheduled_validation_freshness.py \ + --report "${RUNNER_TEMP}/scheduled-validation-freshness.md" + status=$? + cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}" + exit "${status}" + - name: Open or update freshness issue + if: failure() + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + details-file: ${{ runner.temp }}/scheduled-validation-freshness.md diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml new file mode 100644 index 000000000..e778ec640 --- /dev/null +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -0,0 +1,63 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Watchdog + +on: + workflow_run: + workflows: + - "Security Audit" + - "Build and Release" + - "Continuous Integration" + - "coverage" + - "e2e-nightly" + - "e2e-s3tests" + - "Fuzz" + - "mint" + - "minio-interop" + - "Nightly GNU Build" + - "Performance A/B" + - "Runner Hygiene" + types: [completed] + +permissions: + contents: read + +jobs: + alert-on-incomplete-run: + name: Alert on incomplete scheduled run + if: >- + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion != 'success' && + github.event.workflow_run.conclusion != 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update incomplete-run issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + workflow-name: ${{ github.event.workflow_run.name }} + source-run-id: ${{ github.event.workflow_run.id }} + source-run-attempt: ${{ github.event.workflow_run.run_attempt }} + source-event: ${{ github.event.workflow_run.event }} + source-ref-name: ${{ github.event.workflow_run.head_branch }} + source-sha: ${{ github.event.workflow_run.head_sha }} diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py new file mode 100644 index 000000000..d9bdf0597 --- /dev/null +++ b/scripts/check_scheduled_validation_freshness.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Fail when a critical scheduled validation has not started recently.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import re +import sys +import tempfile +import unittest +from unittest import mock +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_validations(path: Path) -> list[tuple[str, int]]: + data = json.loads(path.read_text()) + if not isinstance(data, list) or not data: + raise ValueError("scheduled validation config must be a non-empty list") + + validations: list[tuple[str, int]] = [] + seen: set[str] = set() + for item in data: + if not isinstance(item, dict): + raise ValueError("scheduled validation entries must be objects") + workflow = item.get("workflow") + max_age_hours = item.get("max_age_hours") + if not isinstance(workflow, str) or not re.fullmatch( + r"\.github/workflows/[a-z0-9-]+\.yml", workflow + ): + raise ValueError(f"invalid scheduled validation workflow: {workflow!r}") + if workflow in seen: + raise ValueError(f"duplicate scheduled validation workflow: {workflow}") + if ( + not isinstance(max_age_hours, int) + or isinstance(max_age_hours, bool) + or max_age_hours <= 0 + ): + raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}") + seen.add(workflow) + validations.append((workflow, max_age_hours)) + return validations + + +def parse_timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise ValueError(f"invalid run timestamp: {value!r}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError(f"run timestamp has no timezone: {value!r}") + return parsed.astimezone(timezone.utc) + + +def stale_reason( + run: dict[str, object] | None, now: datetime, max_age_hours: int +) -> str | None: + if run is None: + return "no scheduled run has been recorded" + created_at = parse_timestamp(run.get("created_at")) + age = now - created_at + if age > timedelta(hours=max_age_hours): + return f"last scheduled run is {age.total_seconds() / 3600:.1f}h old" + return None + + +def fetch_latest_scheduled_run( + repository: str, workflow: str, token: str, api_url: str +) -> dict[str, object] | None: + owner, repo = repository.split("/", 1) + workflow_name = Path(workflow).name + endpoint = ( + f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" + f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" + + urlencode({"event": "schedule", "per_page": 1}) + ) + request = Request( + endpoint, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urlopen(request, timeout=30) as response: + payload = json.load(response) + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") + if not runs: + return None + if not isinstance(runs[0], dict): + raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") + return runs[0] + + +def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: + lines = ["## Scheduled validation freshness"] + if not failures: + lines.append("") + lines.append("All critical scheduled validations have a recent scheduled run.") + else: + lines.extend( + [ + "", + "The following critical validations are stale or could not be inspected:", + "", + "| Workflow | Limit | Result | Last run |", + "| --- | ---: | --- | --- |", + ] + ) + for workflow, max_age_hours, reason, run_url in failures: + link = f"[open]({run_url})" if run_url else "—" + lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") + path.write_text("\n".join(lines) + "\n") + + +def check_freshness( + config: Path, report: Path, repository: str, token: str, api_url: str +) -> int: + now = datetime.now(timezone.utc) + failures: list[tuple[str, int, str, str]] = [] + for workflow, max_age_hours in load_validations(config): + try: + run = fetch_latest_scheduled_run(repository, workflow, token, api_url) + reason = stale_reason(run, now, max_age_hours) + if reason is not None: + run_url = str(run.get("html_url", "")) if run else "" + failures.append((workflow, max_age_hours, reason, run_url)) + except Exception as error: + failures.append( + (workflow, max_age_hours, f"inspection failed: {error}", "") + ) + write_report(report, failures) + return 1 if failures else 0 + + +class SelfTests(unittest.TestCase): + NOW = datetime(2026, 8, 22, 12, tzinfo=timezone.utc) + + def test_freshness_boundaries(self) -> None: + at_limit = {"created_at": "2026-08-21T00:00:00Z"} + past_limit = {"created_at": "2026-08-20T23:59:59Z"} + self.assertIsNone(stale_reason(at_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(None, self.NOW, 36)) + + def test_config_rejects_duplicate_and_invalid_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "validations.json" + path.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}, + ] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + path.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + + def test_check_reports_missing_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = root / "validations.json" + report = root / "report.md" + config.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, + ] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + side_effect=[ + {"created_at": "2999-01-01T00:00:00Z"}, + None, + RuntimeError("API unavailable"), + ], + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 1, + ) + contents = report.read_text() + self.assertIn(".github/workflows/fuzz.yml", contents) + self.assertIn("inspection failed: API unavailable", contents) + self.assertNotIn(".github/workflows/ci.yml`", contents) + + config.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + return_value={"created_at": "2999-01-01T00:00:00Z"}, + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 0, + ) + self.assertIn("All critical scheduled validations", report.read_text()) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=Path, default=ROOT / ".github/scheduled-validations.json" + ) + parser.add_argument("--report", type=Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + load_validations(args.config) + suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) + return ( + 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + ) + if args.report is None: + parser.error("--report is required unless --self-test is used") + + repository = os.environ.get("GITHUB_REPOSITORY", "") + token = os.environ.get("GH_TOKEN", "") + api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): + parser.error("GITHUB_REPOSITORY must be owner/repository") + if not token: + parser.error("GH_TOKEN is required") + return check_freshness(args.config, args.report, repository, token, api_url) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 4edd41f0d..aa97dc944 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -10,11 +10,17 @@ import sys import tempfile import tomllib import unittest +from datetime import datetime, timezone from unittest import mock from pathlib import Path +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError ROOT = Path(__file__).resolve().parents[1] +SCHEDULED_ALERT_WORKFLOWS = tuple( + item["workflow"] + for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text()) +) def words(value: str) -> set[str]: @@ -252,6 +258,292 @@ def check_profile_definitions(root: Path) -> list[str]: return errors +def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None: + try: + start = lines.index(f"{' ' * indent}{key}:") + 1 + except ValueError: + return None + end = next( + ( + index + for index in range(start, len(lines)) + if lines[index].strip() + and not lines[index].lstrip().startswith("#") + and len(lines[index]) - len(lines[index].lstrip()) <= indent + ), + len(lines), + ) + return lines[start:end] + + +def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None: + uses_index = next( + ( + index + for index, line in enumerate(job_lines) + if ( + line.split("#", 1)[0].strip() == f"- uses: {action}" + and len(line) - len(line.lstrip()) == 6 + ) + or ( + line.split("#", 1)[0].strip() == f"uses: {action}" + and len(line) - len(line.lstrip()) == 8 + ) + ), + None, + ) + if uses_index is None: + return None + start = next( + ( + index + for index in range(uses_index, -1, -1) + if job_lines[index].lstrip().startswith("- ") + ), + uses_index, + ) + indent = len(job_lines[start]) - len(job_lines[start].lstrip()) + end = next( + ( + index + for index in range(start + 1, len(job_lines)) + if len(job_lines[index]) - len(job_lines[index].lstrip()) == indent + and job_lines[index].lstrip().startswith("- ") + ), + len(job_lines), + ) + return start, job_lines[start:end] + + +def alert_step_errors( + job_lines: list[str], + expected_action_if: str | None, + required_permissions: tuple[str, ...], + required_action_tokens: tuple[str, ...], +) -> list[str]: + checkout = workflow_step_block(job_lines, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0") + action = workflow_step_block(job_lines, "./.github/actions/schedule-failure-issue") + errors: list[str] = [] + permissions = yaml_block(job_lines, "permissions", 4) + permission_text = "\n".join(line.split("#", 1)[0] for line in permissions or []) + missing_permissions = [token for token in required_permissions if token not in permission_text] + if missing_permissions: + errors.append("alert job permissions missing " + ", ".join(missing_permissions)) + if checkout is None: + errors.append("checkout step is missing") + if action is None: + errors.append("local alert action step is missing") + if checkout is None or action is None: + return errors + + if checkout[0] >= action[0]: + errors.append("checkout must run before the local alert action") + checkout_ifs = [line.strip() for line in checkout[1] if line.strip().startswith("if:")] + if checkout_ifs: + errors.append("checkout step must not be conditional") + action_ifs = [line.strip() for line in action[1] if line.strip().startswith("if:")] + expected_ifs = [] if expected_action_if is None else [expected_action_if] + if action_ifs != expected_ifs: + errors.append("alert action has an invalid step condition") + action_text = "\n".join(line.split("#", 1)[0] for line in action[1]) + missing_action_tokens = [token for token in required_action_tokens if token not in action_text] + if missing_action_tokens: + errors.append("alert action inputs missing " + ", ".join(missing_action_tokens)) + return errors + + +def schedule_utc_slots(hour: int, minute: int, timezone_name: str | None) -> set[tuple[int, int]]: + if timezone_name is None: + return {(hour, minute)} + zone = ZoneInfo(timezone_name) + return { + (utc.hour, utc.minute) + for year in (2025, 2026) + for month in range(1, 13) + for utc in [datetime(year, month, 1, hour, minute, tzinfo=zone).astimezone(timezone.utc)] + } + + +def check_scheduled_alerts(root: Path) -> list[str]: + errors: list[str] = [] + schedule_slots: dict[tuple[int, int], list[str]] = {} + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + errors.append(f"{relative}: missing scheduled validation workflow") + continue + + on_block = yaml_block(lines, "on", 0) + schedule_block = yaml_block(on_block or [], "schedule", 2) + schedule_lines = schedule_block or [] + cron_indices = [index for index, line in enumerate(schedule_lines) if re.match(r"^\s*-\s+cron:", line)] + if not cron_indices: + errors.append(f"{relative}: missing simple numeric schedule") + else: + for position, cron_index in enumerate(cron_indices): + cron_line = schedule_lines[cron_index] + schedule = re.match(r"^\s*-\s+cron:\s*[\"']?(\d+)\s+(\d+)\s+", cron_line) + if not schedule: + errors.append(f"{relative}: missing simple numeric schedule") + continue + minute, hour = map(int, schedule.groups()) + if minute == 0: + errors.append(f"{relative}: scheduled validation must avoid minute zero") + entry_end = cron_indices[position + 1] if position + 1 < len(cron_indices) else len(schedule_lines) + entry = "\n".join(schedule_lines[cron_index + 1 : entry_end]) + timezone_match = re.search(r"^\s*timezone:\s*[\"']?([^\"'\s]+)", entry, re.MULTILINE) + timezone_name = timezone_match.group(1) if timezone_match else None + try: + utc_slots = schedule_utc_slots(hour, minute, timezone_name) + except ZoneInfoNotFoundError: + errors.append(f"{relative}: unknown schedule timezone {timezone_name}") + continue + for slot in utc_slots: + schedule_slots.setdefault(slot, []).append(relative) + + job_lines = yaml_block(lines, "alert-on-failure", 2) + if job_lines is None: + errors.append(f"{relative}: missing alert-on-failure job") + continue + job = "\n".join(line.split("#", 1)[0] for line in job_lines) + required = ( + "always()", + "github.event_name == 'schedule'", + "contains(needs.*.result, 'failure')", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + ) + missing = [token for token in required if token not in job] + if missing: + errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}") + else: + errors.extend( + f"{relative}: {error}" + for error in alert_step_errors(job_lines, None, ("issues: write",), ("github-token: ${{ secrets.GITHUB_TOKEN }}",)) + ) + + for (hour, minute), workflows in schedule_slots.items(): + if len(workflows) > 1: + errors.append( + f"scheduled validations share {hour:02d}:{minute:02d} UTC: {', '.join(workflows)}" + ) + + watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml" + try: + watchdog_lines = watchdog_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog") + return errors + watchdog_on = yaml_block(watchdog_lines, "on", 0) + watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2) + watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4) + if watchdog_workflows is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows") + return errors + watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows) + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + if not path.is_file(): + continue + source = path.read_text() + match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE) + if not match: + errors.append(f"{relative}: missing workflow name") + elif f'- "{match.group(1).strip()}"' not in watchdog_sources: + errors.append(f"{relative}: missing from scheduled completion watchdog") + watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2) + if watchdog_job_lines is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job") + return errors + watchdog_job = "\n".join(line.split("#", 1)[0] for line in watchdog_job_lines) + required = ( + "github.event.workflow_run.event == 'schedule'", + "github.event.workflow_run.conclusion != 'success'", + "github.event.workflow_run.conclusion != 'failure'", + "actions: read", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ) + missing = [token for token in required if token not in watchdog_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-watchdog.yml: " + error + for error in alert_step_errors( + watchdog_job_lines, + None, + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ), + ) + ) + + freshness_path = root / ".github/workflows/scheduled-validation-freshness.yml" + try: + freshness_lines = freshness_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing freshness check") + return errors + freshness_job_lines = yaml_block(freshness_lines, "check-freshness", 2) + if freshness_job_lines is None: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing check-freshness job") + return errors + freshness_job = "\n".join(line.split("#", 1)[0] for line in freshness_job_lines) + required = ( + "python3 scripts/check_scheduled_validation_freshness.py", + "actions: read", + "issues: write", + "if: failure()", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ) + missing = [token for token in required if token not in freshness_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-freshness.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-freshness.yml: " + error + for error in alert_step_errors( + freshness_job_lines, + "if: failure()", + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ), + ) + ) + if not (root / "scripts/check_scheduled_validation_freshness.py").is_file(): + errors.append("scripts/check_scheduled_validation_freshness.py: missing freshness checker") + return errors + + def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: try: expected_digest = profile_selection(root, profile) @@ -281,6 +573,7 @@ def validate(root: Path) -> list[str]: errors.extend(check_runner_selection(root)) errors.extend(check_s3_tests_runner(root)) errors.extend(check_profile_definitions(root)) + errors.extend(check_scheduled_alerts(root)) return errors @@ -363,6 +656,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), + mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]), ): self.assertEqual(len(validate(root)), 1) @@ -413,6 +707,272 @@ class SelfTests(unittest.TestCase): with mock.patch.object(sys, "platform", "linux"): self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1) + def test_scheduled_alerts_require_completion_watchdog(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + alert = ( + " alert-on-failure:\n" + " if: always() && github.event_name == 'schedule' && " + "contains(needs.*.result, 'failure')\n" + " permissions:\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + names: list[str] = [] + for index, relative in enumerate(SCHEDULED_ALERT_WORKFLOWS, start=1): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + names.append(path.stem) + path.write_text( + f'name: "{path.stem}"\n' + f'on:\n schedule:\n - cron: "{index} {index} * * *"\n' + f'jobs:\n{alert}' + ) + watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml" + watchdog.write_text( + "on:\n workflow_run:\n workflows:\n" + + "\n".join(f' - "{name}"' for name in names) + + "\njobs:\n" + + " alert-on-incomplete-run:\n" + + " github.event.workflow_run.event == 'schedule'\n" + + " github.event.workflow_run.conclusion != 'success'\n" + + " github.event.workflow_run.conclusion != 'failure'\n" + + " permissions:\n" + + " actions: read\n" + + " issues: write\n" + + " steps:\n" + + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + + " - uses: ./.github/actions/schedule-failure-issue\n" + + " with:\n" + + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + + " workflow-name: ${{ github.event.workflow_run.name }}\n" + + " source-run-id: ${{ github.event.workflow_run.id }}\n" + + " source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n" + + " source-event: ${{ github.event.workflow_run.event }}\n" + + " source-ref-name: ${{ github.event.workflow_run.head_branch }}\n" + + " source-sha: ${{ github.event.workflow_run.head_sha }}\n" + ) + freshness = root / ".github/workflows/scheduled-validation-freshness.yml" + freshness.write_text( + "jobs:\n" + " check-freshness:\n" + " permissions:\n" + " actions: read\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - run: python3 scripts/check_scheduled_validation_freshness.py\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: failure()\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " details-file: ${{ runner.temp }}/scheduled-validation-freshness.md\n" + ) + checker = root / "scripts/check_scheduled_validation_freshness.py" + checker.parent.mkdir() + checker.write_text("") + self.assertEqual(check_scheduled_alerts(root), []) + + first = root / SCHEDULED_ALERT_WORKFLOWS[0] + mutations = ( + ("contains(needs.*.result, 'failure')", "false"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ) + for required, replacement in mutations: + original = first.read_text() + first.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + first_original = first.read_text() + real_steps = ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + first.write_text( + first_original.replace( + real_steps, + " - run: |\n" + " : <<'MARKER'\n" + " uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " MARKER\n" + " - run: |\n" + " : <<'MARKER'\n" + " uses: ./.github/actions/schedule-failure-issue\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " MARKER\n", + ) + ) + self.assertTrue(check_scheduled_alerts(root)) + first.write_text( + first_original.replace( + real_steps, + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog_mutations = ( + ("actions: read", "actions: none"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ("source-event: ${{ github.event.workflow_run.event }}", "source-event: watchdog"), + ( + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-ref-name: main", + ), + ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: missing"), + ) + for required, replacement in watchdog_mutations: + original = watchdog.read_text() + watchdog.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(original) + + watchdog_original = watchdog.read_text() + watchdog.write_text( + watchdog_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + first_original = first.read_text() + first.write_text( + first_original.replace(' schedule:\n - cron: "1 1 * * *"\n', "") + + ' decoy:\n strategy:\n matrix:\n cron:\n - "1 1 * * *"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "0 5 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "2 2 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog.write_text( + watchdog_original.replace(f' - "{names[0]}"\n', "") + + f' decoy:\n strategy:\n matrix:\n workflow:\n - "{names[0]}"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + watchdog.write_text(watchdog_original.replace(f' - "{names[0]}"\n', "")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + original = first.read_text() + first.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "0 0', original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + second = root / SCHEDULED_ALERT_WORKFLOWS[1] + second_original = second.read_text() + second.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "1 1', second_original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + second.write_text(second_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "7 0 * * *"\n timezone: "Asia/Shanghai"\n', + ) + ) + second.write_text( + second_original.replace( + ' - cron: "2 2 * * *"\n', + ' - cron: "2 2 * * *"\n - cron: "7 16 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + second.write_text(second_original) + + freshness_original = freshness.read_text() + freshness.write_text(freshness_original.replace("details-file:", "report-file:")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("if: failure()", "if: github.event_name == 'workflow_dispatch'") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) @@ -436,7 +996,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 - print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired") + print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired") return 0 From 51e369be6c13a48d87569b9d6fa3eddfeff019cd Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 23 Aug 2026 01:40:44 +0800 Subject: [PATCH 19/32] fix(policy): accept legacy bucket policy ID field (#6362) --- crates/policy/src/policy/policy.rs | 42 +++++++++++++++++++- docs/architecture/compat-cleanup-register.md | 1 + 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index d412fa789..946f930a8 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> { #[derive(Serialize, Deserialize, Clone, Default, Debug)] #[serde(deny_unknown_fields)] pub struct BucketPolicy { - #[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")] + // RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy. + #[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")] pub id: ID, #[serde(rename = "Version")] pub version: String, @@ -2786,7 +2787,7 @@ mod test { let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); // Verify empty fields are omitted - assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted"); + assert!(parsed.get("Id").is_none(), "Empty ID should be omitted"); let statement = &parsed["Statement"][0]; assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted"); @@ -2809,6 +2810,43 @@ mod test { assert_eq!(statement["Principal"]["AWS"], "*"); } + #[test] + fn test_bucket_policy_deserializes_legacy_id() { + let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#; + + let policy: BucketPolicy = + serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize"); + assert!(policy.id.is_empty()); + policy.is_valid().expect("legacy bucket policy should remain valid"); + + let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#) + .expect("non-empty legacy ID should deserialize"); + assert_eq!(policy.id.0, "legacy-policy"); + + let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize"); + assert_eq!(serialized["Id"], "legacy-policy"); + assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized"); + } + + #[test] + fn test_bucket_policy_legacy_id_alias_remains_strict() { + let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#; + let error = + serde_json::from_str::(unknown_field).expect_err("unrelated unknown fields should remain rejected"); + assert!( + error.to_string().contains("unknown field `Unexpected`"), + "unexpected deserialization error: {error}" + ); + + let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#; + let error = serde_json::from_str::(duplicate_id) + .expect_err("canonical and legacy ID fields should not be accepted together"); + assert!( + error.to_string().contains("duplicate field `Id`"), + "unexpected deserialization error: {error}" + ); + } + #[test] fn test_existing_object_tag_condition_helpers() { let identity_policy = Policy::parse_config( diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 9904b41c9..d7ecec7f5 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,7 @@ for later deletion. ## Open Items +- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID". - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. From 26e6508b64401131127a35aebede0d56ffa34507 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:42:11 +0800 Subject: [PATCH 20/32] fix(ecstore): reject equal-time latest identity conflicts before index fallback (#6374) --- crates/ecstore/src/store/rebalance.rs | 387 +++++++++++++++++- crates/ecstore/src/store/rebalance/support.rs | 167 +++++++- 2 files changed, 532 insertions(+), 22 deletions(-) diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 7d7c1d91e..dc2fa2ee5 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl #[cfg(test)] mod tests { use super::*; + use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env}; use crate::disk::error::DiskError; use crate::layout::endpoint::Endpoint; @@ -1423,6 +1424,14 @@ mod tests { } } + fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option) -> ObjectInfo { + ObjectInfo { + version_id: Some(version_id), + etag, + ..object_info_with_mod_time(unix_ts, delete_marker) + } + } + #[test] fn resolve_latest_object_info_candidates_returns_latest_delete_marker() { let candidates = vec![ @@ -1446,7 +1455,7 @@ mod tests { } #[test] - fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() { + fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() { let candidates = vec![ LatestObjectInfoCandidate { info: Some(object_info_with_mod_time(10, false)), @@ -1466,6 +1475,382 @@ mod tests { assert_eq!(idx, 1); } + #[test] + fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 2, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 7, + err: None, + }, + ]; + + let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect("equivalent replicas must resolve deterministically"); + + assert_eq!(idx, 7); + assert_eq!(info.version_id, Some(Uuid::from_u128(1))); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("divergent version ids must not silently resolve to the higher pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("divergent etags must not silently resolve to the higher pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("a delete marker tied with a live version must not be masked by the pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) { + let err = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(left), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(right), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect_err("equal-time identity divergence must fail closed"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + let mut data_dir = base.clone(); + data_dir.data_dir = Some(Uuid::from_u128(2)); + assert_equal_time_identity_conflict(base.clone(), data_dir); + + let mut size = base.clone(); + size.size = 1; + assert_equal_time_identity_conflict(base.clone(), size); + + let mut actual_size = base.clone(); + actual_size.actual_size = 1; + assert_equal_time_identity_conflict(base.clone(), actual_size); + + let mut checksum = base.clone(); + checksum.checksum = Some(bytes::Bytes::from_static(b"checksum")); + assert_equal_time_identity_conflict(base.clone(), checksum); + + let mut parts = base.clone(); + parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + etag: "part-etag".to_string(), + number: 1, + size: 1, + ..Default::default() + }]); + assert_equal_time_identity_conflict(base.clone(), parts); + + let mut transition = base; + transition.transitioned_object.tier = "tier-a".to_string(); + assert_equal_time_identity_conflict( + object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())), + transition, + ); + } + + #[test] + fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + let mut minio_alias = base.clone(); + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "X-MINIO-INTERNAL-COMPRESSION".to_string(), + "zstd".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(rustfs_alias), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(minio_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("same-value internal aliases should resolve"); + assert_eq!(idx, 1); + + let mut dual_alias = base.clone(); + dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([ + ("x-rustfs-internal-compression".to_string(), "zstd".to_string()), + ("x-minio-internal-compression".to_string(), "zstd".to_string()), + ])); + let mut single_alias = base; + single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(dual_alias), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(single_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("dual-key and single-key internal metadata should resolve"); + assert_eq!(idx, 1); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + let mut minio_alias = base; + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-minio-internal-compression".to_string(), + "snappy".to_string(), + )])); + + assert_equal_time_identity_conflict(rustfs_alias, minio_alias); + } + + #[test] + fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() { + for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!( + "X-RUSTFS-INTERNAL-{}{suffix}", + suffix_prefix.to_uppercase(), + suffix = "arn:aws:s3:::Bucket" + ), + "value".to_string(), + )])); + let mut minio_alias = base.clone(); + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"), + "value".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(rustfs_alias.clone()), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(minio_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("dynamic internal aliases with the same target should resolve"); + assert_eq!(idx, 1); + + let mut different_target_case = base; + different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"), + "value".to_string(), + )])); + + assert_equal_time_identity_conflict(rustfs_alias, different_target_case); + } + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut first = base.clone(); + first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([ + ("x-rustfs-internal-compression".to_string(), "zstd".to_string()), + ("x-minio-internal-compression".to_string(), "snappy".to_string()), + ])); + let mut second = base; + second.user_defined = first.user_defined.clone(); + + assert_equal_time_identity_conflict(first, second); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + let mut replication = base.clone(); + replication.replication_status_internal = Some("PENDING".to_string()); + replication.replication_status = ReplicationStatusType::Pending; + assert_equal_time_identity_conflict(base.clone(), replication); + + let mut purge = base.clone(); + purge.version_purge_status_internal = Some("PENDING".to_string()); + purge.version_purge_status = VersionPurgeStatusType::Pending; + assert_equal_time_identity_conflict(base.clone(), purge); + + let mut decision = base; + decision.replication_decision = "replicate".to_string(); + assert_equal_time_identity_conflict( + object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())), + decision, + ); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() { + let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string())); + without_mod_time.mod_time = None; + let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch); + } + + #[test] + fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() { + let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string())); + let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string())); + older.data_dir = Some(Uuid::from_u128(2)); + + let (info, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(latest), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(older), + idx: 9, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("older identity divergence must not affect the latest candidate"); + + assert_eq!(idx, 0); + assert_eq!( + info.mod_time, + Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed")) + ); + } + + #[test] + fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: None, + idx: 1, + err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())), + }, + ]; + + let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect("not-found pools must not block resolution of found candidates"); + + assert_eq!(idx, 0); + assert_eq!(info.version_id, Some(Uuid::from_u128(1))); + } + #[test] fn resolve_latest_object_info_candidates_returns_non_not_found_error() { let err = resolve_latest_object_info_candidates( diff --git a/crates/ecstore/src/store/rebalance/support.rs b/crates/ecstore/src/store/rebalance/support.rs index e37fc97bc..6e4db41b8 100644 --- a/crates/ecstore/src/store/rebalance/support.rs +++ b/crates/ecstore/src/store/rebalance/support.rs @@ -12,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Ordering; +use std::collections::HashMap; use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found}; use crate::object_api::{ObjectInfo, ObjectOptions}; +use rustfs_utils::http::metadata_compat::{ + SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX, + strip_internal_prefix_preserving_case, +}; use rustfs_utils::path::decode_dir_object; use time::OffsetDateTime; @@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p )) } +fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option { + candidate + .info + .as_ref() + .map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)) +} + +fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + left.transition_version_state == right.transition_version_state + && left.transitioned_object.name == right.transitioned_object.name + && left.transitioned_object.version_id == right.transitioned_object.version_id + && left.transitioned_object.tier == right.transitioned_object.tier + && left.transitioned_object.free_version == right.transitioned_object.free_version + && left.transitioned_object.status == right.transitioned_object.status +} + +#[derive(PartialEq, Eq)] +struct LatestUserDefinedIdentity { + internal: HashMap, + other: HashMap, +} + +fn normalize_internal_identity_suffix(key: &str) -> Option { + let suffix = strip_internal_prefix_preserving_case(key)?; + + for dynamic_prefix in [ + SUFFIX_REPLICATION_RESET_ARN_PREFIX, + SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, + ] { + let prefix_len = dynamic_prefix.len(); + if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..)) + && prefix.eq_ignore_ascii_case(dynamic_prefix) + { + return Some(format!("{dynamic_prefix}{remainder}")); + } + } + + Some(suffix.to_lowercase()) +} + +fn normalize_user_defined_identity(user_defined: &HashMap) -> Option { + let mut identity = LatestUserDefinedIdentity { + internal: HashMap::with_capacity(user_defined.len()), + other: HashMap::with_capacity(user_defined.len()), + }; + + for (key, value) in user_defined { + if let Some(suffix) = normalize_internal_identity_suffix(key) { + if identity + .internal + .insert(suffix, value.clone()) + .is_some_and(|previous| previous != *value) + { + return None; + } + } else { + identity.other.insert(key.clone(), value.clone()); + } + } + + Some(identity) +} + +fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + match ( + normalize_user_defined_identity(&left.user_defined), + normalize_user_defined_identity(&right.user_defined), + ) { + (Some(left), Some(right)) => left == right, + _ => false, + } +} + +/// Pool-specific erasure geometry is intentionally excluded: `get_object_info` +/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can +/// differ for the same object version while the selected winner still carries +/// the chosen pool's layout. `put_object_reader` is also intentionally +/// excluded because it is a transient request handle that `ObjectInfo::clone` +/// drops. Every other ObjectInfo field is part of the production-visible +/// identity and must agree before the pool index can provide a deterministic +/// tie-break. +fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + left.bucket == right.bucket + && left.name == right.name + && left.storage_class == right.storage_class + && left.mod_time == right.mod_time + && left.size == right.size + && left.actual_size == right.actual_size + && left.is_dir == right.is_dir + && same_user_defined_identity(left, right) + && left.user_tags == right.user_tags + && left.version_id == right.version_id + && left.data_dir == right.data_dir + && left.delete_marker == right.delete_marker + && same_transition_identity(left, right) + && left.restore_ongoing == right.restore_ongoing + && left.restore_expires == right.restore_expires + && left.parts == right.parts + && left.is_latest == right.is_latest + && left.content_type == right.content_type + && left.content_encoding == right.content_encoding + && left.expires == right.expires + && left.num_versions == right.num_versions + && left.successor_mod_time == right.successor_mod_time + && left.etag == right.etag + && left.inlined == right.inlined + && left.metadata_only == right.metadata_only + && left.version_only == right.version_only + && left.replication_status_internal == right.replication_status_internal + && left.replication_status == right.replication_status + && left.version_purge_status_internal == right.version_purge_status_internal + && left.version_purge_status == right.version_purge_status + && left.replication_decision == right.replication_decision + && left.checksum == right.checksum +} + pub(super) fn resolve_latest_object_info_candidates( - mut candidates: Vec, + candidates: Vec, bucket: &str, object: &str, opts: &ObjectOptions, ) -> Result<(ObjectInfo, usize)> { - candidates.sort_by(|a, b| { - let a_mod = if let Some(info) = &a.info { - info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH + let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max(); + + if let Some(latest_mod_time) = latest_mod_time { + let mut latest_candidates = candidates + .into_iter() + .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time)) + .collect::>(); + + latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx)); + + let Some(winner) = latest_candidates.first() else { + return Err(Error::ErasureReadQuorum); + }; + let Some(winner_info) = winner.info.as_ref() else { + return Err(Error::ErasureReadQuorum); }; - let b_mod = if let Some(info) = &b.info { - info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH - }; - - if a_mod == b_mod { - return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less }; + if latest_candidates.iter().skip(1).any(|candidate| { + candidate + .info + .as_ref() + .is_none_or(|info| !same_latest_object_info_identity(winner_info, info)) + }) { + return Err(Error::ErasureReadQuorum); } - b_mod.cmp(&a_mod) - }); + return Ok((winner_info.clone(), winner.idx)); + } for candidate in candidates { - if let Some(info) = candidate.info { - return Ok((info, candidate.idx)); - } - if let Some(err) = candidate.err && !is_err_object_not_found(&err) && !is_err_version_not_found(&err) From 6d85a9c6a87f456624e1819e72ac9ba327f0d9bb Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:42:58 +0800 Subject: [PATCH 21/32] ci(s3tests): stabilize HAProxy request handling (#6386) --- .github/workflows/e2e-s3tests.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 693fa8450..1be61d73e 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -90,6 +90,10 @@ on: description: "Optional pytest -m expression" required: false default: "" + testexpr: + description: "Optional pytest -k expression" + required: false + default: "" schedule: # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the # single-node and the 4-node distributed topologies (matrix below). @@ -116,6 +120,7 @@ env: XDIST: ${{ github.event.inputs.xdist || '4' }} MAXFAIL: ${{ github.event.inputs.maxfail || '0' }} MARKEXPR: ${{ github.event.inputs.markexpr || '' }} + TESTEXPR: ${{ github.event.inputs.testexpr || '' }} S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }} TEST_TIMEOUT: "300" @@ -269,14 +274,20 @@ jobs: EOF cat > haproxy.cfg <<'EOF' + global + log stdout format raw local0 info + defaults mode http + log global + log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP' timeout connect 5s timeout client 30s timeout server 30s frontend fe_s3 bind *:9000 + option http-buffer-request default_backend be_s3 backend be_s3 @@ -314,6 +325,7 @@ jobs: XDIST="${XDIST}" \ MAXFAIL="${MAXFAIL}" \ MARKEXPR="${MARKEXPR}" \ + TESTEXPR="${TESTEXPR}" \ ./scripts/s3-tests/run.sh - name: Publish compatibility report From c6590182eda61183c5e41ac53bc23938e84426e5 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:43:27 +0800 Subject: [PATCH 22/32] ci(mint): pin manual image default (#6387) --- .github/workflows/mint.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml index b9a7e354a..25ae02916 100644 --- a/.github/workflows/mint.yml +++ b/.github/workflows/mint.yml @@ -45,13 +45,6 @@ # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but # has fewer cores and reintroduces fleet-state risk for no reliability gain. -# DISABLED. This workflow is switched off in the repository's Actions settings -# (state: disabled_manually) and does not run on any trigger, including its cron -# and workflow_dispatch. That state lives in GitHub's UI and is invisible when -# reading this file, which has already misled at least one audit — hence this -# banner. Re-enabling is a UI action; anyone doing so should first check that the -# workflow still matches the current CI layout. See rustfs/backlog#1603. -# name: mint on: @@ -70,9 +63,9 @@ on: - core - full mint-image: - description: "Mint image reference" + description: "Mint image reference (empty = pinned default)" required: false - default: "minio/mint:edge" + default: "" schedule: # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to # 3h) has finished, so the two never contend for the same runner pool. From f44b30c61a1d6b0ae41a45c782892eaa5301c3ac Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:43:52 +0800 Subject: [PATCH 23/32] ci(perf): fix nightly regression baseline (#6389) --- .github/workflows/performance-ab.yml | 159 ++++++++---------- .../security/check_performance_ab_workflow.sh | 23 ++- 2 files changed, 88 insertions(+), 94 deletions(-) diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index a7ecf1d4b..de4986388 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -22,13 +22,6 @@ # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not # blocked (rustfs/backlog#935 correction 1). -# DISABLED. This workflow is switched off in the repository's Actions settings -# (state: disabled_manually) and does not run on any trigger, including its cron -# and workflow_dispatch. That state lives in GitHub's UI and is invisible when -# reading this file, which has already misled at least one audit — hence this -# banner. Re-enabling is a UI action; anyone doing so should first check that the -# workflow still matches the current CI layout. See rustfs/backlog#1603. -# name: Performance A/B on: @@ -37,7 +30,7 @@ on: workflow_dispatch: inputs: duration: - description: "warp duration per round (short by default to fit the double-build budget)" + description: "warp duration per round" required: false default: "12s" type: string @@ -46,12 +39,8 @@ on: required: false default: false type: boolean - push: - # Every main commit pre-builds and caches its release binary (perf-3) so the - # nightly A/B restores a ready baseline instead of paying the double build. - branches: [main] - permissions: + actions: read contents: read env: @@ -59,83 +48,19 @@ env: RUST_BACKTRACE: 1 jobs: - # perf-3: on every push to main, build the release binary once and cache it - # keyed by commit SHA (rustfs-baseline-). The warp-ab measurements - # restore this instead of paying the ~32min-per-side source - # build. That double build is what pushed the expanded 24-cell nightly past its - # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental - # builds off the shared cargo cache keep each push cheap, and building on the - # same sm-standard-2 runner the A/B measures on guarantees the cached binary is - # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those - # are cancelled ~7/8 of the time and are not a reliable baseline. - build-baseline-cache: - name: Build + cache baseline binary - if: github.event_name == 'push' - runs-on: sm-standard-2 - # Latest-wins: consumers only ever restore the binary for the *current* - # origin/main tip, so when pushes land faster than the ~65min build, a - # superseded build's output is dead weight — cancel it instead of stacking - # hour-long jobs on the shared runner pool. A skipped intermediate SHA at - # most costs one same-commit self-heal in the A/B job. - concurrency: - group: perf-baseline-build-main - cancel-in-progress: true - # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a - # single release build past 60min on this runner — every cache build on - # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution - # time of 1h0m0s") and the cache never populated. The measured binary must - # keep the production profile, so the budget absorbs the build instead. - timeout-minutes: 100 - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - - name: Setup Rust environment - uses: ./.github/actions/setup - with: - rust-version: stable - cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} - cache-save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release rustfs - run: cargo build --release --bin rustfs - - - name: Stage binary for cache - run: | - set -euo pipefail - mkdir -p baseline-bin - cp target/release/rustfs baseline-bin/rustfs - - - name: Cache baseline binary by SHA - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ github.sha }} - warp-ab: name: Warp A/B budget gate - # Always run on schedule / manual dispatch. Never on push — that event only - # feeds build-baseline-cache above. - if: >- - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' runs-on: sm-standard-2 - # With perf-3's cached baseline binary the common (cache-hit) nightly is - # measurement-only and finishes well under 50min. This ceiling stays - # generous only to absorb the same-commit cache-miss self-heal (~65min - # single build with the post-#4806 LTO profile + measurement). A timeout - # surfaces via the alert-on-failure job (it fires on cancelled/timed-out, - # not just failure). perf-6 recalibrates the budget once the noise study - # lands. - timeout-minutes: 120 + # A normal nightly restores the last successful binary and builds only the + # candidate; daily access keeps that cache warm. A cache miss may build both + # and needs room for the A/B run plus artifact and cache publication. + timeout-minutes: 180 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - fetch-depth: 0 # baseline is built from origin/main + fetch-depth: 0 # baseline may be an earlier successful scheduled head - name: Setup Rust environment uses: ./.github/actions/setup @@ -163,24 +88,55 @@ jobs: fi echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" - # perf-3: resolve the commits so the cache can be keyed by SHA. The - # baseline is origin/main; the candidate is the checked-out ref. On the - # nightly (checkout == main) they are the same commit, so one cached binary - # serves both phases and the run does zero source builds. + # A failed regression run must keep comparing against the last known-good + # scheduled head. Otherwise the next nightly would absorb the regression + # into its baseline and turn green without a fix. + - name: Find last successful scheduled baseline + id: scheduled_baseline + if: github.event_name == 'schedule' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + result-encoding: string + script: | + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "performance-ab.yml", + event: "schedule", + status: "success", + per_page: 1, + }); + return data.workflow_runs[0]?.head_sha ?? ""; + + # Manual runs compare a selected ref with current main. Scheduled runs + # compare current main with the last successful scheduled head. With no + # history, the first run measures the candidate against itself and seeds + # that head only if the complete rig succeeds. - name: Resolve baseline / candidate commits id: commits + env: + SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }} run: | set -euo pipefail - baseline_sha="$(git rev-parse origin/main)" candidate_sha="$(git rev-parse HEAD)" + if [[ "${{ github.event_name }}" == "schedule" ]]; then + baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}" + if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then + echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2 + exit 1 + fi + else + baseline_sha="$(git rev-parse origin/main)" + fi + git cat-file -e "${baseline_sha}^{commit}" echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT" echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT" echo "baseline commit: $baseline_sha" echo "candidate commit: $candidate_sha" - # Exact-key restore of the baseline binary built by build-baseline-cache - # when origin/main last landed. A miss (binary evicted or not built yet) - # leaves cache-hit unset and the rig falls back to a source build. + # Exact-key restore of the candidate binary saved by its successful + # scheduled run. A miss leaves cache-hit unset and falls back to a source + # build of that known-good head. - name: Restore cached baseline binary id: baseline_cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 @@ -270,11 +226,11 @@ jobs: elif [[ "$selfheal_built" == "true" ]]; then base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)" else - base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)" + base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)" fi if [[ "$candidate_sha" == "$baseline_sha" ]]; then - # Nightly on main: the candidate is the same commit as the baseline, - # so reuse the one binary for both phases and skip all builds. + # No commits landed since the last successful baseline, so reuse + # the one binary for both phases and measure only rig drift. args+=(--candidate-bin "$base_bin") cand_src="same binary as baseline (same commit)" elif [[ "$candidate_built" == "true" ]]; then @@ -362,6 +318,23 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" + - name: Stage successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + run: | + set -euo pipefail + cp candidate-bin/rustfs baseline-bin/rustfs + + - name: Cache successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: baseline-bin/rustfs + key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }} + # Scheduled failure alerting is handled by the alert-on-failure job below # (perf-2 consuming ci-8's schedule-failure-issue composite action). diff --git a/scripts/security/check_performance_ab_workflow.sh b/scripts/security/check_performance_ab_workflow.sh index 5465e4fdf..09b97cc0d 100755 --- a/scripts/security/check_performance_ab_workflow.sh +++ b/scripts/security/check_performance_ab_workflow.sh @@ -13,8 +13,29 @@ require_absent_pattern() { fi } +require_present_pattern() { + local pattern="$1" + local description="$2" + + if ! grep -Eq -- "$pattern" "$workflow"; then + echo "invalid performance A/B workflow contract: $description" >&2 + exit 1 + fi +} + require_absent_pattern '(^|[^[:alnum:]_])pull_request(_target)?([^[:alnum:]_]|$)' "the workflow must not contain PR event handling" require_absent_pattern 'pull-requests[[:space:]]*:[[:space:]]*write' "the workflow must not receive PR write permission" require_absent_pattern 'permissions[[:space:]]*:[[:space:]]*write-all' "the workflow must not receive broad write permission" +require_absent_pattern '^[[:space:]]*push:' "the workflow must not spend a release build on every main push" +require_present_pattern 'listWorkflowRuns' "the scheduled baseline must come from workflow history" +require_present_pattern 'status:[[:space:]]*"success"' "the scheduled baseline must be a successful run" +require_present_pattern 'SCHEDULED_BASELINE_SHA' "the resolved scheduled baseline must reach the comparison" +require_present_pattern "SCHEDULED_BASELINE_SHA:-\\\$candidate_sha" "the first scheduled run must seed from its verified candidate" +require_present_pattern 'git merge-base --is-ancestor' "the scheduled baseline must stay on candidate history" +require_present_pattern 'Cache successful candidate baseline' "a successful candidate must become the next cached baseline" +if ! sed -n '/^ warp-ab:/,/^ alert-on-failure:/p' "$workflow" | grep -Eq '^ timeout-minutes:[[:space:]]*180([[:space:]]|$)'; then + echo "invalid performance A/B workflow contract: the cold-cache path must fit both builds, the A/B run, and evidence publication" >&2 + exit 1 +fi -echo "Performance A/B workflow trust boundary ok." +echo "Performance A/B workflow contract ok." From 7ba6f8cb33a3eeb875321ea08d245e751d10c292 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:44:35 +0800 Subject: [PATCH 24/32] test(e2e): fix cluster nightly oracles (#6397) --- .github/workflows/e2e-replication-nightly.yml | 3 ++ crates/e2e_test/src/object_lambda_test.rs | 37 +++++++++++++------ .../stale_multipart_cleanup_cluster_test.rs | 37 +++++++------------ 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index ef312d180..145ad317e 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -196,6 +196,9 @@ jobs: cache-save-if: 'false' install-build-packaging-tools: 'false' + - name: Verify protocol socket oracle + run: ss -tn state CLOSE-WAIT >/dev/null + # The suite owns fixed protocol ports and serializes its internal cases. - name: Verify protocol e2e membership env: diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index aa6f7d4a1..d2d5f0036 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo use aws_sdk_s3::primitives::ByteStream; use http::header::{CONTENT_TYPE, HOST}; use reqwest::StatusCode; +use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE}; use rustfs_signer::pre_sign_v4; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use s3s::Body; @@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box Result<(), Box Resul init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let bucket = "object-lambda-e2e-invalid-endpoint"; @@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let response = send_configure_webhook_target_request( &env, @@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box init_logging(); let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + cluster.set_env(ENV_NOTIFY_ENABLE, "true"); + cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1"); cluster.start().await?; let bucket = "listen-notification-cluster"; diff --git a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs index c5fc45e17..82d0924bc 100644 --- a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs +++ b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::CompletedMultipartUpload; use tokio::time::{Duration, sleep}; use tracing::info; use uuid::Uuid; @@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload( } } -async fn complete_reports_missing_upload( +async fn multipart_listing_reports_missing_upload( client: &aws_sdk_s3::Client, bucket: &str, key: &str, upload_id: &str, ) -> Result> { - let result = client - .complete_multipart_upload() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .multipart_upload(CompletedMultipartUpload::builder().build()) - .send() - .await; - match result { - Ok(_) => Ok(false), - Err(SdkError::ServiceError(err)) => { - let code = err.err().meta().code().unwrap_or(""); - if code == "NoSuchUpload" { - Ok(true) - } else { - Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into()) - } - } - Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()), - } + let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?; + + Ok(!result + .uploads() + .iter() + .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id))) } async fn wait_for_cleanup_on_all_nodes( @@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes( let mut all_cleaned = true; for (idx, client) in clients.iter().enumerate() { let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?; - let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?; - if !(list_parts_missing && complete_missing) { + let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?; + if !(list_parts_missing && listing_missing) { info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1); all_cleaned = false; break; @@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster() 1, "multipart upload should be visible before background cleanup" ); + assert!( + !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?, + "multipart upload listing should contain the upload before background cleanup" + ); wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?; From b91845c98cdf55e17170e8338bba772f81480e75 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:44:50 +0800 Subject: [PATCH 25/32] ci: align cache writer and reader keys (#6398) --- .github/workflows/cache-warm.yml | 9 ++++++--- scripts/security/check_cache_save_if.sh | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index a9902fc4d..660d96b09 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -94,6 +94,9 @@ concurrency: env: CARGO_TERM_COLOR: always + # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with + # ci.yml or the writer and readers use disjoint cache keys. + RUST_BACKTRACE: 1 jobs: # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary, @@ -101,7 +104,7 @@ jobs: warm-ci-dev: name: Warm ci-dev runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: @@ -191,7 +194,7 @@ jobs: warm-ci-feat-rio: name: Warm ci-feat-rio runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: @@ -219,7 +222,7 @@ jobs: warm-ci-feat-proto: name: Warm ci-feat-proto runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: diff --git a/scripts/security/check_cache_save_if.sh b/scripts/security/check_cache_save_if.sh index 3e63bdc8a..27abc6545 100755 --- a/scripts/security/check_cache_save_if.sh +++ b/scripts/security/check_cache_save_if.sh @@ -64,3 +64,25 @@ if [ "$status" -ne 0 ]; then fi echo "OK: every ./.github/actions/setup call states cache-save-if explicitly" + +# rust-cache hashes every CARGO*, CC*, CFLAGS*, CXX*, CMAKE*, and RUST* +# variable that is present when the setup action runs. The dedicated writer +# and the CI readers therefore need identical workflow-level compiler env. +compiler_env() { + awk ' + /^env:[[:space:]]*$/ { in_env = 1; next } + in_env && /^[^[:space:]]/ { exit } + in_env && /^ (CARGO|CC|CFLAGS|CXX|CMAKE|RUST)[A-Z0-9_]*:/ { print } + ' "$1" | sort +} + +ci_env="$(compiler_env .github/workflows/ci.yml)" +warm_env="$(compiler_env .github/workflows/cache-warm.yml)" + +if [ "$ci_env" != "$warm_env" ]; then + echo "CI and cache-warm compiler environments differ; rust-cache keys will not match:" >&2 + diff -u <(printf '%s\n' "$ci_env") <(printf '%s\n' "$warm_env") >&2 || true + exit 1 +fi + +echo "OK: cache-warm and CI compiler environments match" From 2d7120460b327fc2bb9bbd701e2fb1cb5cfacfba Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:45:15 +0800 Subject: [PATCH 26/32] test(e2e): remove fake KMS suite results (#6401) --- .config/e2e-full-selection.txt | 4 +- crates/e2e_test/src/kms/mod.rs | 3 - crates/e2e_test/src/kms/test_runner.rs | 499 ------------------------- docs/testing/e2e-suite-inventory.md | 4 +- 4 files changed, 4 insertions(+), 506 deletions(-) delete mode 100644 crates/e2e_test/src/kms/test_runner.rs diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index bb9022327..dfad0f0bd 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b -sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba +sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52 +sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6 diff --git a/crates/e2e_test/src/kms/mod.rs b/crates/e2e_test/src/kms/mod.rs index 5e6b9fe19..3b849fa1b 100644 --- a/crates/e2e_test/src/kms/mod.rs +++ b/crates/e2e_test/src/kms/mod.rs @@ -39,9 +39,6 @@ mod kms_edge_cases_test; #[cfg(test)] mod kms_fault_recovery_test; -#[cfg(test)] -mod test_runner; - #[cfg(test)] mod bucket_default_encryption_test; diff --git a/crates/e2e_test/src/kms/test_runner.rs b/crates/e2e_test/src/kms/test_runner.rs deleted file mode 100644 index 558c14631..000000000 --- a/crates/e2e_test/src/kms/test_runner.rs +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -#![allow(dead_code)] -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Unified KMS test suite runner -//! -//! This module provides a unified interface for running KMS tests with categorization, -//! filtering, and comprehensive reporting capabilities. - -use crate::common::init_logging; -use std::time::Instant; -use tokio::time::{Duration, sleep}; -use tracing::{debug, error, info, warn}; - -/// Test category for organization and filtering -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TestCategory { - CoreFunctionality, - MultipartEncryption, - EdgeCases, - FaultRecovery, - Comprehensive, - Performance, -} - -impl TestCategory { - pub fn as_str(&self) -> &'static str { - match self { - TestCategory::CoreFunctionality => "core-functionality", - TestCategory::MultipartEncryption => "multipart-encryption", - TestCategory::EdgeCases => "edge-cases", - TestCategory::FaultRecovery => "fault-recovery", - TestCategory::Comprehensive => "comprehensive", - TestCategory::Performance => "performance", - } - } -} - -/// Test definition with metadata -#[derive(Debug, Clone)] -pub struct TestDefinition { - pub name: String, - pub description: String, - pub category: TestCategory, - pub estimated_duration: Duration, - pub is_critical: bool, -} - -impl TestDefinition { - pub fn new( - name: impl Into, - description: impl Into, - category: TestCategory, - estimated_duration: Duration, - is_critical: bool, - ) -> Self { - Self { - name: name.into(), - description: description.into(), - category, - estimated_duration, - is_critical, - } - } -} - -/// Test execution result -#[derive(Debug, Clone)] -pub struct TestResult { - pub test_name: String, - pub category: TestCategory, - pub success: bool, - pub duration: Duration, - pub error_message: Option, -} - -impl TestResult { - pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self { - Self { - test_name, - category, - success: true, - duration, - error_message: None, - } - } - - pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self { - Self { - test_name, - category, - success: false, - duration, - error_message: Some(error), - } - } -} - -/// Comprehensive test suite configuration -#[derive(Debug, Clone)] -pub struct TestSuiteConfig { - pub categories: Vec, - pub include_critical_only: bool, - pub max_duration: Option, - pub parallel_execution: bool, -} - -impl Default for TestSuiteConfig { - fn default() -> Self { - Self { - categories: vec![ - TestCategory::CoreFunctionality, - TestCategory::MultipartEncryption, - TestCategory::EdgeCases, - TestCategory::FaultRecovery, - TestCategory::Comprehensive, - ], - include_critical_only: false, - max_duration: None, - parallel_execution: false, - } - } -} - -/// Unified KMS test suite runner -pub struct KMSTestSuite { - tests: Vec, - config: TestSuiteConfig, -} - -impl KMSTestSuite { - /// Create a new test suite with default configuration - pub fn new() -> Self { - let tests = vec![ - // Core Functionality Tests - TestDefinition::new( - "test_local_kms_end_to_end", - "End-to-end KMS test with all encryption types", - TestCategory::CoreFunctionality, - Duration::from_secs(60), - true, - ), - TestDefinition::new( - "test_local_kms_key_isolation", - "Test KMS key isolation and security", - TestCategory::CoreFunctionality, - Duration::from_secs(45), - true, - ), - // Multipart Encryption Tests - TestDefinition::new( - "test_local_kms_multipart_upload", - "Test large file multipart upload with encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(120), - true, - ), - TestDefinition::new( - "test_step1_basic_single_file_encryption", - "Basic single file encryption test", - TestCategory::MultipartEncryption, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_step2_basic_multipart_upload_without_encryption", - "Basic multipart upload without encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_step3_multipart_upload_with_sse_s3", - "Multipart upload with SSE-S3 encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(60), - true, - ), - TestDefinition::new( - "test_step4_large_multipart_upload_with_encryption", - "Large file multipart upload with encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(90), - false, - ), - TestDefinition::new( - "test_step5_all_encryption_types_multipart", - "All encryption types multipart test", - TestCategory::MultipartEncryption, - Duration::from_secs(120), - true, - ), - // Edge Cases Tests - TestDefinition::new( - "test_kms_zero_byte_file_encryption", - "Test encryption of zero-byte files", - TestCategory::EdgeCases, - Duration::from_secs(20), - false, - ), - TestDefinition::new( - "test_kms_single_byte_file_encryption", - "Test encryption of single-byte files", - TestCategory::EdgeCases, - Duration::from_secs(20), - false, - ), - TestDefinition::new( - "test_kms_multipart_boundary_conditions", - "Test multipart upload boundary conditions", - TestCategory::EdgeCases, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_kms_invalid_key_scenarios", - "Test invalid key scenarios", - TestCategory::EdgeCases, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_kms_concurrent_encryption", - "Test concurrent encryption operations", - TestCategory::EdgeCases, - Duration::from_secs(60), - false, - ), - TestDefinition::new( - "test_kms_key_validation_security", - "Test key validation security", - TestCategory::EdgeCases, - Duration::from_secs(30), - false, - ), - // Fault Recovery Tests - TestDefinition::new( - "test_kms_key_directory_unavailable", - "Test KMS when key directory is unavailable", - TestCategory::FaultRecovery, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_kms_corrupted_key_files", - "Test KMS with corrupted key files", - TestCategory::FaultRecovery, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_kms_multipart_upload_interruption", - "Test multipart upload interruption recovery", - TestCategory::FaultRecovery, - Duration::from_secs(60), - false, - ), - TestDefinition::new( - "test_kms_resource_constraints", - "Test KMS under resource constraints", - TestCategory::FaultRecovery, - Duration::from_secs(90), - false, - ), - // Comprehensive Tests - TestDefinition::new( - "test_comprehensive_kms_full_workflow", - "Full KMS workflow comprehensive test", - TestCategory::Comprehensive, - Duration::from_secs(300), - true, - ), - TestDefinition::new( - "test_comprehensive_stress_test", - "KMS stress test with large datasets", - TestCategory::Comprehensive, - Duration::from_secs(400), - false, - ), - TestDefinition::new( - "test_comprehensive_key_isolation", - "Comprehensive key isolation test", - TestCategory::Comprehensive, - Duration::from_secs(180), - false, - ), - TestDefinition::new( - "test_comprehensive_concurrent_operations", - "Comprehensive concurrent operations test", - TestCategory::Comprehensive, - Duration::from_secs(240), - false, - ), - TestDefinition::new( - "test_comprehensive_performance_benchmark", - "KMS performance benchmark test", - TestCategory::Comprehensive, - Duration::from_secs(360), - false, - ), - ]; - - Self { - tests, - config: TestSuiteConfig::default(), - } - } - - /// Configure the test suite - pub fn with_config(mut self, config: TestSuiteConfig) -> Self { - self.config = config; - self - } - - /// Filter tests based on category - pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> { - self.tests.iter().filter(|test| &test.category == category).collect() - } - - /// Filter tests based on criticality - pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> { - self.tests.iter().filter(|test| test.is_critical).collect() - } - - /// Get test summary by category - pub fn get_category_summary(&self) -> std::collections::HashMap> { - let mut summary = std::collections::HashMap::new(); - for test in &self.tests { - summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test); - } - summary - } - - /// Run the complete test suite - pub async fn run_test_suite(&self) -> Vec { - init_logging(); - info!("🚀 Starting unified KMS test suite"); - - let start_time = Instant::now(); - let mut results = Vec::new(); - - // Filter tests based on configuration - let tests_to_run: Vec<&TestDefinition> = self - .tests - .iter() - .filter(|test| self.config.categories.contains(&test.category)) - .filter(|test| !self.config.include_critical_only || test.is_critical) - .collect(); - - info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len()); - for (i, test) in tests_to_run.iter().enumerate() { - info!(" {}. {} ({})", i + 1, test.name, test.category.as_str()); - } - - // Execute tests - for (i, test_def) in tests_to_run.iter().enumerate() { - info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name); - info!(" 📝 Description: {}", test_def.description); - info!(" 🏷️ Category: {}", test_def.category.as_str()); - info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration); - - let test_start = Instant::now(); - let result = self.run_single_test(test_def).await; - let test_duration = test_start.elapsed(); - - match result { - Ok(_) => { - info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64()); - results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration)); - } - Err(e) => { - error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e); - results.push(TestResult::failure( - test_def.name.clone(), - test_def.category.clone(), - test_duration, - e.to_string(), - )); - } - } - - // Add delay between tests to avoid resource conflicts - if i < tests_to_run.len() - 1 { - debug!("⏸️ Waiting two seconds before the next test..."); - sleep(Duration::from_secs(2)).await; - } - } - - let total_duration = start_time.elapsed(); - self.print_test_summary(&results, total_duration); - - results - } - - /// Run a single test by dispatching to the appropriate test function - async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box> { - // This is a placeholder for test dispatch logic - // In a real implementation, this would dispatch to actual test functions - warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name); - Ok(()) - } - - /// Print comprehensive test summary - fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) { - info!("📊 KMS test suite summary"); - info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64()); - info!("📈 Total tests: {}", results.len()); - - let passed = results.iter().filter(|r| r.success).count(); - let failed = results.iter().filter(|r| !r.success).count(); - - info!("✅ Passed: {}", passed); - info!("❌ Failed: {}", failed); - info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0); - - // Summary by category - let mut category_summary: std::collections::HashMap = std::collections::HashMap::new(); - for result in results { - let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0)); - *total += 1; - if result.success { - *passed_count += 1; - } - } - - info!("📊 Category summary:"); - for (category, (total, passed_count)) in category_summary { - info!( - " 🏷️ {}: {}/{} ({:.1}%)", - category.as_str(), - passed_count, - total, - (passed_count as f64 / total as f64) * 100.0 - ); - } - - // List failed tests - if failed > 0 { - warn!("❌ Failing tests:"); - for result in results.iter().filter(|r| !r.success) { - warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error")); - } - } - } -} - -/// Quick test suite for critical tests only -#[tokio::test] -async fn test_kms_critical_suite() -> Result<(), Box> { - let config = TestSuiteConfig { - categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption], - include_critical_only: true, - max_duration: Some(Duration::from_secs(600)), // 10 minutes max - parallel_execution: false, - }; - - let suite = KMSTestSuite::new().with_config(config); - let results = suite.run_test_suite().await; - - let failed_count = results.iter().filter(|r| !r.success).count(); - if failed_count > 0 { - return Err(format!("Critical test suite failed: {failed_count} tests failed").into()); - } - - info!("✅ All critical tests passed"); - Ok(()) -} - -/// Full comprehensive test suite -#[tokio::test] -async fn test_kms_full_suite() -> Result<(), Box> { - let suite = KMSTestSuite::new(); - let results = suite.run_test_suite().await; - - let total_tests = results.len(); - let failed_count = results.iter().filter(|r| !r.success).count(); - let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0; - - info!("📊 Full suite success rate: {:.1}%", success_rate); - - // Allow up to 10% failure rate for non-critical tests - if success_rate < 90.0 { - return Err(format!("Test suite success rate too low: {success_rate:.1}%").into()); - } - - info!("✅ Full test suite succeeded"); - Ok(()) -} diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 7d80a802d..df2fc0847 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -58,7 +58,7 @@ | heal_erasure_disk_rebuild_test | 4 | 🌙 | | inline_fast_path_cluster_test | 16 | | | internode_rpc_signature_e2e_test | 5 | | -| kms | 48 | | +| kms | 46 | | | leading_slash_key_test | 2 | ✅ | | lifecycle_regression_test | 4 | | | list_buckets_auth_test | 1 | ✅ | @@ -99,4 +99,4 @@ | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -**Total listed: 577 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 455 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-21. +**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23. From f7003dfdddc4165006f467a854479383b911503c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 04:24:04 +0800 Subject: [PATCH 27/32] fix(admin): four site-replication interop correctness fixes (B5-rc T2) (#6399) * fix(admin): send versioningEnabled on site replication make-bucket ops The outbound make-with-versioning bucket-op query only carried operation/createdAt/lockEnabled. MinIO's own create-bucket hook sends versioningEnabled=true on this op, so align the outbound query with MinIO's site-replication make-bucket wire contract. Route both outbound builders (bootstrap plan and create-bucket hook) through one shared builder that always appends versioningEnabled=true. RustFS's own inbound handler force-enables versioning either way, so RustFS-to-RustFS behavior is unchanged; the MinIO release verified against (RELEASE.2025-09-07) also force-enables versioning regardless of the flag, so this aligns the wire contract rather than changing observable behavior there. * fix(admin): propagate purge-deleted-bucket errors in site replication The purge-deleted-bucket branch of the peer bucket-ops handler dropped the delete_bucket error and answered 200, so a peer-driven purge that failed (disk full, quorum loss) was reported as success while the bucket survived on this site. Tolerate only bucket-not-found (the purge raced an earlier replay or a local delete) and propagate every other error through ApiError like the sibling delete branches do. * fix(admin): derive fallback site deployment ID with UUIDv5 deployment_id_for_endpoint used DefaultHasher, whose algorithm is not guaranteed stable across Rust releases. The fallback fires when a peer response carries an empty deploymentID; the result is persisted in site-replication state, used for collision disambiguation, and broadcast to peers, so a toolchain bump could re-derive a different ID for the same endpoint. Note that the add preflight currently rejects that case upstream of this fallback. Derive UUIDv5 (NAMESPACE_URL) over the canonical endpoint instead, and log a structured warn when a peer metainfo response arrives without a deploymentID. Already persisted fallback IDs are non-empty and therefore never re-derived, so existing state is unaffected. * fix(admin): stream site replication devnull body without 1MB cap The site-replication devnull endpoint buffered the request body through read_plain_admin_body, which enforces the 1MB admin body cap. MinIO peers stream multi-megabyte probe bodies to this endpoint during site netperf link checks and expect an unbounded discard, so any larger probe got a 400 and was misreported as a broken link. Stream and discard the body chunk by chunk with no size cap instead, mirroring MinIO's io.Discard drain. The response stays 204 with an empty body. --- Cargo.lock | 1 + rustfs/Cargo.toml | 4 +- rustfs/src/admin/handlers/site_replication.rs | 153 ++++++++++++++---- rustfs/src/admin/site_replication_identity.rs | 29 +++- rustfs/src/admin/storage_api.rs | 3 +- 5 files changed, 150 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 637202fd5..b7b1ec220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12688,6 +12688,7 @@ dependencies = [ "js-sys", "rand 0.10.2", "serde_core", + "sha1_smol", "wasm-bindgen", ] diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 28f4ad259..af5514870 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -322,7 +322,7 @@ thiserror = { workspace = true } tracing.workspace = true url = { workspace = true } urlencoding = { workspace = true } -uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } +uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] } zip = { workspace = true } libc = { workspace = true } rand = { workspace = true, features = ["serde"] } @@ -345,7 +345,7 @@ libsystemd.workspace = true libmimalloc-sys.workspace = true [dev-dependencies] -uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } +uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] } serial_test = { workspace = true } tempfile = { workspace = true } aws-config = { workspace = true } diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 1bb0379df..23c8b680b 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -41,7 +41,7 @@ use crate::admin::storage_api::config::save_admin_config; use crate::admin::storage_api::contract::bucket::{ BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, }; -use crate::admin::storage_api::error::Error as StorageError; +use crate::admin::storage_api::error::{Error as StorageError, is_err_bucket_not_found}; use crate::admin::storage_api::runtime::ECStore; use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; use crate::auth::constant_time_eq; @@ -55,6 +55,7 @@ use crate::storage::storage_api::{ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use futures::StreamExt; use hmac::{Hmac, Mac}; use http::header::{CONTENT_TYPE, HOST}; use http::{HeaderMap, HeaderValue, Uri}; @@ -2096,6 +2097,18 @@ async fn remote_add_preflight_info(site: &PeerSite) -> S3Result Option { query_pairs(uri).get("bootstrapToken").cloned() } -fn bootstrap_bucket_make_op_path(bucket: &SRBucketInfo) -> String { +/// Query for a peer `make-with-versioning` bucket op. `versioningEnabled` +/// always travels so the outbound query matches MinIO's site-replication +/// make-bucket wire contract: MinIO's own create-bucket hook sends +/// `versioningEnabled=true` on this op. RustFS's inbound handler +/// force-enables versioning either way. +fn make_with_versioning_bucket_op_path(bucket: &str, created_at: Option<&str>, lock_enabled: bool) -> String { let mut query = form_urlencoded::Serializer::new(String::new()); - query.append_pair("bucket", &bucket.bucket); - query.append_pair("operation", "make-with-versioning"); - if let Some(created_at) = bucket - .created_at - .and_then(|value| value.format(&time::format_description::well_known::Rfc3339).ok()) - { - query.append_pair("createdAt", &created_at); + query.append_pair("bucket", bucket); + query.append_pair("operation", SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING); + query.append_pair("versioningEnabled", "true"); + if let Some(created_at) = created_at { + query.append_pair("createdAt", created_at); } - if bucket.object_lock_config.is_some() { + if lock_enabled { query.append_pair("lockEnabled", "true"); } - format!("/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", query.finish()) + format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?{}", query.finish()) +} + +fn bootstrap_bucket_make_op_path(bucket: &SRBucketInfo) -> String { + let created_at = bucket + .created_at + .and_then(|value| value.format(&time::format_description::well_known::Rfc3339).ok()); + make_with_versioning_bucket_op_path(&bucket.bucket, created_at.as_deref(), bucket.object_lock_config.is_some()) } fn bootstrap_bucket_meta_item(bucket: &SRBucketInfo, item_type: &str, updated_at: Option) -> SRBucketMeta { @@ -4246,16 +4269,7 @@ async fn broadcast_site_replication_make_bucket( .format(&time::format_description::well_known::Rfc3339) .unwrap_or_default(); - let path = { - let mut query = form_urlencoded::Serializer::new(String::new()); - query.append_pair("bucket", bucket); - query.append_pair("operation", "make-with-versioning"); - query.append_pair("createdAt", &created_at); - if lock_enabled { - query.append_pair("lockEnabled", "true"); - } - format!("/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", query.finish()) - }; + let path = make_with_versioning_bucket_op_path(bucket, Some(&created_at), lock_enabled); let path = if let Some(token) = bootstrap_token { with_site_replication_bootstrap_token(&path, token) } else { @@ -10206,13 +10220,25 @@ impl Operation for SiteReplicationStatusHandler { } } +/// `POST /v3/site-replication/devnull` — peer link-check upload drain. +/// MinIO streams multi-megabyte probe bodies here during site netperf link +/// checks and expects an unbounded discard (its handler copies to io.Discard); +/// buffering through the 1MB admin body cap turned any larger probe into a +/// 400 and a false link failure. Stream and discard instead — no size cap. +async fn drain_site_replication_devnull(mut input: Body) -> S3Result<()> { + while let Some(chunk) = input.next().await { + chunk.map_err(|e| s3_error!(InvalidRequest, "failed to read devnull stream: {}", e))?; + } + Ok(()) +} + pub struct SiteReplicationDevNullHandler {} #[async_trait::async_trait] impl Operation for SiteReplicationDevNullHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; - let _ = read_plain_admin_body(req.input).await?; + drain_site_replication_devnull(req.input).await?; Ok(empty_response(StatusCode::NO_CONTENT)) } } @@ -10471,6 +10497,19 @@ impl Operation for SRPeerJoinHandler { } } +/// Outcome of a peer-driven `purge-deleted-bucket` replay. A bucket that is +/// already gone means the purge raced an earlier replay or a local delete — +/// that is success — but any other failure must reach the sender like the +/// sibling delete branches do: swallowing it answered 200 while the bucket +/// survived on this site. +fn purge_deleted_bucket_result(result: Result<(), StorageError>) -> S3Result<()> { + match result { + Ok(()) => Ok(()), + Err(err) if is_err_bucket_not_found(&err) => Ok(()), + Err(err) => Err(ApiError::from(err).into()), + } +} + pub struct SRPeerBucketOpsHandler {} #[async_trait::async_trait] @@ -10570,16 +10609,18 @@ impl Operation for SRPeerBucketOpsHandler { .map_err(ApiError::from)?; } "purge-deleted-bucket" => { - let _ = store - .delete_bucket( - &bucket, - &DeleteBucketOptions { - force: true, - srdelete_op: SRBucketDeleteOp::Purge, - ..Default::default() - }, - ) - .await; + purge_deleted_bucket_result( + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + srdelete_op: SRBucketDeleteOp::Purge, + ..Default::default() + }, + ) + .await, + )?; } _ => return Err(s3_error!(InvalidRequest, "unsupported site replication bucket operation")), } @@ -13925,6 +13966,54 @@ mod tests { assert!(!query_flag(&uri, "missing")); } + /// A5 red-light: a `purge-deleted-bucket` replay must report success when + /// the bucket is already gone, and must propagate every other failure — + /// the swallowed error answered 200 while the bucket survived. + #[test] + fn test_purge_deleted_bucket_result_tolerates_only_missing_bucket() { + assert!(purge_deleted_bucket_result(Ok(())).is_ok()); + assert!(purge_deleted_bucket_result(Err(StorageError::BucketNotFound("photos".to_string()))).is_ok()); + assert!(purge_deleted_bucket_result(Err(StorageError::VolumeNotFound)).is_ok()); + let err = purge_deleted_bucket_result(Err(StorageError::StorageFull)) + .expect_err("non-not-found delete failures must propagate"); + assert_ne!(*err.code(), S3ErrorCode::NoSuchBucket); + } + + /// C5 red-light: the site-replication devnull drain must accept bodies + /// beyond the 1MB admin body cap — MinIO's link check streams large + /// probe bodies and treats a 400 as a broken link. + #[tokio::test] + async fn test_site_replication_devnull_drains_body_beyond_admin_cap() { + let body = Body::from(vec![0u8; MAX_ADMIN_REQUEST_BODY_SIZE + 1]); + drain_site_replication_devnull(body) + .await + .expect("devnull must drain bodies larger than the admin body cap"); + } + + /// A3 red-light: `versioningEnabled` must travel on every outbound + /// make-with-versioning bucket op so the query matches MinIO's + /// site-replication make-bucket wire contract (MinIO's own hook sends + /// `versioningEnabled=true` on this op). + #[test] + fn test_make_with_versioning_op_paths_send_versioning_enabled() { + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + created_at: Some(OffsetDateTime::UNIX_EPOCH), + object_lock_config: Some(BASE64_STANDARD.encode("")), + ..Default::default() + }; + let bootstrap = bootstrap_bucket_make_op_path(&bucket); + assert!(bootstrap.contains("operation=make-with-versioning"), "{bootstrap}"); + assert!(bootstrap.contains("versioningEnabled=true"), "{bootstrap}"); + assert!(bootstrap.contains("createdAt="), "{bootstrap}"); + assert!(bootstrap.contains("lockEnabled=true"), "{bootstrap}"); + + // The broadcast path (create-bucket hook) shares the same builder. + let broadcast = make_with_versioning_bucket_op_path("photos", Some("1970-01-01T00:00:00Z"), false); + assert!(broadcast.contains("versioningEnabled=true"), "{broadcast}"); + assert!(!broadcast.contains("lockEnabled"), "{broadcast}"); + } + #[tokio::test] #[serial] async fn test_add_bootstrap_scope_only_allows_expected_bucket_setup_until_guard_drops() { diff --git a/rustfs/src/admin/site_replication_identity.rs b/rustfs/src/admin/site_replication_identity.rs index dc6440b4d..24784160b 100644 --- a/rustfs/src/admin/site_replication_identity.rs +++ b/rustfs/src/admin/site_replication_identity.rs @@ -13,9 +13,9 @@ // limitations under the License. use rustfs_madmin::{PeerInfo, SyncStatus}; -use std::collections::{BTreeMap, hash_map::DefaultHasher}; -use std::hash::{Hash, Hasher}; +use std::collections::BTreeMap; use url::Url; +use uuid::Uuid; fn has_http_scheme(endpoint: &str) -> bool { endpoint.get(..7).is_some_and(|prefix| prefix.eq_ignore_ascii_case("http://")) @@ -66,10 +66,12 @@ pub fn site_identity_key(endpoint: &str) -> String { .unwrap_or_else(|| trimmed.to_ascii_lowercase()) } +/// Fallback deployment ID for a peer that reported none. UUIDv5 over the +/// canonical endpoint: the ID is persisted in site-replication state and +/// broadcast to peers, so it must be identical across Rust toolchains +/// (`DefaultHasher` is not) and across spellings of the same endpoint. pub fn deployment_id_for_endpoint(endpoint: &str) -> String { - let mut hasher = DefaultHasher::new(); - endpoint.hash(&mut hasher); - format!("{:016x}", hasher.finish()) + Uuid::new_v5(&Uuid::NAMESPACE_URL, canonical_endpoint(endpoint).as_bytes()).to_string() } pub fn same_identity_endpoint(left: &str, right: &str) -> bool { @@ -174,6 +176,23 @@ mod tests { } } + /// B8 red-light: the fallback deployment ID must be a toolchain-stable + /// UUIDv5 over the canonical endpoint — `DefaultHasher` output is not + /// guaranteed stable across Rust releases, yet the ID is persisted in + /// site-replication state and broadcast to peers. + #[test] + fn deployment_id_for_endpoint_is_stable_uuid_v5_over_canonical_endpoint() { + let endpoint = "https://node-a.example.com:9000"; + let id = deployment_id_for_endpoint(endpoint); + let parsed = uuid::Uuid::parse_str(&id).expect("fallback deployment ID must be a UUID"); + assert_eq!(parsed.get_version_num(), 5, "fallback deployment ID must be UUIDv5"); + // Deterministic for the same endpoint and for spelling variants that + // share a canonical form; distinct endpoints stay distinct. + assert_eq!(id, deployment_id_for_endpoint(endpoint)); + assert_eq!(id, deployment_id_for_endpoint(" HTTPS://Node-A.Example.Com:9000/ ")); + assert_ne!(id, deployment_id_for_endpoint("https://node-b.example.com:9000")); + } + #[test] fn canonical_endpoint_accepts_case_insensitive_scheme() { assert_eq!( diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 318718b49..bdf28f15e 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -51,7 +51,7 @@ mod ecstore_disk { } mod ecstore_error { - pub(crate) use crate::storage::storage_api::ecstore_error::StorageError; + pub(crate) use crate::storage::storage_api::ecstore_error::{StorageError, is_err_bucket_not_found}; } #[allow(unused_imports)] @@ -919,6 +919,7 @@ pub(crate) mod contract { } pub(crate) mod error { + pub(crate) use super::ecstore_error::is_err_bucket_not_found; pub(crate) use super::{Error, StorageError}; } From 20d1266496ba50dbec09b2263c504416840c61f7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 04:24:48 +0800 Subject: [PATCH 28/32] fix(heal): fence format repair during pool transitions (#6342) * fix(heal): fence format repair during pool transitions * fix(heal): fence format writes during transitions --- crates/ecstore/src/core/pools.rs | 2 +- crates/ecstore/src/core/sets.rs | 28 +- crates/ecstore/src/store/heal.rs | 335 +++++++++++++++++- crates/heal/src/heal/task/heal_erasure_set.rs | 11 +- crates/heal/src/heal/task/tests.rs | 28 ++ crates/test-utils/src/lib.rs | 12 + 6 files changed, 402 insertions(+), 14 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 9d8e0d01a..4ca460c20 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -2026,7 +2026,7 @@ impl PoolMeta { self.load_no_lock(pool).await } - async fn load_no_lock(&mut self, pool: Arc) -> Result<()> + pub(crate) async fn load_no_lock(&mut self, pool: Arc) -> Result<()> where S: EcstoreObjectIO, { diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index d9b354a08..0c79c080f 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -988,14 +988,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets { } } -#[async_trait::async_trait] -impl crate::storage_api_contracts::heal::HealOperations for Sets { - type Error = Error; - type HealResultItem = HealResultItem; - type HealOptions = HealOpts; - - #[tracing::instrument(skip(self))] - async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { +impl Sets { + pub(crate) async fn heal_format_with_fence(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option)> + where + F: Fn() -> bool + Send + Sync, + { let (disks, init_errs) = init_storage_disks_with_errors( &self.endpoints.endpoints, &DiskOption { @@ -1068,6 +1065,9 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { // Save new formats `format.json` on unformatted disks. for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() { if fm.is_some() && disk.is_some() { + if fence_lost() { + return Ok((res, Some(StorageError::SlowDown))); + } if let Err(err) = save_format_file(disk, fm).await { if let Some(disk) = disk.as_ref() { let _ = disk.close().await; @@ -1101,6 +1101,18 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { } Ok((res, None)) } +} + +#[async_trait::async_trait] +impl crate::storage_api_contracts::heal::HealOperations for Sets { + type Error = Error; + type HealResultItem = HealResultItem; + type HealOptions = HealOpts; + + #[tracing::instrument(skip(self))] + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + self.heal_format_with_fence(dry_run, || false).await + } #[tracing::instrument(skip(self))] async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { let mut result = HealResultItem { diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index ffac77751..a7aed758d 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -13,7 +13,12 @@ // limitations under the License. use super::*; +use crate::core::pools::POOL_META_NAME; +use crate::services::rebalance::{REBAL_META_NAME, RebalStatus}; +use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::heal::HealOperations as _; +use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use rustfs_lock::NamespaceLockGuard; use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error { ) } +#[derive(Debug, Clone, Copy)] +enum HealFormatPoolSkip { + Completed, + Retryable, +} + +fn classify_heal_format_pool( + pool_idx: usize, + pool_cmd_line: &str, + pool_meta: &PoolMeta, + rebalance_meta: Option<&RebalanceMeta>, +) -> Option { + let Some(pool) = pool_meta.pools.get(pool_idx) else { + return Some(HealFormatPoolSkip::Retryable); + }; + + if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line { + return Some(HealFormatPoolSkip::Retryable); + } + + if let Some(decommission) = pool.decommission.as_ref() { + if decommission.complete { + return Some(HealFormatPoolSkip::Completed); + } + if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) { + return Some(HealFormatPoolSkip::Retryable); + } + } + + if let Some(meta) = rebalance_meta { + let Some(pool_stats) = meta.pool_stats.get(pool_idx) else { + return Some(HealFormatPoolSkip::Retryable); + }; + if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) { + return Some(HealFormatPoolSkip::Retryable); + } + } + + None +} + +fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error { + match skip { + HealFormatPoolSkip::Completed => StorageError::NoHealRequired, + HealFormatPoolSkip::Retryable => StorageError::SlowDown, + } +} + +fn heal_format_fence_lost_error() -> Error { + StorageError::SlowDown +} + impl ECStore { + async fn acquire_heal_format_fence( + &self, + ) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option)> { + let metadata_pool = self + .pools + .first() + .cloned() + .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?; + + // Metadata fence order is part of the decommission/rebalance protocol: + // pool.bin must always be acquired before rebalance.bin. + let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; + let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?; + let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?; + + if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { + return Err(heal_format_fence_lost_error()); + } + + let mut pool_meta = PoolMeta::default(); + pool_meta.load_no_lock(metadata_pool.clone()).await?; + if pool_meta.pools.len() != self.pools.len() + || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| { + pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line + }) + { + return Err(heal_format_fence_lost_error()); + } + + let mut rebalance_meta = RebalanceMeta::new(); + let rebalance_meta = match rebalance_meta + .load_with_opts( + metadata_pool, + ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(()) => Some(rebalance_meta), + Err(Error::ConfigNotFound) => None, + Err(err) => return Err(err), + }; + + if rebalance_meta + .as_ref() + .is_some_and(|meta| meta.pool_stats.len() != self.pools.len()) + { + return Err(heal_format_fence_lost_error()); + } + + if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { + return Err(heal_format_fence_lost_error()); + } + + Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta)) + } + fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result>> { match opts.pool { Some(pool_idx) => Ok(vec![ @@ -52,9 +169,26 @@ impl ECStore { }; let mut count_no_heal = 0; + let mut count_completed = 0; let mut first_error = None; - for pool in self.pools.iter() { - let (mut result, err) = pool.heal_format(dry_run).await?; + for (pool_idx, pool) in self.pools.iter().enumerate() { + let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?; + if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { + first_error.get_or_insert(heal_format_fence_lost_error()); + break; + } + if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) + { + if matches!(skip, HealFormatPoolSkip::Completed) { + count_completed += 1; + } else { + first_error.get_or_insert(heal_format_pool_skip_error(skip)); + } + continue; + } + + let fence_lost = || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost(); + let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?; if let Some(err) = err { match err { StorageError::NoHealRequired => { @@ -69,11 +203,18 @@ impl ECStore { r.set_count += result.set_count; r.before.drives.append(&mut result.before.drives); r.after.drives.append(&mut result.after.drives); + + // A lease can be lost after the final write; fail closed before + // reporting the pool as successfully healed. + if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { + first_error.get_or_insert(heal_format_fence_lost_error()); + break; + } } if let Some(err) = first_error { return Ok((r, Some(err))); } - if count_no_heal == self.pools.len() { + if count_no_heal + count_completed == self.pools.len() { info!( event = EVENT_HEAL_FORMAT_COMPLETED, component = LOG_COMPONENT_ECSTORE, @@ -302,6 +443,7 @@ mod tests { use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::runtime::instance::InstanceContext; + use crate::services::rebalance::{RebalanceInfo, RebalanceStats}; use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations}; use crate::store::init_format::{load_format_erasure, save_format_file}; @@ -353,6 +495,164 @@ mod tests { } } + fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta { + PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(info), + }], + ..Default::default() + } + } + + #[test] + fn heal_format_pool_state_barriers_are_classified() { + let active = pool_meta_with_decommission(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }); + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &active, None), + Some(HealFormatPoolSkip::Retryable) + )); + + for info in [ + PoolDecommissionInfo { + failed: true, + ..Default::default() + }, + PoolDecommissionInfo { + canceled: true, + ..Default::default() + }, + ] { + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None), + Some(HealFormatPoolSkip::Retryable) + )); + } + + let completed = pool_meta_with_decommission(PoolDecommissionInfo { + complete: true, + ..Default::default() + }); + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &completed, None), + Some(HealFormatPoolSkip::Completed) + )); + } + + #[test] + fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() { + let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default()); + let rebalance = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)), + Some(HealFormatPoolSkip::Retryable) + )); + + let stopping = RebalanceMeta { + pool_stats: vec![RebalanceStats { + info: RebalanceInfo { + stopping: true, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)), + Some(HealFormatPoolSkip::Retryable) + )); + + let identity = pool_meta_with_decommission(PoolDecommissionInfo::default()); + assert!(matches!( + classify_heal_format_pool(0, "pool-new", &identity, None), + Some(HealFormatPoolSkip::Retryable) + )); + + let identity_without_decommission = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + assert!(matches!( + classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None), + Some(HealFormatPoolSkip::Retryable) + )); + + assert!(matches!( + classify_heal_format_pool(0, "", &identity_meta, None), + Some(HealFormatPoolSkip::Retryable) + )); + + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None), + Some(HealFormatPoolSkip::Retryable) + )); + + let stopped = RebalanceMeta { + stopped_at: Some(OffsetDateTime::UNIX_EPOCH), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Stopped, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none()); + + let stopping_after_stop = RebalanceMeta { + stopped_at: Some(OffsetDateTime::UNIX_EPOCH), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + stopping: true, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + assert!(matches!( + classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)), + Some(HealFormatPoolSkip::Retryable) + )); + } + + #[test] + fn skipped_heal_format_pool_is_never_reported_as_success() { + assert!(matches!( + heal_format_pool_skip_error(HealFormatPoolSkip::Retryable), + StorageError::SlowDown + )); + assert!(matches!( + heal_format_pool_skip_error(HealFormatPoolSkip::Completed), + StorageError::NoHealRequired + )); + } + async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc, CancellationToken) { let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created"); let mut pool_endpoints = Vec::new(); @@ -889,6 +1189,18 @@ mod tests { bucket_fence_registry: std::sync::Arc::default(), }; + let err = store + .handle_heal_format(false) + .await + .expect_err("missing pool metadata must fail closed before format writes"); + assert!(matches!(err, StorageError::SlowDown)); + + let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default()); + pool_meta + .save(store.pools.clone()) + .await + .expect("pool metadata should be persisted before format heal"); + let (result, err) = store .handle_heal_format(false) .await @@ -902,5 +1214,22 @@ mod tests { .await .expect("the later pool should be healed despite the first pool error"); assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]); + + let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default()); + for status in &mut completed_meta.pools { + status.decommission = Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }); + } + completed_meta + .save(store.pools.clone()) + .await + .expect("completed pool metadata should be persisted"); + let (_, err) = store + .handle_heal_format(false) + .await + .expect("completed pools should be reported as a no-op"); + assert!(matches!(err, Some(StorageError::NoHealRequired))); } } diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index e0cfe5a90..7b25bcba2 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -231,6 +231,10 @@ impl HealTask { "Heal erasure set format repair skipped because no format heal was required" ); } else { + let error = e; + if error.is_recoverable_heal() { + return Err(error); + } error!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_RESULT, @@ -239,7 +243,7 @@ impl HealTask { task_id = %self.id, set_disk_id, result = "format_failed", - error = %e, + error = %error, "Heal erasure set failed" ); { @@ -247,7 +251,7 @@ impl HealTask { progress.update_progress(4, 4, 0, 0); } return Err(Error::TaskExecutionFailed { - message: format!("Failed to heal disk format for {set_disk_id}: {e}"), + message: format!("Failed to heal disk format for {set_disk_id}: {error}"), }); } } else { @@ -284,6 +288,9 @@ impl HealTask { Err(Error::TaskCancelled) => return Err(Error::TaskCancelled), Err(Error::TaskTimeout) => return Err(Error::TaskTimeout), Err(e) => { + if e.is_recoverable_heal() { + return Err(e); + } error!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_RESULT, diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 464ff3606..f2b442205 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -547,6 +547,7 @@ struct MockStorage { heal_object_outcome: Mutex>, heal_object_outcomes: Mutex>>, format_no_heal_required: Mutex, + format_error: Mutex>, global_format_calls: Mutex, replacement_format_calls: Mutex)>>, replacement_targets_ready: Mutex, @@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage { async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option)> { *self.global_format_calls.lock().unwrap() += 1; + if let Some(error) = self.format_error.lock().unwrap().take() { + return Err(error); + } let no_heal_required = *self.format_no_heal_required.lock().unwrap(); if no_heal_required { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired)))) @@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() { ); } +#[tokio::test] +async fn erasure_set_format_slowdown_is_propagated() { + let storage = Arc::new(MockStorage { + format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: Vec::new(), + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + let error = task + .execute() + .await + .expect_err("format SlowDown must remain recoverable for the task manager"); + + assert!(matches!(error, Error::Storage(EcstoreError::SlowDown))); +} + #[tokio::test] async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() { let temp = TempDir::new().expect("temporary directory should be created"); diff --git a/crates/test-utils/src/lib.rs b/crates/test-utils/src/lib.rs index 59b4f3e0c..0188c7064 100644 --- a/crates/test-utils/src/lib.rs +++ b/crates/test-utils/src/lib.rs @@ -245,6 +245,18 @@ impl TestECStoreEnvBuilder { .await .expect("build test ECStore"); + // The production bootstrap only persists pool.bin from the elected + // first cluster node. Test stores intentionally have no cluster + // election, but heal-format still requires that durable fence before + // it can write any disk format. Materialize the validated topology + // here so the shared fixture models a ready single-node store. + let mut pool_meta = ecstore.pool_meta.read().await.clone(); + pool_meta.dont_save = false; + pool_meta + .save(ecstore.pools.clone()) + .await + .expect("persist test pool metadata"); + if self.init_bucket_metadata { let buckets_list = ecstore .list_bucket(&BucketOptions { From 84eb5aebefcd5295e1408bc58db9bee357dc631a Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 12:07:20 +0800 Subject: [PATCH 29/32] fix(ecstore): remove inline write debug noise (#6408) * fix(ecstore): remove inline write debug noise Co-Authored-By: heihutu * fix(ecstore): satisfy warning-as-error lints Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/ecstore/src/set_disk/ops/object.rs | 15 +-------------- crates/ecstore/src/store/object.rs | 2 +- crates/ecstore/src/store/rebalance/support.rs | 2 +- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 8f07a7ff7..bb3c9fbf9 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -2124,26 +2124,13 @@ impl SetDisks { let put_object_size = known_put_object_storage_size(data.size()); let shard_file_size_raw = erasure.shard_file_size(put_object_size); - let is_inline_buffer = - storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned); + let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned); let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled(); let shard_file_size = shard_file_size_raw; let shard_size = erasure.shard_size(); let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size); let direct_inline_commit = matches!(write_path, SmallWritePath::Inline); - { - use std::io::Write; - let msg = format!( - "INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n", - bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size, - is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned - ); - if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") { - let _ = f.write_all(msg.as_bytes()); - } - let _ = std::io::stderr().write_all(msg.as_bytes()); - } rustfs_io_metrics::record_put_object_path(write_path.metric_label()); let writer_setup_stage_start = collect_stage_timing.then(Instant::now); let (mut writers, errors) = if direct_inline_commit { diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 79a8232f7..7a8653415 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -3194,7 +3194,7 @@ impl ECStore { // Default return value let mut del_objects = vec![DeletedObject::default(); objects.len()]; - let mut accounting = vec![None; objects.len()]; + let accounting = vec![None; objects.len()]; let mut del_errs = Vec::with_capacity(objects.len()); for _ in 0..objects.len() { diff --git a/crates/ecstore/src/store/rebalance/support.rs b/crates/ecstore/src/store/rebalance/support.rs index 6e4db41b8..434f9c29d 100644 --- a/crates/ecstore/src/store/rebalance/support.rs +++ b/crates/ecstore/src/store/rebalance/support.rs @@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates( .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time)) .collect::>(); - latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx)); + latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx)); let Some(winner) = latest_candidates.first() else { return Err(Error::ErasureReadQuorum); From 648d5166e24b7afac72f3dc6e36b836744941d72 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 12:07:25 +0800 Subject: [PATCH 30/32] feat(allocator): replace mimalloc/libmimalloc-sys with rustfs-mimalloc/rustfs-mimalloc-sys (#6404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the upstream xonatius/mimalloc_rust.git fork (mimalloc + libmimalloc-sys) with the published rustfs-mimalloc (v0.5.0) and rustfs-mimalloc-sys (v0.5.0) crates from crates.io. The new crates are based on mimalloc V3 (v3.5.0) and provide: - MiMalloc global allocator with safe API (collect, stats_json, process_info) - Heap management and arena operations (heap module) - Full FFI bindings to mimalloc V3 Changes: - Workspace deps: mimalloc + libmimalloc-sys (git) → rustfs-mimalloc + rustfs-mimalloc-sys (crates.io) - allocator_reclaim.rs: libmimalloc_sys::mi_collect → rustfs_mimalloc::MiMalloc::collect - memory_observability.rs: raw FFI mi_stats_get_json → MiMalloc::stats_json() - main.rs: heap ownership tests use Heap::contains() (V3 API) - deny.toml: remove xonatius/mimalloc_rust.git from allow-git Co-authored-by: heihutu --- Cargo.lock | 49 ++++++++++++-------------- Cargo.toml | 4 +-- deny.toml | 3 -- rustfs/Cargo.toml | 4 +-- rustfs/src/allocator_reclaim.rs | 8 +---- rustfs/src/main.rs | 18 +++++----- rustfs/src/memory_observability.rs | 55 +++++++++++------------------- 7 files changed, 56 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7b1ec220..517983884 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,9 +1858,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -2522,12 +2522,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "cty" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" - [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -5988,15 +5982,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" -dependencies = [ - "cc", - "cty", -] - [[package]] name = "libredox" version = "0.1.20" @@ -6397,14 +6382,6 @@ dependencies = [ "synstructure 0.13.2", ] -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.17" @@ -9162,13 +9139,11 @@ dependencies = [ "insta", "jiff", "libc", - "libmimalloc-sys", "libsystemd", "matchit 0.9.2", "md-5 0.11.0", "metrics", "metrics-util", - "mimalloc", "mime_guess", "opentelemetry", "opentelemetry_sdk", @@ -9204,6 +9179,8 @@ dependencies = [ "rustfs-lock", "rustfs-log-analyzer", "rustfs-madmin", + "rustfs-mimalloc", + "rustfs-mimalloc-sys", "rustfs-notify", "rustfs-object-capacity", "rustfs-object-data-cache", @@ -9875,6 +9852,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "rustfs-mimalloc" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6" +dependencies = [ + "rustfs-mimalloc-sys", +] + +[[package]] +name = "rustfs-mimalloc-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd" +dependencies = [ + "cc", +] + [[package]] name = "rustfs-notify" version = "1.0.0-rc.3" diff --git a/Cargo.toml b/Cargo.toml index 918af3cce..717a0c6e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,8 +350,8 @@ russh-sftp = "2.4.0" dav-server = "0.11.0" # Performance Analysis and Memory Profiling -mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" } -libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] } +rustfs-mimalloc = { version = "0.5.0" } +rustfs-mimalloc-sys = { version = "0.5.0" } hotpath = { version = "0.23.3", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/deny.toml b/deny.toml index 296b229e4..c6fb8facc 100644 --- a/deny.toml +++ b/deny.toml @@ -43,9 +43,6 @@ allow-git = [ # RustFS fork carrying presigned expiry and constant-time authentication fixes. # owner: rustfs-maintainers review: 2026-10 "https://github.com/rustfs/s3s.git", - # MiMalloc fork pinned for hotpath allocation counting support. - # owner: houseme review: 2026-10 - "https://github.com/xonatius/mimalloc_rust.git", ] [bans] diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index af5514870..03afb5628 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -336,13 +336,13 @@ opentelemetry = { workspace = true } tracing-opentelemetry = { workspace = true } # Data structures hashbrown = { workspace = true, features = ["serde", "rayon"] } -mimalloc = { workspace = true } +rustfs-mimalloc = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] libsystemd.workspace = true [target.'cfg(not(target_os = "windows"))'.dependencies] -libmimalloc-sys.workspace = true +rustfs-mimalloc-sys.workspace = true [dev-dependencies] uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] } diff --git a/rustfs/src/allocator_reclaim.rs b/rustfs/src/allocator_reclaim.rs index 0c31390c3..eba1a8948 100644 --- a/rustfs/src/allocator_reclaim.rs +++ b/rustfs/src/allocator_reclaim.rs @@ -369,14 +369,8 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat } #[cfg(not(target_os = "windows"))] -#[allow(unsafe_code)] fn collect_allocator_memory(force: bool) -> Result<(), String> { - // SAFETY: `mi_collect` is provided by the active global allocator backend - // on this target family. It is explicitly intended to reclaim retained - // pages/segments and does not require additional invariants from the caller. - unsafe { - libmimalloc_sys::mi_collect(force); - } + rustfs_mimalloc::MiMalloc::collect(force); Ok(()) } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 5e9d5075b..7f4bea1d2 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -26,22 +26,22 @@ struct MiMallocAllocator; unsafe impl GlobalAlloc for MiMallocAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { // SAFETY: the caller upholds GlobalAlloc's contract for layout. - unsafe { mimalloc::MiMalloc.alloc(layout) } + unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) } } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { // SAFETY: the caller upholds GlobalAlloc's contract for layout. - unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) } + unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) } } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { // SAFETY: ptr and layout came from this allocator and are forwarded unchanged. - unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) } + unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) } } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { // SAFETY: ptr and layout came from this allocator and are forwarded unchanged. - unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) } + unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) } } } @@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator = hotpath::Counting #[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))] #[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc; fn main() { let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build(); @@ -71,8 +71,9 @@ mod tests { allocation.extend_from_slice(&[7_u8; 64]); assert_eq!(allocation.len(), 64); + let heap = rustfs_mimalloc::heap::Heap::main(); // SAFETY: the live Vec pointer is valid to inspect for heap ownership. - assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) }); + assert!(unsafe { heap.contains(allocation.as_ptr()) }); } #[test] @@ -85,12 +86,13 @@ mod tests { let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout"); let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout"); let allocator = super::MiMallocAllocator; + let heap = rustfs_mimalloc::heap::Heap::main(); // SAFETY: The pointer is checked for null before use and later released // through the same allocator with the corresponding layout. let ptr = unsafe { allocator.alloc_zeroed(layout) }; assert!(!ptr.is_null()); - assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) }); + assert!(unsafe { heap.contains(ptr) }); assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) }); // SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure @@ -102,7 +104,7 @@ mod tests { panic!("mimalloc realloc failed in allocator smoke test"); } - assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) }); + assert!(unsafe { heap.contains(grown_ptr) }); // SAFETY: `grown_ptr` was reallocated by `allocator` and is released // with the matching grown layout. unsafe { allocator.dealloc(grown_ptr, grown_layout) }; diff --git a/rustfs/src/memory_observability.rs b/rustfs/src/memory_observability.rs index 3e29d24a0..12bd4b3c5 100644 --- a/rustfs/src/memory_observability.rs +++ b/rustfs/src/memory_observability.rs @@ -17,10 +17,7 @@ use rustfs_io_metrics::{ record_cpu_usage, record_memory_usage, record_process_memory_split, }; use serde::Serialize; -#[cfg(any(test, not(target_os = "windows")))] use serde_json::Value; -#[cfg(not(target_os = "windows"))] -use std::ffi::CStr; use std::path::Path; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; @@ -231,7 +228,18 @@ fn read_cgroup_memory_snapshot() -> Option { read_cgroup_v2().or_else(read_cgroup_v1) } -#[cfg(any(test, not(target_os = "windows")))] +fn read_allocator_memory_snapshot() -> Option { + let json = rustfs_mimalloc::MiMalloc::stats_json(); + if json.is_empty() { + return None; + } + let observation = parse_mimalloc_stats_json(&json)?; + Some(AllocatorMemorySnapshot { + backend: crate::allocator_reclaim::allocator_backend(), + observation, + }) +} + fn numeric_json_value(value: &Value) -> Option { match value { Value::Number(number) => number @@ -242,7 +250,6 @@ fn numeric_json_value(value: &Value) -> Option { } } -#[cfg(any(test, not(target_os = "windows")))] fn numeric_json_field(value: &Value, field: &str) -> Option { match value { Value::Object(fields) => fields @@ -254,7 +261,6 @@ fn numeric_json_field(value: &Value, field: &str) -> Option { } } -#[cfg(any(test, not(target_os = "windows")))] fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option { match value { Value::Object(fields) => { @@ -271,12 +277,10 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option } } -#[cfg(any(test, not(target_os = "windows")))] fn mimalloc_stat_current(value: &Value, metric: &str) -> Option { mimalloc_stat_field(value, metric, "current") } -#[cfg(any(test, not(target_os = "windows")))] fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option { metrics .iter() @@ -285,7 +289,6 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option 0) } -#[cfg(any(test, not(target_os = "windows")))] fn parse_mimalloc_stats_json(stats_json: &str) -> Option { let value = serde_json::from_str::(stats_json).ok()?; let malloc_metrics = ["malloc_normal", "malloc_huge"]; @@ -312,33 +315,6 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option Option { - // SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by - // mimalloc when called with a null input buffer. The mimalloc API requires - // freeing that buffer with `mi_free`; parsing finishes before the buffer is freed. - let observation = unsafe { - let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut()); - if stats_ptr.is_null() { - return None; - } - - let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json); - libmimalloc_sys::mi_free(stats_ptr.cast()); - observation? - }; - Some(AllocatorMemorySnapshot { - backend: crate::allocator_reclaim::allocator_backend(), - observation, - }) -} - -#[cfg(target_os = "windows")] -fn read_allocator_memory_snapshot() -> Option { - None -} - fn configured_memory_observability_interval_secs() -> u64 { rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1) } @@ -566,6 +542,13 @@ mod tests { assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None); } + #[test] + fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() { + let snapshot = super::read_allocator_memory_snapshot(); + #[cfg(not(target_os = "windows"))] + assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows"); + } + #[test] fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() { let snapshot = build_memory_observability_status_snapshot(false, 15, false); From b6ba89d9e4216deb52fd49a97079623dfaeceddf Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 12:09:06 +0800 Subject: [PATCH 31/32] docs(testing): document CI gate matrix (#6412) --- CLAUDE.md | 3 +- CONTRIBUTING.md | 2 + docs/testing/ci-gates.md | 149 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 docs/testing/ci-gates.md diff --git a/CLAUDE.md b/CLAUDE.md index 5e75fe3cb..59893c48b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04 - Crate membership: `Cargo.toml` `[workspace].members` - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md) - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md) -- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs) +- CI workflow steps: `.github/workflows/`; event, timeout, and required-status + matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md) - Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake policy: [docs/testing/README.md](docs/testing/README.md) - Tier/ILM transition debugging (xl.meta inspection, versionId tracing): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7882a486e..5842395f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,8 @@ make pre-pr > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md). +> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md). + ### 🔒 Automated Pre-commit Hooks #### What `make pre-commit` and `make pre-pr` actually run diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md new file mode 100644 index 000000000..d87ce1668 --- /dev/null +++ b/docs/testing/ci-gates.md @@ -0,0 +1,149 @@ +# CI gate matrix + +This file is the source of truth for which validation runs on each event, its +configured wall-clock budget, and whether it can block a merge. Test taxonomy, +naming, and nextest serialization rules remain in [README.md](README.md); e2e +membership and counts remain in +[e2e-suite-inventory.md](e2e-suite-inventory.md). + +The distinction between **required** and **report-only** is load-bearing: +a failing job blocks a merge only when its exact check name is present in the +live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check +does not make a job required by itself. + +## Required merge checks + +The live `main` ruleset (`6436880`) currently requires exactly these contexts: + +| Required context | Producer | Validation | +|---|---|---| +| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement | +| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts | +| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs | + +For pull requests limited to the paths excluded by the main CI workflow, +`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and +`Test and Lint` under the same names. It runs the real quick checks and the +planning-document guard; it does not claim that Rust compilation or runtime +tests ran. Despite the workflow name, these paths also include selected deploy, +workflow, and lock files. + +Verify the live rule rather than trusting this snapshot before changing merge +policy: + +```bash +gh api repos/rustfs/rustfs/rulesets/6436880 \ + --jq '.rules[] | select(.type == "required_status_checks") | .parameters' +``` + +The ruleset currently has `strict_required_status_checks_policy=false`. +`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for +them, but `End-to-End Tests (full merge gate)` is not currently a required +context. Therefore the repository is prepared to test a merge-queue SHA, but +the workflow alone does not prove that every merge passed that lane. + +## Pull request and merge matrix + +Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only” +means the result is visible and actionable but is not in the live required +context list. + +| Event | Validation | Budget | Merge status | Reproduction | +|---|---|---:|---|---| +| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) | +| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` | +| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` | +| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` | +| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set | +| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` | +| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` | +| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs ` | +| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` | +| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` | +| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | +| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` | +| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` | +| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed | +| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` | +| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` | +| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` | +| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib ` commands in `windows-filesystem.yml` on Windows | + +The authoritative e2e filters live in `.config/nextest.toml`; extend a profile +instead of adding a second ad-hoc selector. Before a profile runs, +`scripts/check_test_wiring.py` compares its exact membership to the committed +digest so a silent test drop fails closed. + +## Scheduled and manual validation + +Scheduled lanes are independent fault domains. They do not block a pull +request, but their workflow-local gate can fail the run and scheduled failures +are routed to the shared failure-issue action. The scheduled-validation +watchdog and freshness workflow separately detect incomplete runs and missing +schedules. + +| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction | +|---|---|---:|---|---| +| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | +| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps | +| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | +| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` | +| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` | +| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | +| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` | +| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` | +| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set | +| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target | +| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it | +| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` | +| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` | + +Manual `workflow_dispatch` exists for the scheduled workflows above. Manual +runs are debugging evidence and intentionally do not open scheduled-failure +issues. A manual performance run may explicitly allow a known regression; that +override must not be treated as an ordinary passing baseline. + +## Release validation + +Release validation is post-merge and tag-driven; it does not substitute for a +pull-request gate. + +| Event | Validation | Budget | Result | +|---|---|---:|---| +| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push | +| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step | +| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report | +| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release | +| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index | + +Use an exact preview tag for end-to-end release rehearsal. Manual dispatches +are backfill/debug paths and do not prove the automatic `workflow_run` chain. + +## Evidence requirements + +A green check is useful only when it proves the intended behavior ran: + +- Record the exact commit SHA and run URL. +- Separate product failure from runner prerequisites, service readiness, and + cancellation. Repair the precondition, then rerun the exact workload. +- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and + server logs where the workflow provides them. +- For a bug fix or a new fault checker, provide sensitivity evidence: the old + behavior or an intentional mutation must fail the new oracle, and the fixed + behavior must pass it. +- Never promote a report-only lane to required from one green run. Require at + least 14 days and 30 representative pull requests with at least 99% complete + execution, then update the ruleset and this table together. + +## Change checklist + +Update this file in the same pull request when any of these change: + +- workflow triggers, job names, timeouts, or nextest profile ownership; +- required status contexts or strict/merge-queue policy; +- scheduled cadence, alert routing, artifact contract, or local reproduction; +- report-only versus gating semantics. + +Do not copy per-module test counts here. Update +[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership +digest instead. From 5f7220944622712df4376f301fe4921a0e0e0e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 12:29:52 +0800 Subject: [PATCH 32/32] fix(ecstore): keep unknown-size sentinel in create_bitrot_writer (#6380) SSE and compression wrap the payload so its length is unknown and advertise HashReader::SIZE_PRESERVE_LAYER (-1). Every layer preserved that sentinel except create_bitrot_writer, which clamped it to 0 before calling DiskAPI::create_file. RemoteDisk forwards that size verbatim in the put_file_stream query, so remote peers were told the body was empty. Since the authenticated put-file trailer (#5868) the receiver used the declared size to split body from trailer, turning the clamp into a fatal "auth trailer has trailing data" failure for every SSE PUT on multi-node deployments (rc.2). #6320 relaxed the receiver to only trust size > 0; this change fixes the sender so the sentinel survives end to end and the wire no longer conflates empty objects with unknown-length streams. Refs #6331 --- crates/ecstore/src/io_support/bitrot.rs | 44 +++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 05c81c048..88a3da910 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -784,6 +784,24 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle( /// /// # Returns /// A Result containing the BitrotWriterWrapper or an error +/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard. +/// +/// A known length is grown by one checksum per shard so the on-disk file size +/// matches what the bitrot writer emits. A negative length is the +/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and +/// compression) and must be preserved: `RemoteDisk::create_file` forwards it +/// in the `put_file_stream` query, and the receiver only treats `size > 0` as +/// a fixed body length when locating the authenticated trailer. Clamping it +/// to `0` would claim an empty body and misframe the stream. `0` stays `0` +/// because a genuinely empty object still means an empty body. +fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 { + if length <= 0 { + return length; + } + let length = length as usize; + (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64 +} + pub async fn create_bitrot_writer( is_inline_buffer: bool, disk: Option<&DiskStore>, @@ -796,12 +814,7 @@ pub async fn create_bitrot_writer( let writer = if is_inline_buffer { CustomWriter::new_inline_buffer() } else if let Some(disk) = disk { - let length = if length > 0 { - let length = length as usize; - (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64 - } else { - 0 - }; + let length = bitrot_create_file_size(length, shard_size, &checksum_algo); let file = disk.create_file("", volume, path, length).await?; #[cfg(feature = "hotpath")] @@ -820,6 +833,25 @@ mod tests { use rustfs_rio::ChunkReader; use std::collections::VecDeque; + #[test] + fn bitrot_create_file_size_grows_known_length_by_checksums() { + // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash. + assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32); + assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10); + } + + #[test] + fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() { + assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0); + // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote + // put_file_stream receiver relies on a non-positive size to parse the auth + // trailer from the stream tail, so the sentinel must survive untouched. + assert_eq!( + bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256), + rustfs_rio::HashReader::SIZE_PRESERVE_LAYER + ); + } + struct TestChunkReader { chunks: VecDeque, }