From 2bc88418f5d05ca3a71d0f4fa1df767e56920682 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 12:06:34 +0800 Subject: [PATCH 01/19] fix(list): satisfy clippy in the source listing merge (#7117) --- rustfs/src/app/bucket_list_through.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index fe883d913..02c4c7420 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -336,7 +336,7 @@ async fn fetch_source_page( delimiter: params.delimiter.as_deref(), // S3 ignores start-after once a continuation token is present, so // the client's own start-after only applies to the first page. - start_after: token.is_none().then(|| params.start_after_for_query.as_deref()).flatten(), + start_after: token.is_none().then_some(params.start_after_for_query.as_deref()).flatten(), continuation_token: token, max_keys: params.max_keys, }, @@ -363,9 +363,11 @@ async fn fetch_source_page( SourceListPlan::Folded { common_prefix, .. } => { let exists = !page.objects.is_empty() || !page.common_prefixes.is_empty(); Ok(( - exists - .then(|| vec![SideEntry::Prefix(common_prefix.clone())]) - .unwrap_or_default(), + if exists { + vec![SideEntry::Prefix(common_prefix.clone())] + } else { + Vec::new() + }, false, None, )) From 5f739944a3db42635cb3554f69dfbb05e36b6ee0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 14:14:26 +0800 Subject: [PATCH 02/19] fix: resolve clippy warnings in bucket_list_through.rs (#7121) From fdb28e611f0358f0175a5308d6132ceafd04d186 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 16:12:01 +0800 Subject: [PATCH 03/19] fix: bound tier free-version recovery sweeps (#7125) --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 336 +++++++++++++++--- .../lifecycle/tier_free_version_recovery.rs | 80 ++++- docs/operations/tier-ilm-debugging.md | 10 + 3 files changed, 369 insertions(+), 57 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 1649552c6..d9e352e26 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -2777,7 +2777,8 @@ async fn run_tier_free_version_recovery_loop( match recovery_result { Ok(stats) => { let elapsed = started_at.elapsed(); - schedule.record_success(&stats, elapsed); + schedule.record_success(&stats); + rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0); let (pending_tasks, active_tasks) = { let state = expiry_state.read().await; (state.pending_tasks(), state.stats.active_tasks()) @@ -2806,7 +2807,7 @@ async fn run_tier_free_version_recovery_loop( } Err(err) => { let elapsed = started_at.elapsed(); - schedule.record_failure(elapsed); + schedule.record_failure(); rustfs_io_metrics::record_stage_duration( "lifecycle_free_version_recovery_failed", elapsed.as_secs_f64() * 1000.0, @@ -2838,10 +2839,10 @@ async fn wait_for_tier_free_version_recovery( } else { schedule.next_delay }; - let sleep_delay = next_delay.saturating_sub(schedule.previous_run_duration); - schedule.previous_run_duration = StdDuration::ZERO; + // Recovery delays are completion-relative. Discounting the previous run + // would let a page that took at least one interval restart with no cooldown. schedule.jitter_next_delay = false; - let sleep = tokio::time::sleep(sleep_delay); + let sleep = tokio::time::sleep(next_delay); tokio::pin!(sleep); let mut recovery_request_consumed = false; @@ -2891,7 +2892,6 @@ struct TierFreeVersionRecoverySchedule { next_delay: StdDuration, idle_interval: StdDuration, failure_interval: StdDuration, - previous_run_duration: StdDuration, jitter_next_delay: bool, bucket_marker: Option, object_marker: Option, @@ -2904,7 +2904,6 @@ impl Default for TierFreeVersionRecoverySchedule { next_delay: StdDuration::ZERO, idle_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, failure_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, - previous_run_duration: StdDuration::ZERO, jitter_next_delay: false, bucket_marker: None, object_marker: None, @@ -2917,7 +2916,6 @@ impl TierFreeVersionRecoverySchedule { fn reset_idle_interval(&mut self) { self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; - self.previous_run_duration = StdDuration::ZERO; self.jitter_next_delay = false; } @@ -2928,18 +2926,15 @@ impl TierFreeVersionRecoverySchedule { self.reset_idle_interval(); } - fn record_failure(&mut self, _run_duration: StdDuration) { + fn record_failure(&mut self) { self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; self.next_delay = self.failure_interval; self.failure_interval = std::cmp::min(self.failure_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL); - // Keep the full backoff even after a long failed run, so a run whose - // duration exceeds the interval cannot restart immediately. - self.previous_run_duration = StdDuration::ZERO; self.jitter_next_delay = false; } - fn record_success(&mut self, stats: &FreeVersionRecoveryStats, run_duration: StdDuration) { + fn record_success(&mut self, stats: &FreeVersionRecoveryStats) { self.failure_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; if stats.enqueued > 0 || stats.failed > 0 { self.follow_up_sweep = true; @@ -2949,7 +2944,6 @@ impl TierFreeVersionRecoverySchedule { self.object_marker = stats.next_object_marker.clone(); if stats.truncated { self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; - self.previous_run_duration = run_duration; self.jitter_next_delay = false; return; } @@ -2960,13 +2954,11 @@ impl TierFreeVersionRecoverySchedule { self.follow_up_sweep = false; self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL; - self.previous_run_duration = run_duration; self.jitter_next_delay = false; return; } self.next_delay = self.idle_interval; - self.previous_run_duration = run_duration; self.jitter_next_delay = true; self.idle_interval = std::cmp::min(self.idle_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL); } @@ -5623,8 +5615,9 @@ mod tests { use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType}; use crate::bucket::lifecycle::runtime_boundary as runtime_sources; use crate::bucket::lifecycle::tier_free_version_recovery::{ - FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel, - set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook, + FreeVersionRecoveryStats, RecoveryWalkTestAction, RecoveryWorkBudget, list_tier_free_versions, + list_tier_free_versions_with_budget, recover_tier_free_versions_with_cancel, set_recovery_bucket_list_wait_hook, + set_recovery_walk_test_hook, }; use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats; use crate::bucket::lifecycle::tier_sweeper::Jentry; @@ -5712,7 +5705,7 @@ mod tests { assert_eq!(schedule.next_delay, StdDuration::ZERO); for expected in [60, 120, 240, 480, 600, 600, 600, 600] { - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); assert_eq!(schedule.next_delay, StdDuration::from_secs(expected)); } assert_eq!(schedule.next_delay.as_secs() / TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL.as_secs(), 10); @@ -5723,15 +5716,13 @@ mod tests { let mut schedule = TierFreeVersionRecoverySchedule::default(); for expected in [60, 120, 240, 480, 600, 600] { - schedule.record_failure(StdDuration::from_secs(75)); + schedule.record_failure(); assert_eq!(schedule.next_delay, StdDuration::from_secs(expected)); - assert_eq!(schedule.previous_run_duration, StdDuration::ZERO); } - schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO); - schedule.record_failure(StdDuration::from_secs(75)); + schedule.record_success(&free_version_recovery_stats(0, 0, false)); + schedule.record_failure(); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); - assert_eq!(schedule.previous_run_duration, StdDuration::ZERO); } #[test] @@ -5746,22 +5737,22 @@ mod tests { fn tier_free_version_recovery_pagination_preserves_full_sweep_backoff() { let idle = free_version_recovery_stats(0, 0, false); let mut schedule = TierFreeVersionRecoverySchedule::default(); - schedule.record_success(&idle, StdDuration::ZERO); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); + schedule.record_success(&idle); assert_eq!(schedule.next_delay, StdDuration::from_secs(120)); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); - schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO); + schedule.record_success(&free_version_recovery_stats(0, 0, true)); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket")); assert_eq!(schedule.object_marker.as_deref(), Some("object")); - schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO); + schedule.record_success(&free_version_recovery_stats(0, 0, true)); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); assert_eq!(schedule.next_delay, StdDuration::from_secs(240)); assert_eq!(schedule.idle_interval, StdDuration::from_secs(480)); assert!(schedule.bucket_marker.is_none()); @@ -5771,7 +5762,7 @@ mod tests { #[test] fn tier_free_version_recovery_wake_during_pagination_keeps_one_full_follow_up() { let mut schedule = TierFreeVersionRecoverySchedule::default(); - schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO); + schedule.record_success(&free_version_recovery_stats(0, 0, true)); schedule.request_retry(); schedule.request_retry(); @@ -5779,7 +5770,7 @@ mod tests { assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket")); assert_eq!(schedule.object_marker.as_deref(), Some("object")); - schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO); + schedule.record_success(&free_version_recovery_stats(0, 0, false)); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert!(!schedule.follow_up_sweep); assert!(schedule.bucket_marker.is_none()); @@ -5794,22 +5785,22 @@ mod tests { free_version_recovery_stats(0, 1, true), ] { let mut schedule = TierFreeVersionRecoverySchedule::default(); - schedule.record_success(&idle, StdDuration::ZERO); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); + schedule.record_success(&idle); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); - schedule.record_success(&work, StdDuration::ZERO); + schedule.record_success(&work); assert!(schedule.follow_up_sweep); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); assert!(!schedule.jitter_next_delay); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); assert!(!schedule.follow_up_sweep); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert!(!schedule.jitter_next_delay); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, StdDuration::from_secs(120)); } @@ -5823,17 +5814,17 @@ mod tests { free_version_recovery_stats(0, 1, false), ] { let mut schedule = TierFreeVersionRecoverySchedule::default(); - schedule.record_success(&idle, StdDuration::ZERO); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); + schedule.record_success(&idle); assert_eq!(schedule.idle_interval, StdDuration::from_secs(240)); - schedule.record_success(&work, StdDuration::ZERO); + schedule.record_success(&work); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert!(!schedule.follow_up_sweep); assert!(!schedule.jitter_next_delay); - schedule.record_success(&idle, StdDuration::ZERO); + schedule.record_success(&idle); assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL); assert_eq!(schedule.idle_interval, StdDuration::from_secs(120)); } @@ -5972,7 +5963,7 @@ mod tests { ); } - async fn tier_free_version_recovery_page_call_times(run_duration: StdDuration) -> Vec { + async fn tier_free_version_recovery_page_call_times(run_durations: &[StdDuration]) -> Vec { RECOVERY_JITTER_CALLS.store(0, Ordering::SeqCst); let cancel = CancellationToken::new(); let state = ExpiryState::new(); @@ -5981,6 +5972,7 @@ mod tests { let recorded_call_times = Arc::clone(&call_times); let call_index = Arc::new(AtomicUsize::new(0)); let recorded_call_index = Arc::clone(&call_index); + let recovery_run_durations = run_durations.to_vec(); let loop_cancel = cancel.clone(); let recovery_cancel = cancel.clone(); let worker = tokio::spawn(async move { @@ -5991,8 +5983,9 @@ mod tests { .push(tokio::time::Instant::now().duration_since(started_at)); let index = recorded_call_index.fetch_add(1, Ordering::SeqCst); let cancel = recovery_cancel.clone(); + let run_duration = recovery_run_durations.get(index).copied(); async move { - if index == 0 { + if let Some(run_duration) = run_duration { tokio::time::sleep(run_duration).await; Ok(free_version_recovery_stats(0, 0, true)) } else { @@ -6005,15 +5998,14 @@ mod tests { }); tokio::task::yield_now().await; - tokio::time::advance(run_duration).await; - tokio::task::yield_now().await; - if run_duration < TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL { - let remaining = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - run_duration; - tokio::time::advance(remaining - StdDuration::from_secs(1)).await; - assert_eq!(call_index.load(Ordering::SeqCst), 1); + for (index, run_duration) in run_durations.iter().copied().enumerate() { + tokio::time::advance(run_duration).await; + tokio::task::yield_now().await; + tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - StdDuration::from_secs(1)).await; + assert_eq!(call_index.load(Ordering::SeqCst), index + 1); tokio::time::advance(StdDuration::from_secs(1)).await; + tokio::task::yield_now().await; } - tokio::task::yield_now().await; worker.await.expect("recovery loop should stop after cancellation"); assert_eq!( RECOVERY_JITTER_CALLS.load(Ordering::SeqCst), @@ -6029,14 +6021,75 @@ mod tests { #[tokio::test(start_paused = true)] #[serial] - async fn tier_free_version_recovery_preserves_start_to_start_page_cadence() { + async fn tier_free_version_recovery_waits_after_each_page_completes() { assert_eq!( - tier_free_version_recovery_page_call_times(StdDuration::from_secs(45)).await, - vec![StdDuration::ZERO, StdDuration::from_secs(60)] + tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(45)]).await, + vec![StdDuration::ZERO, StdDuration::from_secs(105)] ); assert_eq!( - tier_free_version_recovery_page_call_times(StdDuration::from_secs(75)).await, - vec![StdDuration::ZERO, StdDuration::from_secs(75)] + tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(60)]).await, + vec![StdDuration::ZERO, StdDuration::from_secs(120)] + ); + assert_eq!( + tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75)]).await, + vec![StdDuration::ZERO, StdDuration::from_secs(135)] + ); + assert_eq!( + tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75), StdDuration::from_secs(45)]).await, + vec![StdDuration::ZERO, StdDuration::from_secs(135), StdDuration::from_secs(240)] + ); + } + + #[tokio::test(start_paused = true)] + async fn tier_free_version_recovery_notify_during_page_keeps_completion_cooldown() { + let cancel = CancellationToken::new(); + let state = ExpiryState::new(); + let recovery_notify = Arc::clone(&state.read().await.recovery_notify); + let started_at = tokio::time::Instant::now(); + let call_times = Arc::new(StdMutex::new(Vec::new())); + let recorded_call_times = Arc::clone(&call_times); + let call_index = Arc::new(AtomicUsize::new(0)); + let recorded_call_index = Arc::clone(&call_index); + let loop_cancel = cancel.clone(); + let recovery_cancel = cancel.clone(); + let worker = tokio::spawn(async move { + run_tier_free_version_recovery_loop(loop_cancel, state, std::convert::identity, move |_, _, _| { + recorded_call_times + .lock() + .expect("recovery call times lock should not be poisoned") + .push(tokio::time::Instant::now().duration_since(started_at)); + let index = recorded_call_index.fetch_add(1, Ordering::SeqCst); + let cancel = recovery_cancel.clone(); + async move { + if index == 0 { + tokio::time::sleep(StdDuration::from_secs(75)).await; + Ok(free_version_recovery_stats(0, 0, true)) + } else { + cancel.cancel(); + Ok(free_version_recovery_stats(0, 0, false)) + } + } + }) + .await; + }); + + tokio::task::yield_now().await; + tokio::time::advance(StdDuration::from_secs(30)).await; + recovery_notify.notify_one(); + tokio::time::advance(StdDuration::from_secs(45)).await; + tokio::task::yield_now().await; + tokio::time::advance(StdDuration::from_secs(59)).await; + assert_eq!(call_index.load(Ordering::SeqCst), 1); + tokio::time::advance(StdDuration::from_secs(1)).await; + tokio::task::yield_now().await; + worker.await.expect("recovery loop should stop after cancellation"); + + assert_eq!( + call_times + .lock() + .expect("recovery call times lock should not be poisoned") + .as_slice(), + &[StdDuration::ZERO, StdDuration::from_secs(135)] ); } @@ -12557,6 +12610,181 @@ mod tests { .expect("empty recovery test bucket should be removed"); } + #[tokio::test] + #[serial] + async fn tier_free_version_recovery_object_budget_is_global_across_buckets() { + let (_disk_paths, ecstore) = setup_test_env().await; + for budget in [ + RecoveryWorkBudget { + max_objects: 0, + max_buckets: 1, + }, + RecoveryWorkBudget { + max_objects: 1, + max_buckets: 0, + }, + ] { + let err = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget) + .await + .expect_err("a zero recovery work budget must fail before starting an unbounded walk"); + assert!(matches!(err, Error::Io(_))); + } + + let suffix = Uuid::new_v4().simple(); + let buckets = [ + format!("zzzz-recovery-budget-{suffix}-a"), + format!("zzzz-recovery-budget-{suffix}-b"), + format!("zzzz-recovery-budget-{suffix}-c"), + ]; + for bucket in &buckets { + create_test_bucket(&ecstore, bucket).await; + } + + let first_bucket = buckets[0].clone(); + let first_bucket_for_hook = first_bucket.clone(); + let _walk = set_recovery_walk_test_hook(move |bucket| { + (bucket == first_bucket_for_hook).then(|| { + RecoveryWalkTestAction::SendItems( + ["ordinary-object-0", "ordinary-object-0", "ordinary-object-1"] + .into_iter() + .map(|name| ObjectInfo { + bucket: first_bucket_for_hook.clone(), + name: name.to_string(), + ..Default::default() + }) + .collect(), + ) + }) + }); + let page = list_tier_free_versions_with_budget( + Arc::clone(&ecstore), + 1, + None, + None, + CancellationToken::new(), + RecoveryWorkBudget { + max_objects: 1, + max_buckets: 10, + }, + ) + .await + .expect("the decoded-object boundary should enforce the recovery budget"); + + assert!(page.items.is_empty()); + assert_eq!(page.scanned_entries, 3); + assert_eq!(page.buckets_scanned, 1); + assert!(page.truncated); + assert_eq!(page.next_bucket_marker.as_deref(), Some(first_bucket.as_str())); + assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object-0")); + drop(_walk); + + let walked = Arc::new(StdMutex::new(Vec::new())); + let walked_by_hook = Arc::clone(&walked); + let test_buckets = buckets.clone(); + let _walk = set_recovery_walk_test_hook(move |bucket| { + test_buckets + .iter() + .find(|candidate| candidate.as_str() == bucket) + .map(|bucket| { + walked_by_hook + .lock() + .expect("recovery walk log should not be poisoned") + .push(bucket.clone()); + RecoveryWalkTestAction::SendItems(vec![ObjectInfo { + bucket: bucket.clone(), + name: "ordinary-object".to_string(), + ..Default::default() + }]) + }) + }); + + let page = list_tier_free_versions_with_budget( + Arc::clone(&ecstore), + 1, + None, + None, + CancellationToken::new(), + RecoveryWorkBudget { + max_objects: 2, + max_buckets: 10, + }, + ) + .await + .expect("the bounded recovery page should be listed"); + + assert!(page.items.is_empty()); + assert_eq!(page.scanned_entries, 2); + assert_eq!(page.buckets_scanned, 2); + assert!(page.truncated); + assert_eq!(page.next_bucket_marker.as_deref(), Some(buckets[1].as_str())); + assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object")); + assert_eq!(walked.lock().expect("recovery walk log should not be poisoned").as_slice(), &buckets[..2]); + + for bucket in &buckets { + ecstore + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("empty recovery test bucket should be removed"); + } + } + + #[tokio::test] + #[serial] + async fn tier_free_version_recovery_bucket_budget_resumes_at_unscanned_bucket() { + let (_disk_paths, ecstore) = setup_test_env().await; + let suffix = Uuid::new_v4().simple(); + let buckets = [ + format!("zzzz-recovery-buckets-{suffix}-a"), + format!("zzzz-recovery-buckets-{suffix}-b"), + ]; + for bucket in &buckets { + create_test_bucket(&ecstore, bucket).await; + } + + let test_buckets = buckets.clone(); + let _walk = set_recovery_walk_test_hook(move |bucket| { + test_buckets + .iter() + .any(|candidate| candidate.as_str() == bucket) + .then(|| RecoveryWalkTestAction::SendItems(Vec::new())) + }); + let budget = RecoveryWorkBudget { + max_objects: 10, + max_buckets: 1, + }; + let first = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget) + .await + .expect("the first bucket-bounded page should be listed"); + + assert_eq!(first.buckets_scanned, 1); + assert!(first.truncated); + assert_eq!(first.next_bucket_marker.as_deref(), Some(buckets[1].as_str())); + assert!(first.next_object_marker.is_none()); + + let second = list_tier_free_versions_with_budget( + Arc::clone(&ecstore), + 1, + first.next_bucket_marker, + first.next_object_marker, + CancellationToken::new(), + budget, + ) + .await + .expect("the second bucket-bounded page should resume"); + + assert_eq!(second.buckets_scanned, 1); + assert!(!second.truncated); + assert!(second.next_bucket_marker.is_none()); + assert!(second.next_object_marker.is_none()); + + for bucket in &buckets { + ecstore + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("empty recovery test bucket should be removed"); + } + } + #[tokio::test] #[serial] async fn tier_free_version_recovery_propagates_walk_item_error() { diff --git a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs index d00c74e5a..4ccec6f23 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs @@ -32,7 +32,10 @@ use crate::store::ECStore; use rustfs_filemeta::FileInfo; pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000; +// These are page-wide repair budgets. Applying them per bucket would still let +// one recovery pass walk an unbounded namespace before the scheduler can cool down. const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000; +const DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT: usize = 100; #[cfg(not(test))] const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(test)] @@ -41,6 +44,12 @@ const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); type ObjectInfoOrErr = StorageObjectInfoOrErr; type WalkOptions = StorageWalkOptions bool>; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct RecoveryWorkBudget { + pub(super) max_objects: usize, + pub(super) max_buckets: usize, +} + fn recovery_walk_options(limit: usize, marker: Option) -> WalkOptions { WalkOptions { include_free_versions: true, @@ -58,6 +67,7 @@ fn recovery_walk_options(limit: usize, marker: Option) -> WalkOptions { #[cfg(test)] pub(super) enum RecoveryWalkTestAction { + SendItems(Vec), SendItemsThenError(Vec, crate::error::Error), SendItemsThenHang(Vec, Arc), SendItemsUntilReceiverCloses(Arc), @@ -277,6 +287,17 @@ pub(super) async fn list_tier_free_versions( bucket_marker: Option, object_marker: Option, cancel_token: CancellationToken, +) -> Result { + list_tier_free_versions_with_budget(api, limit, bucket_marker, object_marker, cancel_token, recovery_work_budget(limit)).await +} + +pub(super) async fn list_tier_free_versions_with_budget( + api: Arc, + limit: usize, + bucket_marker: Option, + object_marker: Option, + cancel_token: CancellationToken, + work_budget: RecoveryWorkBudget, ) -> Result { let mut page = FreeVersionRecoveryPage { items: Vec::new(), @@ -290,6 +311,9 @@ pub(super) async fn list_tier_free_versions( if limit == 0 { return Ok(page); } + if work_budget.max_objects == 0 || work_budget.max_buckets == 0 { + return Err(std::io::Error::other("free-version recovery work budget must be greater than zero").into()); + } let bucket_options = BucketOptions::default(); let list_buckets = async { @@ -313,7 +337,7 @@ pub(super) async fn list_tier_free_versions( }; let mut bucket_seen = bucket_marker.is_none(); let mut truncated_after: Option = None; - let walk_scan_limit = recovery_walk_scan_limit(limit); + let mut remaining_scan_objects = work_budget.max_objects; for bucket in buckets { if cancel_token.is_cancelled() { @@ -329,12 +353,20 @@ pub(super) async fn list_tier_free_versions( bucket_seen = true; } + if page.buckets_scanned >= work_budget.max_buckets { + page.truncated = true; + page.next_bucket_marker = Some(bucket.name); + page.next_object_marker = None; + break; + } + page.buckets_scanned += 1; let bucket_object_marker = if bucket_marker.as_deref() == Some(bucket.name.as_str()) { object_marker.clone() } else { None }; + let bucket_walk_limit = remaining_scan_objects; let (tx, mut rx) = mpsc::channel::(100); let cancel = cancel_token.child_token(); @@ -358,6 +390,21 @@ pub(super) async fn list_tier_free_versions( #[cfg(test)] if let Some(action) = test_action { match action { + RecoveryWalkTestAction::SendItems(items) => { + for item in items { + if tx + .send(ObjectInfoOrErr { + item: Some(item), + err: None, + }) + .await + .is_err() + { + return Ok(()); + } + } + return Ok(()); + } RecoveryWalkTestAction::SendItemsThenError(items, err) => { for item in items { if tx @@ -425,7 +472,7 @@ pub(super) async fn list_tier_free_versions( } } - api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(walk_scan_limit, object_marker)) + api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(bucket_walk_limit, object_marker)) .await } }); @@ -466,6 +513,18 @@ pub(super) async fn list_tier_free_versions( let Some(oi) = item.item else { continue; }; + if last_seen_object.as_deref() != Some(oi.name.as_str()) && scanned_objects >= bucket_walk_limit { + // The disk listing limit follows S3-visible counting rules and + // does not charge metadata containing only hidden/free versions. + // Enforce the repair budget again at the decoded-object boundary. + page.truncated = true; + page.next_bucket_marker = Some(bucket.name.clone()); + page.next_object_marker = last_seen_object.clone(); + cancel.cancel(); + draining_after_truncation = true; + drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT); + continue; + } record_scanned_object(&mut last_seen_object, &mut scanned_objects, &oi.name); if let Some(cursor) = &truncated_after && (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != oi.name.as_str()) @@ -509,7 +568,8 @@ pub(super) async fn list_tier_free_versions( return Err(err); } walk_result?; - mark_scan_truncated_if_needed(&mut page, scanned_objects, walk_scan_limit, &bucket.name, last_seen_object.as_deref()); + remaining_scan_objects = remaining_scan_objects.saturating_sub(scanned_objects); + mark_scan_truncated_if_needed(&mut page, scanned_objects, bucket_walk_limit, &bucket.name, last_seen_object.as_deref()); if page.truncated { break; @@ -528,6 +588,13 @@ fn recovery_walk_scan_limit(limit: usize) -> usize { DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT.max(limit.saturating_add(1)) } +fn recovery_work_budget(limit: usize) -> RecoveryWorkBudget { + RecoveryWorkBudget { + max_objects: recovery_walk_scan_limit(limit), + max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT, + } +} + fn record_scanned_object(last_seen_object: &mut Option, scanned_objects: &mut usize, object: &str) { if last_seen_object.as_deref() == Some(object) { return; @@ -700,6 +767,13 @@ mod tests { recovery_walk_scan_limit(DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT), DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT + 1 ); + assert_eq!( + recovery_work_budget(DEFAULT_FREE_VERSION_RECOVERY_LIMIT), + RecoveryWorkBudget { + max_objects: DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT, + max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT, + } + ); } #[test] diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 014bf2a14..44b9de648 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -21,6 +21,16 @@ | `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` | | Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` | +## Free-version recovery controls + +The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism. + +Normal transitioned deletes pass a post-commit receipt directly to the lifecycle queue. The namespace walk is the crash, queue-pressure, mixed-version, and historical-record fallback. It cannot safely skip a bucket merely because that bucket has no current lifecycle rule or the referenced tier was removed: an older `xl.meta` free-version can still be the only owner of a required remote DELETE. + +One background recovery page completes at most 10,000 logical objects across at most 100 buckets and enqueues at most 1,000 recoverable free versions. The scanner can decode one additional object to detect truncation; a continuation marker preserves the first unscanned bucket or the last completely scanned object. Every truncated page and follow-up sweep waits at least 60 seconds after the previous page completes; failed pages back off from 60 seconds to 10 minutes, and complete idle sweeps exponentially back off to 10 minutes with jitter. Individual walks have no fixed total timeout, but inherit the drive walk stall timeout so a large healthy bucket can make progress without recreating the old timeout/restart loop. + +The structured `lifecycle_worker_state` recovery event reports `duration_ms`, `scanned_entries`, `buckets_scanned`, queue counts, truncation, and continuation markers. `rustfs_internal_stage_duration_ms{stage="lifecycle_free_version_recovery"}` records successful page duration; `stage="lifecycle_free_version_recovery_failed"` records failures. + ## Metadata key conventions Internal metadata is stored under both `x-rustfs-internal-` and `x-minio-internal-` for MinIO interoperability. `get_bytes` prefers the RustFS key and falls back to the MinIO key. From a9373288f28ff62b58bca1e96278d5d1b44f3689 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 16:30:17 +0800 Subject: [PATCH 04/19] docs(architecture): decide how remote credentials are sealed at rest (#7131) Replication targets, remote tiers and on-demand migration sources each store a remote secret in clear text under .rustfs.sys today. Record one design for all three before any of them invents its own: seal only the secret fields, into an added field so an old reader finds the credential absent rather than a ciphertext it would sign with, unseal at remote-client construction, and reach KMS through an installed hook because ECStore does not depend on rustfs-kms. Covers the envelope format and its encryption context, the mixed-version compatibility matrix, the rollout gate, rotation ownership and the fail-closed rules. --- docs/architecture/README.md | 1 + .../remote-credential-sealing-adr.md | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 docs/architecture/remote-credential-sealing-adr.md diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 6cbbf569b..8609c55f2 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -48,6 +48,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch | [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration | | [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced | | [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch | +| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format and its mixed-version rules | | [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request | ## Support and compatibility matrices (release-facing, keep current) diff --git a/docs/architecture/remote-credential-sealing-adr.md b/docs/architecture/remote-credential-sealing-adr.md new file mode 100644 index 000000000..901948a8e --- /dev/null +++ b/docs/architecture/remote-credential-sealing-adr.md @@ -0,0 +1,52 @@ +# Remote Credential Sealing ADR + +**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, its fail-closed rules, and the mixed-version compatibility matrix. +**Source of truth:** the three stores that hold remote credentials today — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, and `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — plus the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier.rs`, and `crates/ecstore/src/bucket/on_demand_migration/config.rs`. + +## Decision + +Remote credentials are sealed **per field, into an added field, behind one shared seam**, and ECStore reaches KMS through an installed hook rather than a crate dependency. + +1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr`, and `OnDemandMigrationSys` seal and unseal through a single ECStore-owned envelope type. No consumer talks to KMS, and no consumer defines its own ciphertext layout. +2. **Only secret material is sealed.** `secret_key` and `session_token` are sealed. Endpoint, region, ARN, bucket, prefixes, path style, TLS flags, and the custom CA bundle stay in clear text: they are needed for validation, listing, and support diagnosis, and none of them is a secret. +3. **Sealed material lives in an added field, never in place of the plaintext field.** A record carries either the plaintext field or the sealed field. A reader that does not understand the sealed field therefore finds the credential *absent* rather than finding a ciphertext string it would sign requests with. +4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the single point that needs plaintext, so admin reads, listings, validation, and status paths never call KMS. + +## Envelope format + +A versioned, self-describing record: envelope version, KMS key id, KMS key version, algorithm, nonce, and ciphertext. It is stored base64 in the two JSON stores and as raw bytes inside the msgpack payload of the tier blob; the tier blob's own `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header constants are unchanged, because the envelope carries its own version. + +The KMS encryption context binds each ciphertext to the record that owns it — store kind, owning bucket or tier name, and field name — so a ciphertext copied into another bucket, another tier, or another field fails to decrypt instead of silently authorizing a different remote. + +## Why a hook instead of a dependency + +`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering. The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup: `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs`, installed by `install_ecstore_event_dispatch_hook` in `rustfs/src/server/event.rs`, and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`. Sealing uses the same shape, with the binary supplying an implementation backed by `get_global_kms_service_manager` in `crates/kms/src/service_manager.rs`. + +## Compatibility matrix + +| Stored form | Reader | Behavior | +|---|---|---| +| Plaintext (today's format) | Old node | Unchanged. | +| Plaintext | New node | Read as plaintext, no KMS call. Carries a `RUSTFS_COMPAT_TODO` marker per [compat-cleanup-register.md](compat-cleanup-register.md). | +| Sealed | New node, hook installed | Unsealed at client construction. | +| Sealed | New node, no hook or decrypt failure | Typed error; the target, tier, or source is unusable and reports why. Never a default, an empty credential, or the ciphertext bytes. | +| Sealed | Old node | The credential field is absent, so the old node fails closed on its existing "missing credentials" path. This is the migration hazard the rollout gate exists for. | + +## Rollout gate + +Sealing is written only when KMS is configured **and** a module switch in `rustfs/src/module_switches.rs` is on, defaulting off in the release that introduces it. Reading sealed records is always supported; writing them is what waits. Operators enable the switch after every node in the cluster can read the format, and existing plaintext records are sealed by re-submitting the configuration through its admin API — this task ships no in-place migration sweep. + +## Rotation + +The envelope records the key id and key version it was wrapped under. Re-wrapping is the KMS side's job and follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md); nothing in this design rotates, re-wraps, or expires a key on its own. + +## Fail-closed rules + +- A missing hook, a malformed envelope, an unknown envelope version, a failed decrypt, or an encryption-context mismatch is a typed error, per the AGENTS.md rule that required values return a typed error when absent or corrupt. +- A seal failure fails the admin write. A configuration is never persisted with the secret dropped or left in clear text after the operator asked for sealing. +- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed. +- Logs may carry the key id and envelope version. They never carry ciphertext, plaintext, or the encryption context's secret-adjacent values. + +## Non-goals + +Sealing the server config, IAM credentials, or object data keys; changing which principals may read a configuration; key material migration between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the existing trust boundary (reserved bucket paths plus admin authorization) is unchanged. From 3654c147e2c15997351fb138b30b2cc4458a1e7f Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 16:49:14 +0800 Subject: [PATCH 05/19] fix(ilm): defer aborted dispatch cleanup to recovery (#7126) --- .../bucket/lifecycle/tier_delete_journal.rs | 34 ++---- crates/ecstore/src/runtime/instance.rs | 8 -- crates/ecstore/src/store/init.rs | 101 ++++++++++++++++++ 3 files changed, 112 insertions(+), 31 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 3840199c2..d31295f5b 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -2388,22 +2388,8 @@ async fn prepare_tier_delete_dispatch_inner( return Err(Error::other("a tier delete dispatch rollback is still in progress")); } TierDeleteDispatchManifestState::Aborted => { - for name in &existing.journal_names { - if read_tier_delete_journal_with_etag(api.clone(), name).await?.is_some() { - return Err(Error::other("an aborted tier delete dispatch still owns journal records")); - } - } - let data = encode_tier_delete_dispatch_manifest(&existing)?; - let fences_current = || { - !bucket_fence.is_lock_lost() - && !operation_guard.is_lock_lost() - && tier_delete_journal_fleet_proof_matches(&fleet_proof) - && tier_delete_journal_topology_generation(&fleet_proof) == existing.topology_generation - }; - match delete_durable_config_if_match(api.clone(), &manifest_name, &data, &etag, &fences_current).await { - Ok(()) | Err(Error::ConfigNotFound) | Err(Error::PreconditionFailed) => continue, - Err(err) => return Err(err), - } + schedule_aborted_tier_delete_dispatch_cleanup(api.clone(), manifest_name.clone()); + return Err(Error::other("an aborted tier delete dispatch is awaiting bounded recovery cleanup")); } } } @@ -4193,6 +4179,15 @@ async fn schedule_tier_delete_dispatch_manifest_recovery( } } +fn schedule_aborted_tier_delete_dispatch_cleanup(api: Arc, manifest_name: String) { + api.ctx.wake_tier_delete_journal_recovery(); + tokio::spawn(async move { + let _ = + schedule_tier_delete_dispatch_manifest_recovery(api, manifest_name, TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_TIMEOUT) + .await; + }); +} + #[cfg(all(test, feature = "test-util"))] pub(crate) async fn recover_test_tier_delete_dispatch_manifest_with_page_budget( api: Arc, @@ -4474,19 +4469,12 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_tok let mut manifest_marker: Option = None; loop { - #[cfg(test)] tokio::select! { biased; _ = cancel_token.cancelled() => return, _ = interval.tick() => {}, _ = api.ctx.wait_for_tier_delete_journal_recovery() => {}, } - #[cfg(not(test))] - tokio::select! { - biased; - _ = cancel_token.cancelled() => return, - _ = interval.tick() => {}, - } let manifest_recovery = recover_tier_delete_dispatch_manifests( api.clone(), diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index b7090bcdd..ad65354ed 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -213,7 +213,6 @@ pub struct InstanceContext { object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, transition_transaction_recovery_stores: std::sync::Mutex>, - #[cfg(test)] tier_delete_journal_recovery_wakeup: tokio::sync::Notify, } @@ -260,7 +259,6 @@ impl InstanceContext { object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), - #[cfg(test)] tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(), } } @@ -655,16 +653,10 @@ impl InstanceContext { .insert(store_id) } - #[cfg(test)] - #[allow( - dead_code, - reason = "driven by the tier-delete-journal recovery test behind `--features test-util` (backlog#1823)" - )] pub(crate) fn wake_tier_delete_journal_recovery(&self) { self.tier_delete_journal_recovery_wakeup.notify_one(); } - #[cfg(test)] pub(crate) async fn wait_for_tier_delete_journal_recovery(&self) { self.tier_delete_journal_recovery_wakeup.notified().await; } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 7a169fe33..859415707 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -9063,6 +9063,107 @@ mod tests { assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier"); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn aborted_dispatch_retry_defers_residual_scan_to_recovery() { + const JOURNAL_COUNT: usize = 65; + + let temp_dir = tempfile::tempdir().expect("create aborted retry store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "aborted-dispatch-retry", &[4])).await; + shutdown.cancel(); + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "aborted-dispatch-retry-bucket"; + let prefix = "archive/"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("aborted retry bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("aborted retry bucket incarnation should resolve"); + let tier_name = "ABORTED-DISPATCH-RETRY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("aborted retry tier lease should resolve") + .backend_identity(); + let aborted_entries = (0..JOURNAL_COUNT) + .map(|index| { + ( + synthetic_v6_dispatch_entry( + bucket, + &format!("{prefix}{index:06}.bin"), + tier_name, + identity, + &uuid::Uuid::new_v4().to_string(), + ), + Some(TierDeleteJournalState::Prepared), + ) + }) + .collect::>(); + let entries = aborted_entries.iter().map(|(entry, _)| entry.clone()).collect::>(); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + prefix, + aborted_entries, + TierDeleteDispatchManifestState::Aborted, + ) + .await + .expect("Aborted retry fixture should persist"); + + let lifecycle_guard = store + .acquire_bucket_lifecycle_write_lock(bucket) + .await + .expect("aborted retry should acquire the bucket lifecycle fence"); + let mut fence_opts = ObjectOptions::default(); + fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard); + let bucket_fence = fence_opts + .bucket_lifecycle_lock_fence + .clone() + .expect("aborted retry should capture the bucket lifecycle fence"); + let fleet_proof = acquire_tier_delete_journal_fleet_proof().expect("aborted retry fixture should have a fleet proof"); + let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation); + + let error = + match prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence) + .await + { + Ok(_) => panic!("an Aborted manifest must be retained for bounded recovery cleanup"), + Err(err) => err, + }; + + assert!( + error.to_string().contains("bounded recovery cleanup"), + "unexpected aborted retry error: {error}" + ); + assert_eq!( + hook.entry_count(), + 0, + "request retry must not scan the retained Aborted journal set under bucket write lock" + ); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("retained Aborted manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Aborted) + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT); + assert_eq!(backend.remove_count().await, 0, "retry must not call the remote tier"); + drop(hook); + drop(lifecycle_guard); + + recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect("bounded recovery should clean the retained Aborted manifest"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] From 6f717b2752192b1d0cc5387f70387755e52943d7 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 17:02:03 +0800 Subject: [PATCH 06/19] fix(filemeta): accept MinIO RFC1123 restore expiry-date (#7115) --- crates/filemeta/src/fileinfo.rs | 97 ++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 42613185e..27721300d 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -28,7 +28,7 @@ use serde::de::{self, MapAccess, SeqAccess, Visitor, value::MapAccessDeserialize use serde::ser::SerializeMap; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use time::{OffsetDateTime, PrimitiveDateTime, format_description::well_known::Rfc3339}; use time::{format_description::FormatItem, macros::format_description}; use uuid::Uuid; @@ -1417,6 +1417,31 @@ impl RestoreStatusOps for RestoreStatus { } } +/// Parse the `expiry-date` value carried by a persisted `x-amz-restore` header. +/// +/// RustFS has always serialized this field as RFC3339, while MinIO (and any +/// object migrated in place from a MinIO deployment) writes Go's +/// `http.TimeFormat` — RFC1123 with a literal `GMT` zone. Accept both shapes +/// so a migrated "restored" object stays readable. +/// +/// This is not only a HEAD 500: `is_restored_object_on_disk` fails open when +/// the header cannot be parsed, and `MetaObject::uses_data_dir` falls back to +/// it, so an unparseable restore header makes a live restored data dir look +/// unused (backlog#1342). +/// +/// The write side deliberately stays on RFC3339 for now: an older node only +/// parses RFC3339, so switching the persisted format before this parser is +/// deployed everywhere would break rolling upgrades and rollbacks in the +/// opposite direction. +fn parse_restore_expiry_date(value: &str) -> Result { + if let Ok(expiry) = OffsetDateTime::parse(value, &Rfc3339) { + return Ok(expiry); + } + PrimitiveDateTime::parse(value, &RFC1123) + .map(PrimitiveDateTime::assume_utc) + .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED)) +} + pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect(); let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect(); @@ -1445,8 +1470,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { if expiry_tokens[0].trim() != "expiry-date" { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } - let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339) - .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))?; + let expiry = parse_restore_expiry_date(expiry_tokens[1].trim_matches('"'))?; return Ok(RestoreStatus { is_restore_in_progress: Some(false), restore_expiry_date: Some(Timestamp::from(expiry)), @@ -1471,6 +1495,7 @@ mod tests { use super::*; use proptest::collection::{hash_map, vec}; use proptest::prelude::*; + use time::macros::datetime; // backlog#959 / ECA-18: the interleaved per-block bitrot subsystem in // rustfs-ecstore (BitrotWriter / bitrot_verify / bitrot_shard_file_size) is @@ -2685,4 +2710,70 @@ mod tests { let empty = FileInfo::default(); assert!(format!("{empty:?}").contains("data: None")); } + + /// backlog#1342: MinIO writes the persisted `x-amz-restore` completion + /// header with Go's `http.TimeFormat` (RFC1123/GMT). Before this parser + /// accepted that shape, every object migrated in place from MinIO failed + /// `parse_restore_obj_status`. + #[test] + fn parses_minio_rfc1123_restore_expiry() { + let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30 GMT\"") + .expect("RFC1123 expiry-date must parse"); + assert_eq!(status.is_restore_in_progress, Some(false)); + let expiry = status.expiry().expect("completed restore carries an expiry"); + assert_eq!(expiry, datetime!(2025-01-01 10:20:30 UTC)); + } + + /// The RustFS-written shape must keep parsing unchanged: the persisted + /// format stays RFC3339 until every node can read both. + #[test] + fn parses_rustfs_rfc3339_restore_expiry() { + let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"2025-01-01T10:20:30Z\"") + .expect("RFC3339 expiry-date must parse"); + assert_eq!(status.expiry(), Some(datetime!(2025-01-01 10:20:30 UTC))); + } + + /// Round-trip both directions of the migration: what RustFS emits on the + /// wire (`to_string2`, RFC1123) and what it persists (`to_string`, + /// RFC3339) must both be readable by this parser. + #[test] + fn restore_status_round_trips_through_both_formats() { + let status = RestoreStatus { + is_restore_in_progress: Some(false), + restore_expiry_date: Some(Timestamp::from(datetime!(2030-06-15 07:08:09 UTC))), + }; + for rendered in [RestoreStatusOps::to_string(&status), status.to_string2()] { + let parsed = parse_restore_obj_status(&rendered).unwrap_or_else(|e| panic!("{rendered} must parse: {e}")); + assert_eq!(parsed.expiry(), Some(datetime!(2030-06-15 07:08:09 UTC)), "{rendered}"); + } + } + + /// A restored object migrated from MinIO must still be recognised as + /// on-disk: `is_restored_object_on_disk` fails open, and + /// `MetaObject::uses_data_dir` uses it to decide whether a data dir is + /// live, so a parse failure here can make a live data dir look reclaimable. + #[test] + fn minio_restored_object_is_recognised_as_on_disk() { + let mut meta = HashMap::new(); + meta.insert( + X_AMZ_RESTORE.as_str().to_string(), + "ongoing-request=\"false\", expiry-date=\"Fri, 01 Jan 9999 00:00:00 GMT\"".to_string(), + ); + assert!(is_restored_object_on_disk(&meta)); + } + + /// Widening the accepted formats must not weaken malformed-input + /// rejection: an unparseable or non-GMT-shaped expiry still fails closed. + #[test] + fn rejects_malformed_restore_expiry() { + for header in [ + "ongoing-request=\"false\", expiry-date=\"not-a-date\"", + "ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30\"", + "ongoing-request=\"false\", expiry-date=\"01 Jan 2025 10:20:30 GMT\"", + "ongoing-request=\"false\"", + "ongoing-request=\"false\", expires=\"2025-01-01T10:20:30Z\"", + ] { + assert!(parse_restore_obj_status(header).is_err(), "{header} must be rejected"); + } + } } From f16a30b231a78269251cff8b0093b4d4d8c0020d Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Fri, 4 Sep 2026 18:23:13 +0800 Subject: [PATCH 07/19] feat(scanner): expose authenticated dirty bucket snapshots (#7122) * feat(scanner): add peer bucket dirty snapshots * fix(scanner): keep dirty snapshot errors stable * test(protos): satisfy dirty snapshot clippy * fix(scanner): satisfy dirty snapshot clippy --------- Co-authored-by: Henry Guo --- .../e2e_test/src/reliant/grpc_lock_server.rs | 7 + crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/cluster/rpc/mod.rs | 2 +- .../src/cluster/rpc/peer_rest_client.rs | 276 +++++++++++++++++- .../ecstore/src/services/notification_sys.rs | 124 +++++++- .../ecstore/src/storage_api_contracts/mod.rs | 5 +- .../src/generated/proto_gen/node_service.rs | 79 +++++ crates/protos/src/lib.rs | 129 +++++++- crates/protos/src/node.proto | 22 ++ crates/scanner/src/lib.rs | 11 +- crates/scanner/src/scanner_io.rs | 6 +- crates/scanner/src/scanner_io/dirty_usage.rs | 48 +++ crates/scanner/src/scanner_io/tests.rs | 45 +++ crates/scanner/src/storage_api.rs | 5 +- crates/storage-api/src/lib.rs | 3 + rustfs/src/storage/rpc/node_service.rs | 141 ++++++++- 16 files changed, 873 insertions(+), 34 deletions(-) diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index d73a44787..4f534bea6 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -856,6 +856,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn scanner_dirty_usage_snapshot( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + async fn background_heal_status( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 6ad57fb21..aa04369a6 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -525,8 +525,8 @@ pub mod rpc { pub use crate::cluster::rpc::{ AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, - ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor, - build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, + ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX, + TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index 36886b36e..d0987a063 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env; pub(crate) use peer_rest_client::TierConfigReloadOutcome; pub use peer_rest_client::{ KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease, + SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, }; pub(crate) use peer_s3_client::heal_bucket_local_on_disks; pub use peer_s3_client::{ diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 03648d8d6..5316199e3 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -20,7 +20,8 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo use crate::error::{Error, Result}; use crate::storage_api_contracts::internode::{ SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, - SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, + SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, + SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, }; use crate::{ bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats, TierDailyStatsWire}, @@ -47,18 +48,19 @@ use rustfs_protos::proto_gen::node_service::{ HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest, - ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, - ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, - StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, - TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, - node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient, + ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse, + ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest, + SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, + TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, + TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient, + tier_mutation_control_service_client::TierMutationControlServiceClient, }; pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS}; use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection}; use rustfs_utils::XHost; use serde::{Deserialize, Serialize as _}; use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, io::Cursor, sync::{ Arc, Weak, @@ -185,18 +187,31 @@ pub struct ScannerPeerActivity { pub publication_blocked: Option, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScannerPeerDirtyUsageSnapshot { + pub instance_id: String, + pub generation: u64, + pub pending_bucket_count: u64, + pub protocol_version: u32, + pub complete: bool, + pub buckets: BTreeMap, +} + +fn scanner_instance_id_is_valid(instance_id: &str) -> bool { + instance_id.len() == 32 + && instance_id + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) +} + fn decode_scanner_activity_with_verifier( response: ScannerActivityResponse, challenge: &[u8; 16], verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>, ) -> Result { let instance_id = &response.instance_id; - if instance_id.len() != 32 - || !instance_id - .as_bytes() - .iter() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) - { + if !scanner_instance_id_is_valid(instance_id) { return Err(Error::other("peer returned an invalid scanner activity instance ID")); } let ( @@ -318,6 +333,82 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1 }) } +fn decode_scanner_dirty_usage_snapshot_with_verifier( + response: ScannerDirtyUsageSnapshotResponse, + challenge: &[u8; 16], + verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>, +) -> Result { + let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(challenge, &response) + .map_err(|_| Error::other("peer scanner dirty usage snapshot is too large to authenticate"))?; + verify_proof(&canonical, &response.response_proof)?; + + if response.protocol_version != SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION { + return Err(Error::other("peer returned unsupported scanner dirty usage snapshot protocol")); + } + if !scanner_instance_id_is_valid(&response.instance_id) { + return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID")); + } + if response.generation == u64::MAX { + return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation")); + } + if response.pending_bucket_count > 0 && response.generation == 0 { + return Err(Error::other("peer scanner dirty usage snapshot has pending buckets without a generation")); + } + if response.buckets.len() > SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES { + return Err(Error::other("peer scanner dirty usage snapshot exceeds the entry limit")); + } + let bucket_count = u64::try_from(response.buckets.len()) + .map_err(|_| Error::other("peer scanner dirty usage snapshot entry count cannot be represented"))?; + let max_entries = u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES).unwrap_or(u64::MAX); + if response.complete { + if response.pending_bucket_count != bucket_count { + return Err(Error::other( + "complete peer scanner dirty usage snapshot has an inconsistent bucket count", + )); + } + } else if !response.buckets.is_empty() || response.pending_bucket_count <= max_entries { + return Err(Error::other( + "incomplete peer scanner dirty usage snapshot must represent an entry-limit overflow", + )); + } + for pair in response.buckets.windows(2) { + if pair[0].bucket >= pair[1].bucket { + return Err(Error::other("peer scanner dirty usage snapshot buckets are not strictly ordered")); + } + } + for bucket in &response.buckets { + if bucket.bucket.is_empty() { + return Err(Error::other("peer scanner dirty usage snapshot contains an empty bucket name")); + } + if bucket.generation == 0 || bucket.generation > response.generation { + return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation")); + } + } + + Ok(ScannerPeerDirtyUsageSnapshot { + instance_id: response.instance_id, + generation: response.generation, + pending_bucket_count: response.pending_bucket_count, + protocol_version: response.protocol_version, + complete: response.complete, + buckets: response + .buckets + .into_iter() + .map(|bucket| (bucket.bucket, bucket.generation)) + .collect(), + }) +} + +fn decode_scanner_dirty_usage_snapshot( + response: ScannerDirtyUsageSnapshotResponse, + challenge: &[u8; 16], +) -> Result { + decode_scanner_dirty_usage_snapshot_with_verifier(response, challenge, |canonical, proof| { + verify_tonic_rpc_response_proof(canonical, proof) + .map_err(|_| Error::other("peer returned an invalid scanner dirty usage snapshot response proof")) + }) +} + fn scanner_activity_protocol_unsupported(err: &Error) -> bool { matches!( err, @@ -1935,6 +2026,30 @@ impl PeerRestClient { } } + pub async fn scanner_dirty_usage_snapshot(&self) -> Result { + self.finalize_result( + async { + let challenge = Uuid::new_v4(); + let mut client = self + .get_client() + .await? + .max_decoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE) + .max_encoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE); + let mut request = Request::new(ScannerDirtyUsageSnapshotRequest { + challenge: challenge.as_bytes().to_vec().into(), + protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + }); + let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(request.get_ref()) + .map_err(|_| Error::other("scanner dirty usage snapshot request is too large to authenticate"))?; + set_tonic_canonical_body_digest(&mut request, &canonical)?; + let response = client.scanner_dirty_usage_snapshot(request).await?.into_inner(); + decode_scanner_dirty_usage_snapshot(response, challenge.as_bytes()) + } + .await, + ) + .await + } + pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result { let result = self .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION) @@ -2640,6 +2755,141 @@ mod tests { }) } + fn decode_test_scanner_dirty_usage_snapshot( + response: ScannerDirtyUsageSnapshotResponse, + ) -> Result { + decode_scanner_dirty_usage_snapshot_with_verifier(response, &[9; 16], |_canonical, proof| { + (proof == b"proof") + .then_some(()) + .ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot response proof")) + }) + } + + fn test_scanner_dirty_usage_snapshot_response() -> ScannerDirtyUsageSnapshotResponse { + ScannerDirtyUsageSnapshotResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + generation: 7, + pending_bucket_count: 2, + protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete: true, + buckets: vec![ + rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket { + bucket: "archive".to_string(), + generation: 3, + }, + rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket { + bucket: "photos".to_string(), + generation: 7, + }, + ], + response_proof: b"proof".to_vec().into(), + } + } + + #[test] + fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() { + let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response()) + .expect("a complete authenticated dirty usage snapshot should decode"); + assert_eq!(decoded.instance_id, "0123456789abcdef0123456789abcdef"); + assert_eq!(decoded.generation, 7); + assert_eq!(decoded.pending_bucket_count, 2); + assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION); + assert!(decoded.complete); + assert_eq!(decoded.buckets.get("archive"), Some(&3)); + assert_eq!(decoded.buckets.get("photos"), Some(&7)); + + let overflow_count = + u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64"); + let overflow = decode_test_scanner_dirty_usage_snapshot(ScannerDirtyUsageSnapshotResponse { + pending_bucket_count: overflow_count, + complete: false, + buckets: Vec::new(), + ..test_scanner_dirty_usage_snapshot_response() + }) + .expect("an explicit all-or-nothing overflow snapshot should decode"); + assert!(!overflow.complete); + assert!(overflow.buckets.is_empty()); + } + + #[test] + fn scanner_dirty_usage_snapshot_rejects_inconsistent_or_partial_peer_data() { + let mut cases = Vec::new(); + + let mut invalid_instance = test_scanner_dirty_usage_snapshot_response(); + invalid_instance.instance_id = "ABCDEF0123456789ABCDEF0123456789".to_string(); + cases.push((invalid_instance, "instance ID")); + + let mut unsupported = test_scanner_dirty_usage_snapshot_response(); + unsupported.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1; + cases.push((unsupported, "unsupported")); + + let mut exhausted = test_scanner_dirty_usage_snapshot_response(); + exhausted.generation = u64::MAX; + cases.push((exhausted, "exhausted")); + + let mut inconsistent_count = test_scanner_dirty_usage_snapshot_response(); + inconsistent_count.pending_bucket_count = 3; + cases.push((inconsistent_count, "bucket count")); + + let mut unordered = test_scanner_dirty_usage_snapshot_response(); + unordered.buckets.reverse(); + cases.push((unordered, "strictly ordered")); + + let mut future_bucket = test_scanner_dirty_usage_snapshot_response(); + future_bucket.buckets[0].generation = 8; + cases.push((future_bucket, "bucket generation")); + + let mut zero_generation = test_scanner_dirty_usage_snapshot_response(); + zero_generation.buckets[0].generation = 0; + cases.push((zero_generation, "bucket generation")); + + let mut empty_bucket = test_scanner_dirty_usage_snapshot_response(); + empty_bucket.buckets[0].bucket.clear(); + cases.push((empty_bucket, "empty bucket name")); + + let mut partial = test_scanner_dirty_usage_snapshot_response(); + partial.complete = false; + cases.push((partial, "entry-limit overflow")); + + let too_many_buckets = ScannerDirtyUsageSnapshotResponse { + generation: 1, + pending_bucket_count: u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1) + .expect("the test snapshot entry limit should fit in u64"), + buckets: (0..=SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES) + .map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket { + bucket: format!("bucket-{index:04}"), + generation: 1, + }) + .collect(), + ..test_scanner_dirty_usage_snapshot_response() + }; + cases.push((too_many_buckets, "exceeds the entry limit")); + + let overflow_count = + u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64"); + let invalid_overflow = ScannerDirtyUsageSnapshotResponse { + generation: 0, + pending_bucket_count: overflow_count, + complete: false, + buckets: Vec::new(), + ..test_scanner_dirty_usage_snapshot_response() + }; + cases.push((invalid_overflow, "without a generation")); + + for (response, expected) in cases { + let err = + decode_test_scanner_dirty_usage_snapshot(response).expect_err("malformed dirty usage snapshots must fail closed"); + assert!(err.to_string().contains(expected), "expected {expected:?} in {err}"); + } + + let mut invalid_proof = test_scanner_dirty_usage_snapshot_response(); + invalid_proof.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1; + invalid_proof.response_proof = b"invalid".to_vec().into(); + let err = decode_test_scanner_dirty_usage_snapshot(invalid_proof) + .expect_err("an invalid response proof must fail before peer fields are trusted"); + assert!(err.to_string().contains("response proof")); + } + #[test] fn build_clients_from_slots_preserves_missing_remote_topology_slots() { let slots = vec![ diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index a7e399031..a0bb3fac0 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -13,7 +13,9 @@ // limitations under the License. use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats; -use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome}; +use crate::cluster::rpc::{ + PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome, +}; use crate::diagnostics::admin_server_info::get_commit_id; use crate::disk::DiskAPI; use crate::error::{Error, Result}; @@ -2080,6 +2082,32 @@ impl NotificationSys { Ok(generations) } + pub async fn scanner_dirty_usage_snapshots(&self) -> Result> { + if self.peer_clients.is_empty() { + return Err(Error::other("scanner dirty usage snapshot probe has no remote peers")); + } + if self.all_peer_clients.len() != self.peer_clients.len() + 1 { + return Err(Error::other("scanner dirty usage snapshot peer topology is incomplete")); + } + + let mut futures = Vec::with_capacity(self.peer_clients.len()); + for client in self.peer_clients.iter().cloned() { + futures.push(async move { + let client = client.ok_or_else(|| Error::other("scanner dirty usage snapshot peer is unreachable"))?; + let host = client.grid_host.clone(); + scanner_dirty_usage_snapshot_with_retry(&client, &host) + .await + .map(|snapshot| (host, snapshot)) + }); + } + + let mut snapshots = Vec::with_capacity(futures.len()); + for result in join_all(futures).await { + snapshots.push(result?); + } + Ok(snapshots) + } + pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result { let mut by_host = HashMap::with_capacity(acknowledgements.len()); for (host, instance_id, generation) in acknowledgements { @@ -2591,6 +2619,54 @@ async fn scanner_activity_with_retry(client: &PeerRestClient, host: &str) -> Res } } +async fn scanner_dirty_usage_snapshot_with_retry(client: &PeerRestClient, host: &str) -> Result { + let first = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await; + let should_retry = match &first { + Ok(Ok(_)) => false, + Ok(Err(err)) => scanner_activity_should_retry(Some(err), false), + Err(_) => scanner_activity_should_retry(None, true), + }; + + match first { + Ok(Ok(snapshot)) => return Ok(snapshot), + Ok(Err(err)) if !should_retry => return Err(err), + Ok(Err(err)) => { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + result = "retrying", + capability = "scanner_dirty_usage_snapshot", + peer = host, + error = %err, + "notification capability probe retrying" + ); + client.prepare_retry().await; + } + Err(_) => { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + result = "retrying", + capability = "scanner_dirty_usage_snapshot", + peer = host, + timeout = ?SCANNER_ACTIVITY_PROBE_TIMEOUT, + "notification capability probe retrying" + ); + client.prepare_retry().await; + } + } + + match timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await { + Ok(result) => result, + Err(_) => { + client.evict_connection().await; + Err(Error::Timeout) + } + } +} + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn call_peer_with_timeout( timeout_dur: Duration, @@ -3688,6 +3764,52 @@ mod tests { assert!(err.to_string().contains("peer topology is incomplete")); } + #[tokio::test] + async fn scanner_dirty_usage_snapshot_probe_rejects_unusable_peer_topologies() { + let unreachable = NotificationSys { + peer_clients: vec![None], + all_peer_clients: vec![None, None], + peer_topology_hosts: vec!["node-a:9000".to_string()], + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], + tier_config_reload_workers: Default::default(), + }; + let err = unreachable + .scanner_dirty_usage_snapshots() + .await + .expect_err("an unreachable peer must invalidate the distributed dirty usage snapshot"); + assert!(err.to_string().contains("peer is unreachable")); + + let empty = NotificationSys { + peer_clients: Vec::new(), + all_peer_clients: Vec::new(), + peer_topology_hosts: Vec::new(), + peer_admin_caches: Vec::new(), + tier_config_reload_workers: Default::default(), + }; + let err = empty + .scanner_dirty_usage_snapshots() + .await + .expect_err("an empty peer set must not produce a distributed dirty usage snapshot"); + assert!(err.to_string().contains("no remote peers")); + + let client = PeerRestClient::new( + "127.0.0.1:9000".to_string().try_into().expect("peer host should parse"), + "http://127.0.0.1:9000".to_string(), + ); + let incomplete = NotificationSys { + peer_clients: vec![Some(client)], + all_peer_clients: vec![None], + peer_topology_hosts: vec!["127.0.0.1:9000".to_string()], + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], + tier_config_reload_workers: Default::default(), + }; + let err = incomplete + .scanner_dirty_usage_snapshots() + .await + .expect_err("an incomplete topology must not produce a distributed dirty usage snapshot"); + assert!(err.to_string().contains("peer topology is incomplete")); + } + #[tokio::test] async fn server_info_no_client_slot_uses_topology_host_without_counting_rpc_failure() { let sys = NotificationSys { diff --git a/crates/ecstore/src/storage_api_contracts/mod.rs b/crates/ecstore/src/storage_api_contracts/mod.rs index cd48a5266..414080eb7 100644 --- a/crates/ecstore/src/storage_api_contracts/mod.rs +++ b/crates/ecstore/src/storage_api_contracts/mod.rs @@ -32,8 +32,9 @@ pub(crate) mod internode { PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, - SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, - WALK_DIR_STREAM_COMPLETION_V1, + SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, + SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, + WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, }; } diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 037e9e597..f015305f0 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1251,6 +1251,38 @@ pub struct ScannerActivityResponse { #[prost(bool, optional, tag = "11")] pub publication_blocked: ::core::option::Option, } +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerDirtyUsageBucket { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub generation: u64, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerDirtyUsageSnapshotRequest { + #[prost(bytes = "bytes", tag = "1")] + pub challenge: ::prost::bytes::Bytes, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ScannerDirtyUsageSnapshotResponse { + #[prost(string, tag = "1")] + pub instance_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub generation: u64, + #[prost(uint64, tag = "3")] + pub pending_bucket_count: u64, + #[prost(uint32, tag = "4")] + pub protocol_version: u32, + /// Incomplete snapshots are all-or-nothing and carry no bucket entries. + #[prost(bool, tag = "5")] + pub complete: bool, + #[prost(message, repeated, tag = "6")] + pub buckets: ::prost::alloc::vec::Vec, + #[prost(bytes = "bytes", tag = "7")] + pub response_proof: ::prost::bytes::Bytes, +} /// A short-lived storage-owned read admission used only around a final /// scanner metadata publication. It is intentionally separate from the /// ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -2899,6 +2931,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity")); self.inner.unary(req, path, codec).await } + pub async fn scanner_dirty_usage_snapshot( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerDirtyUsageSnapshot"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ScannerDirtyUsageSnapshot")); + self.inner.unary(req, path, codec).await + } pub async fn acquire_scanner_publication_lease( &mut self, request: impl tonic::IntoRequest, @@ -3457,6 +3504,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn scanner_dirty_usage_snapshot( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn acquire_scanner_publication_lease( &self, request: tonic::Request, @@ -5749,6 +5800,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/ScannerDirtyUsageSnapshot" => { + #[allow(non_camel_case_types)] + struct ScannerDirtyUsageSnapshotSvc(pub Arc); + impl tonic::server::UnaryService for ScannerDirtyUsageSnapshotSvc { + type Response = super::ScannerDirtyUsageSnapshotResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::scanner_dirty_usage_snapshot(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ScannerDirtyUsageSnapshotSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/AcquireScannerPublicationLease" => { #[allow(non_camel_case_types)] struct AcquireScannerPublicationLeaseSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 691f9a6f3..fd8704382 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -541,6 +541,34 @@ pub fn canonical_scanner_activity_v7_response_body( Ok(body) } +pub fn canonical_scanner_dirty_usage_snapshot_request_body( + request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-request-v1\0"); + body.push_u32(request.protocol_version); + body.push_bytes(request.challenge.as_ref())?; + Ok(body.finish()) +} + +pub fn canonical_scanner_dirty_usage_snapshot_response_body( + challenge: &[u8], + response: &proto_gen::node_service::ScannerDirtyUsageSnapshotResponse, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-response-v1\0"); + body.push_bytes(challenge)?; + body.push_str(&response.instance_id)?; + body.push_u64(response.generation); + body.push_u64(response.pending_bucket_count); + body.push_u32(response.protocol_version); + body.push_bool(response.complete); + body.push_count(response.buckets.len())?; + for bucket in &response.buckets { + body.push_str(&bucket.bucket)?; + body.push_u64(bucket.generation); + } + Ok(body.finish()) +} + /// Builds the body authenticated by the short-lived remote scanner publication /// lease request. This is a separate domain from ScannerActivity so v6/v7 /// observation proofs remain byte-for-byte compatible. @@ -1751,13 +1779,112 @@ mod scanner_activity_tests { use super::{ canonical_scanner_activity_request_body, canonical_scanner_activity_response_body, canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body, + canonical_scanner_dirty_usage_snapshot_request_body, canonical_scanner_dirty_usage_snapshot_response_body, canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body, canonical_scanner_publication_lease_response_body, proto_gen::node_service::{ - ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, + ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshotRequest, + ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, }, }; + #[test] + fn canonical_scanner_dirty_usage_snapshot_request_binds_every_field() { + let request = ScannerDirtyUsageSnapshotRequest { + challenge: vec![1; 16].into(), + protocol_version: 1, + }; + let baseline = canonical_scanner_dirty_usage_snapshot_request_body(&request) + .expect("scanner dirty usage snapshot request should encode"); + + for variant in [ + ScannerDirtyUsageSnapshotRequest { + challenge: vec![2; 16].into(), + ..request.clone() + }, + ScannerDirtyUsageSnapshotRequest { + protocol_version: 2, + ..request + }, + ] { + assert_ne!( + baseline, + canonical_scanner_dirty_usage_snapshot_request_body(&variant) + .expect("scanner dirty usage snapshot request variant should encode") + ); + } + } + + #[test] + fn canonical_scanner_dirty_usage_snapshot_response_binds_every_field() { + let response = ScannerDirtyUsageSnapshotResponse { + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + generation: 7, + pending_bucket_count: 2, + protocol_version: 1, + complete: true, + buckets: vec![ + ScannerDirtyUsageBucket { + bucket: "archive".to_string(), + generation: 3, + }, + ScannerDirtyUsageBucket { + bucket: "photos".to_string(), + generation: 7, + }, + ], + response_proof: vec![9; 32].into(), + }; + let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response) + .expect("scanner dirty usage snapshot response should encode"); + let mut variants = Vec::new(); + let mut instance = response.clone(); + instance.instance_id = "fedcba9876543210fedcba9876543210".to_string(); + variants.push(instance); + let mut generation = response.clone(); + generation.generation = 8; + variants.push(generation); + let mut count = response.clone(); + count.pending_bucket_count = 3; + variants.push(count); + let mut protocol = response.clone(); + protocol.protocol_version = 2; + variants.push(protocol); + let mut complete = response.clone(); + complete.complete = false; + variants.push(complete); + let mut bucket_name = response.clone(); + bucket_name.buckets[0].bucket = "backups".to_string(); + variants.push(bucket_name); + let mut bucket_generation = response.clone(); + bucket_generation.buckets[0].generation = 4; + variants.push(bucket_generation); + let mut bucket_order = response.clone(); + bucket_order.buckets.reverse(); + variants.push(bucket_order); + + for variant in variants { + assert_ne!( + baseline, + canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &variant) + .expect("scanner dirty usage snapshot response variant should encode") + ); + } + assert_ne!( + baseline, + canonical_scanner_dirty_usage_snapshot_response_body(&[2; 16], &response) + .expect("scanner dirty usage snapshot response challenge variant should encode") + ); + + let mut proof_only = response; + proof_only.response_proof = vec![8; 32].into(); + assert_eq!( + baseline, + canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &proof_only) + .expect("response proof must not authenticate itself") + ); + } + #[test] fn canonical_scanner_activity_request_binds_every_field() { let request = ScannerActivityRequest { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 477b7a120..d3796991c 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -882,6 +882,27 @@ message ScannerActivityResponse { optional bool publication_blocked = 11; } +message ScannerDirtyUsageBucket { + string bucket = 1; + uint64 generation = 2; +} + +message ScannerDirtyUsageSnapshotRequest { + bytes challenge = 1; + uint32 protocol_version = 2; +} + +message ScannerDirtyUsageSnapshotResponse { + string instance_id = 1; + uint64 generation = 2; + uint64 pending_bucket_count = 3; + uint32 protocol_version = 4; + // Incomplete snapshots are all-or-nothing and carry no bucket entries. + bool complete = 5; + repeated ScannerDirtyUsageBucket buckets = 6; + bytes response_proof = 7; +} + // A short-lived storage-owned read admission used only around a final // scanner metadata publication. It is intentionally separate from the // ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -1206,6 +1227,7 @@ service NodeService { // rpc CommitBinary() returns () {}; rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound + rpc ScannerDirtyUsageSnapshot(ScannerDirtyUsageSnapshotRequest) returns (ScannerDirtyUsageSnapshotResponse) {}; // auth-policy: body-bound rpc AcquireScannerPublicationLease(ScannerPublicationLeaseRequest) returns (ScannerPublicationLeaseResponse) {}; // auth-policy: body-bound rpc ReleaseScannerPublicationLease(ScannerPublicationLeaseReleaseRequest) returns (ScannerPublicationLeaseReleaseResponse) {}; // auth-policy: body-bound rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 6d0b84edb..f85a2e0b6 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -89,14 +89,17 @@ pub use scanner::{ scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest, }; pub use scanner_io::{ - ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, - record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state, - scanner_maintenance_generation, + ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, + acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, + scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, }; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; pub use storage_api::ScannerReplicationConfig as ReplicationConfig; -pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION}; +pub use storage_api::scan::{ + SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, + SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, +}; pub use workload_admission::set_scanner_workload_admission_snapshot_provider; static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0); diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 79af335cf..860ebb2cd 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -748,9 +748,9 @@ pub(crate) use cache::{ current_cache_root_or_prepare_with_generation, }; pub use dirty_usage::{ - ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, - record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state, - scanner_maintenance_generation, + ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, + acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, + scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, }; #[cfg(test)] pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests}; diff --git a/crates/scanner/src/scanner_io/dirty_usage.rs b/crates/scanner/src/scanner_io/dirty_usage.rs index 85bd2d806..a5978e263 100644 --- a/crates/scanner/src/scanner_io/dirty_usage.rs +++ b/crates/scanner/src/scanner_io/dirty_usage.rs @@ -27,6 +27,25 @@ pub struct ScannerDirtyUsageState { pub pending: bool, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScannerDirtyUsageBucket { + pub bucket: String, + pub generation: u64, +} + +/// A point-in-time view of the local dirty bucket generations. +/// +/// `complete == false` is an all-or-nothing overflow signal: `buckets` is +/// empty and callers must fall back to the global dirty generation rather than +/// treating a bounded prefix as authoritative. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScannerDirtyUsageSnapshot { + pub generation: u64, + pub pending_bucket_count: u64, + pub complete: bool, + pub buckets: Vec, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] pub enum ScannerDirtyUsageAckError { #[error("scanner process instance changed before dirty usage acknowledgement")] @@ -98,6 +117,35 @@ pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState { } } +pub fn scanner_dirty_usage_snapshot(max_entries: usize) -> ScannerDirtyUsageSnapshot { + let (generation, pending_bucket_count, complete, mut buckets) = { + let dirty_buckets = dirty_usage_buckets(); + let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire); + let pending_bucket_count = usize_to_u64_saturated(dirty_buckets.len()); + let complete = dirty_buckets.len() <= max_entries; + let buckets = if complete { + dirty_buckets + .iter() + .map(|(bucket, generation)| ScannerDirtyUsageBucket { + bucket: bucket.clone(), + generation: *generation, + }) + .collect::>() + } else { + Vec::new() + }; + (generation, pending_bucket_count, complete, buckets) + }; + buckets.sort_unstable_by(|left, right| left.bucket.cmp(&right.bucket)); + + ScannerDirtyUsageSnapshot { + generation, + pending_bucket_count, + complete, + buckets, + } +} + pub fn acknowledge_dirty_usage_generation( instance_id: &str, generation: u64, diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 2b086aa5e..47f1e86a0 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -500,6 +500,51 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() { clear_dirty_usage_buckets_for_tests(); } +#[test] +#[serial] +fn dirty_usage_snapshot_is_sorted_and_reports_its_cutoff() { + clear_dirty_usage_buckets_for_tests(); + let empty = scanner_dirty_usage_snapshot(0); + assert_eq!(empty.pending_bucket_count, 0); + assert!(empty.complete); + assert!(empty.buckets.is_empty()); + + record_dirty_usage_bucket("videos"); + record_dirty_usage_bucket("photos"); + let expected_generation = scanner_dirty_usage_state().generation; + + let snapshot = scanner_dirty_usage_snapshot(2); + + assert_eq!(snapshot.generation, expected_generation); + assert_eq!(snapshot.pending_bucket_count, 2); + assert!(snapshot.complete); + assert_eq!( + snapshot + .buckets + .iter() + .map(|bucket| bucket.bucket.as_str()) + .collect::>(), + vec!["photos", "videos"] + ); + assert!(snapshot.buckets.iter().all(|bucket| bucket.generation <= snapshot.generation)); + clear_dirty_usage_buckets_for_tests(); +} + +#[test] +#[serial] +fn dirty_usage_snapshot_marks_truncated_results_incomplete() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket("archive"); + record_dirty_usage_bucket("photos"); + + let snapshot = scanner_dirty_usage_snapshot(1); + + assert_eq!(snapshot.pending_bucket_count, 2); + assert!(!snapshot.complete); + assert!(snapshot.buckets.is_empty(), "incomplete snapshots must not expose a partial bucket list"); + clear_dirty_usage_buckets_for_tests(); +} + #[test] #[serial] fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() { diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 3c410deed..a0aa874b0 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -304,7 +304,10 @@ pub(crate) mod scan { }; #[cfg(test)] pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO}; - pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION}; + pub use super::storage_contracts::{ + SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, + SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, + }; } pub(crate) mod scanner_io { diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index b16a47152..2726939f0 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -56,6 +56,9 @@ pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5; /// state is not authenticated by that version. pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6; pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7; +pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 1; +pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = 4096; +pub const SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE: usize = 512 * 1024; #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 44578265c..0dd18424e 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -279,6 +279,27 @@ fn scanner_activity_response( } } +fn scanner_dirty_usage_snapshot_response( + snapshot: rustfs_scanner::ScannerDirtyUsageSnapshot, +) -> ScannerDirtyUsageSnapshotResponse { + ScannerDirtyUsageSnapshotResponse { + instance_id: rustfs_scanner::scanner_activity_epoch().to_string(), + generation: snapshot.generation, + pending_bucket_count: snapshot.pending_bucket_count, + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete: snapshot.complete, + buckets: snapshot + .buckets + .into_iter() + .map(|bucket| ScannerDirtyUsageBucket { + bucket: bucket.bucket, + generation: bucket.generation, + }) + .collect(), + response_proof: Bytes::new(), + } +} + fn scanner_activity_response_v7( namespace_generation: u64, topology_digest: [u8; 32], @@ -2085,6 +2106,42 @@ impl Node for NodeService { Ok(Response::new(response)) } + async fn scanner_dirty_usage_snapshot( + &self, + request: Request, + ) -> Result, Status> { + let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("scanner dirty usage snapshot request is too large to authenticate"))?; + verify_tonic_canonical_body_digest(&request, &canonical) + .map_err(|err| Status::permission_denied(format!("scanner dirty usage snapshot authentication failed: {err}")))?; + if request.get_ref().protocol_version != rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION { + return Err(Status::failed_precondition(format!( + "unsupported scanner dirty usage snapshot request protocol {}", + request.get_ref().protocol_version + ))); + } + if request.get_ref().challenge.len() != 16 { + return Err(Status::invalid_argument("scanner dirty usage snapshot challenge must be 16 bytes")); + } + let challenge: [u8; 16] = request + .into_inner() + .challenge + .as_ref() + .try_into() + .map_err(|_| Status::invalid_argument("scanner dirty usage snapshot challenge must be 16 bytes"))?; + let snapshot = rustfs_scanner::scanner_dirty_usage_snapshot(rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES); + if snapshot.generation == u64::MAX { + return Err(Status::resource_exhausted("scanner dirty usage generation is exhausted")); + } + let mut response = scanner_dirty_usage_snapshot_response(snapshot); + let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response) + .map_err(|_| Status::internal("scanner dirty usage snapshot response is too large to authenticate"))?; + response.response_proof = sign_tonic_rpc_response_proof(&canonical) + .map_err(|_| Status::unavailable("scanner dirty usage snapshot response authentication is unavailable"))? + .into(); + Ok(Response::new(response)) + } + async fn acquire_scanner_publication_lease( &self, request: Request, @@ -2579,12 +2636,12 @@ mod tests { LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, - RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest, - ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, - SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, - StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, - TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, - VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, + RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, + ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, + SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, + StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, + TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, + UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -5883,6 +5940,13 @@ mod tests { acknowledge_dirty_usage_generation: 0, } ); + assert_tampered!( + scanner_dirty_usage_snapshot, + ScannerDirtyUsageSnapshotRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + } + ); assert_tampered!( acquire_scanner_publication_lease, ScannerPublicationLeaseRequest { @@ -6056,6 +6120,71 @@ mod tests { assert_eq!(unavailable.code(), tonic::Code::Unavailable); } + #[tokio::test] + async fn test_scanner_dirty_usage_snapshot_requires_body_bound_auth_and_signs_a_consistent_view() { + let _ = rustfs_credentials::set_global_rpc_secret("scanner-dirty-usage-snapshot-test-secret".to_string()); + let service = create_test_node_service(); + let unsigned = service + .scanner_dirty_usage_snapshot(Request::new(ScannerDirtyUsageSnapshotRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + })) + .await + .expect_err("unsigned scanner dirty usage snapshot requests must fail"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + + let mut unsupported = Request::new(ScannerDirtyUsageSnapshotRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1, + }); + let unsupported_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(unsupported.get_ref()) + .expect("scanner dirty usage snapshot request should encode"); + set_tonic_canonical_body_digest(&mut unsupported, &unsupported_body).expect("digest metadata should encode"); + mark_v2_authenticated(&mut unsupported); + let unsupported = service + .scanner_dirty_usage_snapshot(unsupported) + .await + .expect_err("unsupported scanner dirty usage snapshot protocols must fail closed"); + assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition); + + let mut malformed = Request::new(ScannerDirtyUsageSnapshotRequest { + challenge: vec![7; 15].into(), + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + }); + let malformed_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(malformed.get_ref()) + .expect("scanner dirty usage snapshot request should encode"); + set_tonic_canonical_body_digest(&mut malformed, &malformed_body).expect("digest metadata should encode"); + mark_v2_authenticated(&mut malformed); + let malformed = service + .scanner_dirty_usage_snapshot(malformed) + .await + .expect_err("malformed scanner dirty usage snapshot challenges must fail closed"); + assert_eq!(malformed.code(), tonic::Code::InvalidArgument); + + let challenge = [7; 16]; + let mut signed = Request::new(ScannerDirtyUsageSnapshotRequest { + challenge: challenge.to_vec().into(), + protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + }); + let signed_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(signed.get_ref()) + .expect("scanner dirty usage snapshot request should encode"); + set_tonic_canonical_body_digest(&mut signed, &signed_body).expect("digest metadata should encode"); + mark_v2_authenticated(&mut signed); + let response = service + .scanner_dirty_usage_snapshot(signed) + .await + .expect("an authenticated scanner dirty usage snapshot request should succeed") + .into_inner(); + assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch()); + assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION); + let bucket_count = u64::try_from(response.buckets.len()).expect("snapshot bucket count should fit in u64"); + assert_eq!(response.complete, response.pending_bucket_count == bucket_count); + let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response) + .expect("scanner dirty usage snapshot response should encode"); + crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof) + .expect("scanner dirty usage snapshot response proof should verify"); + } + #[test] fn test_scanner_activity_response_uses_process_epoch_and_generations() { let response = scanner_activity_response_v7( From 7ff578ff20e570218c41ef08533a70672506ad7a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 18:25:10 +0800 Subject: [PATCH 08/19] fix(ilm): notify on batch noncurrent version expiry (#7116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch `NewerNoncurrentVersions` expiry path took a lifecycle event argument and ignored it: after `delete_objects` committed it only evicted the cache and scheduled replication deletes, so a successful noncurrent version expiry was invisible to notification subscribers while the equivalent current-version path emitted a lifecycle expiration event. Emit that event from the batch path too, reusing the existing lifecycle audit sink and event contract. Only entries that actually mutated something are announced, and cache eviction and replication scheduling keep their existing order and admission — the event is derived from the committed result and a send failure never rolls back a delete. "No error" is not enough to prove a mutation: the disk layer skips an absent version and reports success, so a batch entry for a version that was already gone came back indistinguishable from a committed delete. The delete plan already resolves whether the source exists, so carry that `source_missing` result on `DeletedObject` and let the lifecycle path stay silent for versions it did not remove. backlog#2202 --- .../lifecycle/bucket_lifecycle_audit.rs | 139 ++++++++++++++++++ .../bucket/lifecycle/bucket_lifecycle_ops.rs | 114 ++++++++++++++ .../lifecycle/object_handlers_common.rs | 7 + .../src/services/event_notification.rs | 77 ++++++++-- crates/ecstore/src/set_disk/ops/object.rs | 2 + crates/storage-api/src/object.rs | 10 ++ 6 files changed, 340 insertions(+), 9 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs index e7dde4652..55e5bf63b 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs @@ -16,6 +16,7 @@ use super::runtime_boundary as runtime_sources; use crate::bucket::lifecycle::lifecycle; use crate::object_api::ObjectInfo; use crate::services::event_notification::{EventArgs, send_event}; +use crate::storage_api_contracts::object::{DeletedObject, ObjectToDelete}; use rustfs_s3_types::EventName; use rustfs_scanner_metrics::metrics::IlmAction; @@ -76,6 +77,60 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source: emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT); } +/// Emit the lifecycle expiration event for one version removed by the batch +/// `NewerNoncurrentVersions` expiry path. +/// +/// That path never sent events, so a successful noncurrent-version expiry was +/// invisible to notification subscribers even though the equivalent +/// current-version path emits one (backlog#2202). +pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) { + if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) { + emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT); + } +} + +/// Decide which event a single batch entry earned, if any. +/// +/// Only an entry that mutated something may be announced. "No error" is not +/// enough, and neither is `found`: the disk layer skips an absent version and +/// reports success (`delete_versions_internal` in `disk/local.rs` continues +/// past `FileVersionNotFound`), so a batch entry for a version that was +/// already gone comes back indistinguishable from a committed delete. The +/// delete plan's own source lookup is the signal that survives that, and the +/// lifecycle batch path always performs it because every target carries an +/// exact version identity. +fn noncurrent_expiration_event( + bucket: &str, + target: &ObjectToDelete, + deleted: &DeletedObject, + failed: bool, +) -> Option<(EventName, ObjectInfo)> { + if failed || !deleted.found || deleted.source_missing { + return None; + } + // A version removed by explicit version id is a plain versioned delete + // even when that version is itself a delete marker; only a request that + // carried no version id can publish a new delete marker. This is the rule + // the S3 DeleteObjects path applies (issue #6745). `delete_object_versions` + // now refuses targets without an exact version identity, so the + // marker-creation shape is unreachable from that caller; the mapping stays + // here so a future caller cannot silently announce the wrong mutation. + let created_delete_marker = deleted.delete_marker && target.version_id.is_none(); + let (event_name, version_id) = if created_delete_marker { + (EventName::LifecycleExpirationDeleteMarkerCreated, deleted.delete_marker_version_id) + } else { + (EventName::LifecycleExpirationDelete, deleted.version_id.or(target.version_id)) + }; + let object = ObjectInfo { + bucket: bucket.to_string(), + name: target.object_name.clone(), + version_id, + delete_marker: deleted.delete_marker, + ..Default::default() + }; + Some((event_name, object)) +} + fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) { send_event(EventArgs { event_name: event_name.to_string(), @@ -113,6 +168,7 @@ fn non_transitioned_expiration_event_name( #[cfg(test)] mod tests { use super::*; + use uuid::Uuid; #[test] fn transitioned_expiration_event_marks_delete_marker_creation() { @@ -129,4 +185,87 @@ mod tests { EventName::LifecycleExpirationDelete ); } + + fn deleted_version(version_id: Uuid) -> DeletedObject { + DeletedObject { + object_name: "object".to_string(), + version_id: Some(version_id), + found: true, + ..Default::default() + } + } + + fn target_version(version_id: Option) -> ObjectToDelete { + ObjectToDelete { + object_name: "object".to_string(), + version_id, + ..Default::default() + } + } + + #[test] + fn noncurrent_expiration_emits_versioned_delete_with_exact_identity() { + let version_id = Uuid::new_v4(); + let (event_name, object) = + noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted_version(version_id), false) + .expect("a committed delete must emit"); + assert_eq!(event_name, EventName::LifecycleExpirationDelete); + assert_eq!(object.bucket, "bucket"); + assert_eq!(object.name, "object"); + assert_eq!(object.version_id, Some(version_id)); + } + + /// Removing a noncurrent version that happens to be a delete marker is a + /// plain versioned delete, not a delete-marker creation. + #[test] + fn noncurrent_expiration_of_a_delete_marker_version_is_a_plain_delete() { + let version_id = Uuid::new_v4(); + let deleted = DeletedObject { + delete_marker: true, + ..deleted_version(version_id) + }; + let (event_name, object) = noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted, false) + .expect("a committed delete must emit"); + assert_eq!(event_name, EventName::LifecycleExpirationDelete); + assert_eq!(object.version_id, Some(version_id)); + } + + #[test] + fn noncurrent_expiration_reports_a_created_delete_marker() { + let marker_version_id = Uuid::new_v4(); + let deleted = DeletedObject { + object_name: "object".to_string(), + delete_marker: true, + delete_marker_version_id: Some(marker_version_id), + found: true, + ..Default::default() + }; + let (event_name, object) = + noncurrent_expiration_event("bucket", &target_version(None), &deleted, false).expect("a committed delete must emit"); + assert_eq!(event_name, EventName::LifecycleExpirationDeleteMarkerCreated); + assert_eq!(object.version_id, Some(marker_version_id)); + } + + /// A batch mixes successes with failures and versions that were already + /// gone; only a real mutation may produce an event. A version that was + /// already gone comes back with no error and `found` set, so + /// `source_missing` is the signal that keeps it silent. + #[test] + fn noncurrent_expiration_skips_failed_and_missing_versions() { + let version_id = Uuid::new_v4(); + let target = target_version(Some(version_id)); + assert!(noncurrent_expiration_event("bucket", &target, &deleted_version(version_id), true).is_none()); + + let absent = DeletedObject { + source_missing: true, + ..deleted_version(version_id) + }; + assert!(noncurrent_expiration_event("bucket", &target, &absent, false).is_none()); + + let not_found = DeletedObject { + found: false, + ..deleted_version(version_id) + }; + assert!(noncurrent_expiration_event("bucket", &target, ¬_found, false).is_none()); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index d9e352e26..7a8b65f75 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -11336,6 +11336,120 @@ mod tests { } } + /// backlog#2202: the batch `NewerNoncurrentVersions` path used to delete + /// noncurrent versions without telling notification subscribers anything, + /// while the current-version path emitted a lifecycle expiration event. + /// Only versions this batch actually removed may produce an event. + #[tokio::test] + #[serial] + async fn lifecycle_noncurrent_batch_expiry_emits_events_only_for_committed_deletes() { + use crate::services::event_notification::test_recorder; + use rustfs_s3_types::EventName; + + let (_disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("lifecycle-noncurrent-events-{}", Uuid::new_v4().simple()); + let object = "logs/object"; + create_test_bucket(&ecstore, &bucket).await; + + metadata_sys::update_in( + &ecstore.ctx, + &bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("bucket versioning should be enabled"); + + let now = OffsetDateTime::now_utc(); + let mut noncurrent_reader = PutObjReader::from_vec(b"noncurrent".to_vec()); + let noncurrent = ecstore + .put_object( + &bucket, + object, + &mut noncurrent_reader, + &ObjectOptions { + versioned: true, + mod_time: Some(now - time::Duration::days(40)), + ..Default::default() + }, + ) + .await + .expect("the noncurrent version should be created"); + let noncurrent_version_id = noncurrent.version_id.expect("a versioned PUT has an exact identity"); + let mut current_reader = PutObjReader::from_vec(b"current".to_vec()); + let current = ecstore + .put_object( + &bucket, + object, + &mut current_reader, + &ObjectOptions { + versioned: true, + mod_time: Some(now - time::Duration::days(2)), + ..Default::default() + }, + ) + .await + .expect("the current version should be created"); + let current_version_id = current.version_id.expect("a versioned PUT has an exact identity"); + + let incarnation = ecstore + .bucket_incarnation_id_from_disk(&bucket) + .await + .expect("bucket incarnation should be available"); + + test_recorder::install(); + + // One version that exists and one that never did: `delete_objects` + // suppresses the not-found error, so only the committed delete may be + // announced. + let missing_version_id = Uuid::new_v4(); + let targets = vec![ + ObjectToDelete { + object_name: object.to_string(), + version_id: Some(noncurrent_version_id), + ..Default::default() + }, + ObjectToDelete { + object_name: object.to_string(), + version_id: Some(missing_version_id), + ..Default::default() + }, + ]; + let failed = crate::bucket::lifecycle::object_handlers_common::delete_object_versions( + &ecstore, + &bucket, + &targets, + lifecycle::Event::default(), + incarnation, + ) + .await; + assert_eq!(failed, 0, "a missing version is not a batch failure"); + + let events = test_recorder::recorded_for_bucket(&bucket); + let announced = events.iter().map(|event| event.version_id).collect::>(); + assert_eq!( + announced, + vec![Some(noncurrent_version_id)], + "only the committed noncurrent delete should be announced \ + (noncurrent={noncurrent_version_id}, current={current_version_id}, missing={missing_version_id}), got {events:?}" + ); + assert_eq!(events[0].event_name, EventName::LifecycleExpirationDelete.to_string()); + assert_eq!(events[0].object, object); + assert!(!events[0].delete_marker); + + let remaining = ecstore + .clone() + .list_object_versions(&bucket, object, None, None, None, 10) + .await + .expect("remaining versions should be listable") + .objects + .into_iter() + .filter(|candidate| candidate.name == object) + .collect::>(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].version_id, Some(current_version_id)); + } + #[tokio::test] #[serial] async fn lifecycle_deletes_only_the_historical_null_version_after_versioning_is_reenabled() { diff --git a/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs index 3f4d1d8ac..6b57afcec 100644 --- a/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs +++ b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs @@ -20,6 +20,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped"; const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed"; +use crate::bucket::lifecycle::bucket_lifecycle_audit::emit_noncurrent_expiration_event; use crate::bucket::lifecycle::lifecycle; use crate::bucket::lifecycle::replication_sink::{self, ReplicationObjectBridge}; use crate::object_api::ObjectOptions; @@ -98,6 +99,12 @@ pub async fn delete_object_versions( // version so it does not sit resident until TTL (ODC-26). if let Some(target) = to_del.get(i) { crate::object_api::notify_object_mutation(bucket, &target.object_name).await; + // Announce the version this batch actually removed. Cache + // eviction and replication scheduling keep their existing + // order and admission; the event is derived from the committed + // result, and a send failure never rolls back a delete that + // already happened (backlog#2202). + emit_noncurrent_expiration_event(bucket, target, deleted_obj, false); } if deleted_obj.replication_state.is_none() { continue; diff --git a/crates/ecstore/src/services/event_notification.rs b/crates/ecstore/src/services/event_notification.rs index d9f5267be..8e37f1c6b 100644 --- a/crates/ecstore/src/services/event_notification.rs +++ b/crates/ecstore/src/services/event_notification.rs @@ -117,26 +117,85 @@ pub fn send_event(args: EventArgs) { ); } +/// Shared event recorder for this crate's tests. +/// +/// [`register_event_dispatch_hook`] backs a `OnceLock`, so only the first +/// caller in a test binary can install a hook. Every test that needs to +/// observe dispatched events must therefore go through this single recorder +/// instead of registering its own. +#[cfg(test)] +pub(crate) mod test_recorder { + use super::register_event_dispatch_hook; + use std::sync::{Mutex, OnceLock}; + use uuid::Uuid; + + /// The fields a test needs from a dispatched event. `EventArgs` itself is + /// not `Clone`, and recording a reduced shape keeps this test seam from + /// constraining the production type. + #[derive(Clone, Debug)] + pub(crate) struct RecordedEvent { + pub(crate) event_name: String, + pub(crate) bucket: String, + pub(crate) object: String, + pub(crate) version_id: Option, + pub(crate) delete_marker: bool, + } + + static RECORDED: OnceLock>> = OnceLock::new(); + + fn recorded() -> &'static Mutex> { + RECORDED.get_or_init(|| Mutex::new(Vec::new())) + } + + pub(crate) fn install() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + assert!( + register_event_dispatch_hook(|args| { + recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent { + event_name: args.event_name, + bucket: args.bucket_name, + object: args.object.name, + version_id: args.object.version_id, + delete_marker: args.object.delete_marker, + }); + }), + "the test event recorder must own this binary's dispatch hook" + ); + }); + } + + /// Everything recorded for one bucket. Tests select by their own unique + /// bucket name rather than draining, because tests that are not + /// `#[serial]` may dispatch events concurrently. + pub(crate) fn recorded_for_bucket(bucket: &str) -> Vec { + recorded() + .lock() + .unwrap_or_else(|err| err.into_inner()) + .iter() + .filter(|event| event.bucket == bucket) + .cloned() + .collect() + } +} + #[cfg(test)] mod tests { use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - - static DISPATCH_COUNT: AtomicUsize = AtomicUsize::new(0); #[test] fn send_event_dispatches_to_registered_hook() { - let _ = register_event_dispatch_hook(|_args| { - DISPATCH_COUNT.fetch_add(1, Ordering::Relaxed); - }); - let before = DISPATCH_COUNT.load(Ordering::Relaxed); + test_recorder::install(); + let bucket = format!("event-dispatch-{}", uuid::Uuid::new_v4().simple()); send_event(EventArgs { event_name: "s3:ObjectCreated:Put".to_string(), - bucket_name: "demo".to_string(), + bucket_name: bucket.clone(), ..Default::default() }); - assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), before + 1); + let dispatched = test_recorder::recorded_for_bucket(&bucket); + assert_eq!(dispatched.len(), 1); + assert_eq!(dispatched[0].event_name, "s3:ObjectCreated:Put"); } } diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 40e45c902..5a220e432 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7935,6 +7935,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { delete_marker_mtime: vr.mod_time.or(goi.mod_time), object_name: vr.name.clone(), replication_state: vr.replication_state_internal.clone(), + source_missing, ..Default::default() } } else { @@ -7946,6 +7947,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { vr.version_id }, replication_state: vr.replication_state_internal.clone(), + source_missing, ..Default::default() }; accounting[i] = Some(DeleteAccounting { diff --git a/crates/storage-api/src/object.rs b/crates/storage-api/src/object.rs index b2a090941..b7aaf28dc 100644 --- a/crates/storage-api/src/object.rs +++ b/crates/storage-api/src/object.rs @@ -243,6 +243,16 @@ impl ObjectToDelete { #[derive(Debug, Default, Clone)] pub struct DeletedObject { pub delete_marker: bool, + /// True when the delete plan looked the target up and found no such + /// object or version. + /// + /// The lookup only runs when the plan needs the source (Object Lock + /// check, replication decision, tier journal, or an expected identity), + /// so this proves absence and never proves presence: it stays false when + /// no lookup ran. Callers that must not announce a delete that removed + /// nothing need this, because the disk layer treats an absent version as + /// an idempotent success and reports `found` regardless. + pub source_missing: bool, pub delete_marker_version_id: Option, pub object_name: String, pub version_id: Option, From f6bed1a73a5cce9fc4b399e6b9c002388c85a085 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 18:25:34 +0800 Subject: [PATCH 09/19] fix(ci): track nightly test selection per platform (#7128) --- .config/e2e-nightly-selection.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index f34c90f2b..163d3235b 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1 +1,2 @@ -sha256=3da900ddcd7e2b9d3544bccfd69e468be5605a9fd2ea0ef5a6218304e42de5b4 +sha256-darwin=a5665318c9bdc0947514fb7008ba1b83b114b739fac775c3c446f207058b7c7a +sha256-linux=45d80e1723de5d25bb5b81f3ef5c82f583efc3e4f036a8cd2bb99e4f1eca9e51 From 4d226998e2b5b18aa9e16473062ad9f07f2f95f7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 18:30:32 +0800 Subject: [PATCH 10/19] fix(ilm): chunk large tier-delete dispatches (#7123) * fix(ilm): chunk large tier-delete dispatches * fix(ilm): bound tier-delete chunk dispatch stack use --- .../src/bucket/lifecycle/durable_namespace.rs | 226 ++- .../bucket/lifecycle/tier_delete_journal.rs | 1700 +++++++++++++++-- crates/ecstore/src/store/init.rs | 612 +++++- crates/ecstore/src/store/object.rs | 479 ++++- docs/architecture/compat-cleanup-register.md | 1 + .../ilm-tiering-persistence-contracts.md | 48 +- 6 files changed, 2765 insertions(+), 301 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs index a8b6262fb..d596ef35f 100644 --- a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -182,6 +182,16 @@ pub(crate) enum DurableIlmRecordCheckpoint { identity_sha256: String, state: tier_delete_journal::TierDeleteDispatchManifestState, }, + TierDeleteDispatchParent { + content_sha256: String, + identity_sha256: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + active_chunk_identity_sha256: Option, + completed: bool, + }, TransitionTransaction { content_sha256: String, identity_sha256: String, @@ -220,6 +230,7 @@ impl DurableIlmRecordCheckpoint { match self { Self::TierDeleteJournal { content_sha256, .. } | Self::TierDeleteDispatchManifest { content_sha256, .. } + | Self::TierDeleteDispatchParent { content_sha256, .. } | Self::TransitionTransaction { content_sha256, .. } | Self::ManualTransitionJob { content_sha256, .. } | Self::ManualTransitionScope { content_sha256, .. } @@ -341,6 +352,52 @@ impl DurableIlmRecordCheckpoint { (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed) ) } + ( + Self::TierDeleteDispatchParent { + identity_sha256: previous_identity, + revision: previous_revision, + next_chunk_sequence: previous_sequence, + completed_journal_count: previous_completed_journals, + active_chunk_identity_sha256: previous_active, + completed: previous_completed, + .. + }, + Self::TierDeleteDispatchParent { + identity_sha256: next_identity, + revision: next_revision, + next_chunk_sequence: next_sequence, + completed_journal_count: next_completed_journals, + active_chunk_identity_sha256: next_active, + completed: next_completed, + .. + }, + ) => { + let Some((sequence_delta, completed_journal_delta)) = tier_delete_dispatch_parent_progress_delta( + *previous_sequence, + *previous_completed_journals, + *next_sequence, + *next_completed_journals, + ) else { + return Err(Error::other("durable ILM record generation is not a monotonic successor")); + }; + let same_position_transition = sequence_delta == 0 + && completed_journal_delta == 0 + && matches!( + (previous_active.as_ref(), next_active.as_ref(), previous_completed, next_completed), + (None, Some(_), false, false) | (Some(_), None, false, false) | (None, None, false, true) + ); + let progress_transition = sequence_delta > 0 + && completed_journal_delta > 0 + && !matches!( + (previous_active.as_ref(), next_active.as_ref()), + (Some(previous), Some(next)) if previous == next + ); + previous_identity == next_identity + && !previous_completed + && next_revision > previous_revision + && (!next_completed || next_active.is_none()) + && (same_position_transition || progress_transition) + } ( Self::TransitionTransaction { identity_sha256: previous_identity, @@ -475,11 +532,58 @@ impl DurableIlmRecordCheckpoint { .. }, ) => previous_identity == terminal_identity, + ( + Self::TierDeleteDispatchParent { + identity_sha256: previous_identity, + revision: previous_revision, + next_chunk_sequence: previous_sequence, + completed_journal_count: previous_completed_journals, + active_chunk_identity_sha256: previous_active, + completed: false, + .. + }, + Self::TierDeleteDispatchParent { + identity_sha256: terminal_identity, + revision: terminal_revision, + next_chunk_sequence: terminal_sequence, + completed_journal_count: terminal_completed_journals, + active_chunk_identity_sha256: None, + completed: true, + .. + }, + ) => { + previous_identity == terminal_identity + && terminal_revision > previous_revision + && tier_delete_dispatch_parent_progress_delta( + *previous_sequence, + *previous_completed_journals, + *terminal_sequence, + *terminal_completed_journals, + ) + .is_some_and(|(sequence_delta, completed_journal_delta)| { + if sequence_delta == 0 && completed_journal_delta == 0 { + previous_active.is_none() + } else { + sequence_delta > 0 && completed_journal_delta > 0 + } + }) + } _ => false, } } } +fn tier_delete_dispatch_parent_progress_delta( + previous_sequence: u64, + previous_completed_journals: u64, + next_sequence: u64, + next_completed_journals: u64, +) -> Option<(u64, u64)> { + let sequence_delta = next_sequence.checked_sub(previous_sequence)?; + let completed_journal_delta = next_completed_journals.checked_sub(previous_completed_journals)?; + (sequence_delta <= completed_journal_delta).then_some((sequence_delta, completed_journal_delta)) +} + fn transition_state_distance( from: transition_transaction::TransitionTransactionState, to: transition_transaction::TransitionTransactionState, @@ -913,17 +1017,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { - let (operation_id, identity_sha256, state) = - tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?; - ( - "operation_id", - hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), - DurableIlmRecordCheckpoint::TierDeleteDispatchManifest { - content_sha256, + match tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)? { + tier_delete_journal::TierDeleteDispatchDurableRecord::Manifest { + operation_id, identity_sha256, state, - }, - ) + } => ( + "operation_id", + hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), + DurableIlmRecordCheckpoint::TierDeleteDispatchManifest { + content_sha256, + identity_sha256, + state, + }, + ), + tier_delete_journal::TierDeleteDispatchDurableRecord::Parent { + operation_id, + identity_sha256, + revision, + next_chunk_sequence, + completed_journal_count, + active_chunk_identity_sha256, + completed, + } => ( + "operation_id", + hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), + DurableIlmRecordCheckpoint::TierDeleteDispatchParent { + content_sha256, + identity_sha256, + revision, + next_chunk_sequence, + completed_journal_count, + active_chunk_identity_sha256, + completed, + }, + ), + } } DurableIlmRecordKind::TransitionTransaction => { let transaction = transition_transaction::decode_transition_transaction_record(path, data) @@ -1143,6 +1272,85 @@ mod tests { assert!(aborted.validate_successor(&preparing).is_err()); } + #[test] + fn tier_delete_dispatch_parent_checkpoint_is_monotonic_across_chunks() { + let identity = "a".repeat(64); + let checkpoint = |revision, sequence, completed_journals, active: Option<&str>, completed| { + DurableIlmRecordCheckpoint::TierDeleteDispatchParent { + content_sha256: format!("{revision:064x}"), + identity_sha256: identity.clone(), + revision, + next_chunk_sequence: sequence, + completed_journal_count: completed_journals, + active_chunk_identity_sha256: active.map(ToOwned::to_owned), + completed, + } + }; + let idle = checkpoint(0, 0, 0, None, false); + let first_child = "b".repeat(64); + let second_child = "c".repeat(64); + let bound = checkpoint(1, 0, 0, Some(&first_child), false); + let advanced = checkpoint(2, 1, 2, None, false); + let next_bound = checkpoint(3, 1, 2, Some(&second_child), false); + let completed = checkpoint(4, 2, 3, None, true); + let terminal_after_more_chunks = checkpoint(6, 4, 7, None, true); + + idle.validate_successor(&bound).expect("an idle parent may bind one child"); + bound + .validate_successor(&advanced) + .expect("a completed child may advance the parent sequence"); + advanced + .validate_successor(&next_bound) + .expect("the next sequence may bind a new immutable child"); + next_bound + .validate_successor(&completed) + .expect("receipt progress may skip directly to a later terminal checkpoint"); + assert!( + bound.is_predecessor_of_terminal(&terminal_after_more_chunks), + "terminal cleanup may still recognize a valid multi-chunk predecessor" + ); + assert!( + advanced.is_predecessor_of_terminal(&terminal_after_more_chunks), + "terminal cleanup may still skip over later valid parent generations" + ); + assert!( + idle.validate_successor(&checkpoint(1, 0, 1, Some(&first_child), false)) + .is_err() + ); + assert!(bound.validate_successor(&checkpoint(2, 1, 0, None, false)).is_err()); + assert!( + bound.validate_successor(&checkpoint(2, 2, 1, None, false)).is_err(), + "sequence cannot advance beyond completed journal evidence" + ); + assert!( + advanced.validate_successor(&checkpoint(3, 1, 3, None, false)).is_err(), + "completed journal count cannot grow without a completed child sequence" + ); + assert!( + bound.validate_successor(&checkpoint(2, 0, 0, None, true)).is_err(), + "an active child cannot be marked completed without completion evidence" + ); + assert!( + bound + .validate_successor(&checkpoint(2, 0, 0, Some(&second_child), false)) + .is_err(), + "an active child cannot be replaced at the same parent sequence" + ); + assert!( + bound + .validate_successor(&checkpoint(2, 1, 1, Some(&first_child), false)) + .is_err(), + "sequence growth cannot retain the same active child identity" + ); + assert!( + !bound.is_predecessor_of_terminal(&checkpoint(2, 0, 0, None, true)), + "terminal cleanup must not treat an active child as completed without count evidence" + ); + assert!(completed.validate_successor(&checkpoint(5, 3, 4, None, true)).is_err()); + assert!(completed.validate_successor(&advanced).is_err()); + assert!(advanced.validate_successor(&idle).is_err()); + } + #[test] fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() { use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared}; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index d31295f5b..ed502e570 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -93,9 +93,127 @@ pub(crate) const TIER_DELETE_JOURNAL_LEGACY_PREFIX: &str = TIER_DELETE_JOURNAL_N pub(crate) const TIER_DELETE_JOURNAL_V6_PREFIX: &str = TIER_DELETE_JOURNAL_V6_NAMESPACE.prefix; pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_PREFIX: &str = "ilm/tier-delete-dispatch-manifests/"; const TIER_DELETE_DISPATCH_MANIFEST_VERSION: u8 = 1; +// RUSTFS_COMPAT_TODO(backlog-2133-tier-delete-chunk-parent): retain the v1 single-manifest reader and fail-closed root sentinel while supported rollback releases do not understand chunk parents. Remove after every supported rollback release validates the parent/child protocol and no retained v1 manifest remains. +const TIER_DELETE_DISPATCH_PARENT_VERSION: u8 = 1; +const TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE: &str = "chunked_parent"; +const TIER_DELETE_DISPATCH_CHUNK_PATH: &str = "chunks"; pub(crate) const MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE: usize = 32 * 1024 * 1024; const MAX_TIER_DELETE_DISPATCH_JOURNALS: usize = 200_000; +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST: AtomicUsize = AtomicUsize::new(0); + +pub(crate) fn tier_delete_dispatch_batch_limit() -> usize { + #[cfg(all(test, feature = "test-util"))] + { + let configured = TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST.load(Ordering::Acquire); + if configured != 0 { + return configured; + } + } + MAX_TIER_DELETE_DISPATCH_JOURNALS +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteDispatchBatchLimitGuard; + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteDispatchBatchLimitGuard { + pub(crate) fn install(limit: usize) -> Self { + assert!(limit > 0 && limit <= MAX_TIER_DELETE_DISPATCH_JOURNALS); + TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST + .compare_exchange(0, limit, Ordering::AcqRel, Ordering::Acquire) + .expect("tier delete dispatch batch limit test override must be exclusive"); + Self + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchBatchLimitGuard { + fn drop(&mut self) { + TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST.store(0, Ordering::Release); + } +} + +#[cfg(all(test, feature = "test-util"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TierDeleteChunkTestStage { + ParentPersisted, + ChildManifestPersisted, + ParentBound, + DispatchAuthorized, + LocalReplayCompleted, + ChildCompleted, + ParentProgressed, + FinalLocalDeletionCompleted, + ParentCompleted, +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteChunkTestBarrierState { + stage: TierDeleteChunkTestStage, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteChunkTestBarrier { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_CHUNK_TEST_BARRIER: OnceLock>>> = OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteChunkTestBarrier { + pub(crate) fn install(stage: TierDeleteChunkTestStage) -> Self { + let state = Arc::new(TierDeleteChunkTestBarrierState { + stage, + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = TIER_DELETE_CHUNK_TEST_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete chunk test barrier should not poison"); + assert!(slot.is_none(), "tier delete chunk test barrier must not already be installed"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + self.state.arrived.notified().await; + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteChunkTestBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + if let Some(slot) = TIER_DELETE_CHUNK_TEST_BARRIER.get() { + let mut slot = slot.lock().expect("tier delete chunk test barrier should not poison"); + if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) { + *slot = None; + } + } + } +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn tier_delete_chunk_test_pause(stage: TierDeleteChunkTestStage) { + let state = TIER_DELETE_CHUNK_TEST_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete chunk test barrier should not poison") + .as_ref() + .filter(|state| state.stage == stage) + .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + } +} + fn valid_tier_delete_topology_generation(generation: &str) -> bool { generation.len() == 64 && generation.bytes().all(|byte| byte.is_ascii_hexdigit()) } @@ -145,13 +263,14 @@ impl TierDeleteDispatchManifest { { return Err(Error::other("tier delete dispatch manifest is invalid")); } + let expected_journal_prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", self.operation_id.simple()); if !self.journal_names.windows(2).all(|pair| pair[0] < pair[1]) - || self.journal_names.iter().any(|name| { - let expected_prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", self.operation_id.simple()); - !name.starts_with(&expected_prefix) || !name.ends_with(".json") - }) + || self + .journal_names + .iter() + .any(|name| !name.starts_with(&expected_journal_prefix) || !name.ends_with(".json")) || tier_delete_dispatch_journal_set_digest(&self.journal_names) != self.journal_set_sha256 - || tier_delete_dispatch_manifest_object_name(&self.bucket, self.bucket_incarnation, &self.prefix) != object_name + || !tier_delete_dispatch_manifest_path_matches(self, object_name) { return Err(Error::other("tier delete dispatch manifest binding is invalid")); } @@ -159,6 +278,91 @@ impl TierDeleteDispatchManifest { } } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum TierDeleteDispatchParentState { + Active, + Completed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct TierDeleteDispatchChunkBinding { + sequence: u64, + operation_id: uuid::Uuid, + manifest_object: String, + journal_set_sha256: String, + journal_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct TierDeleteDispatchParent { + version: u8, + record_type: String, + operation_id: uuid::Uuid, + bucket: String, + bucket_incarnation: uuid::Uuid, + prefix: String, + topology_generation: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + active_chunk: Option, + state: TierDeleteDispatchParentState, +} + +impl TierDeleteDispatchParent { + fn validate(&self, object_name: &str) -> Result<()> { + let max_journal_count = u64::try_from(MAX_TIER_DELETE_DISPATCH_JOURNALS) + .map_err(|_| Error::other("tier delete dispatch journal limit is not representable"))?; + if self.version != TIER_DELETE_DISPATCH_PARENT_VERSION + || self.record_type != TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE + || self.operation_id.is_nil() + || self.bucket.is_empty() + || self.bucket_incarnation.is_nil() + || !valid_tier_delete_topology_generation(&self.topology_generation) + || tier_delete_dispatch_manifest_object_name(&self.bucket, self.bucket_incarnation, &self.prefix) != object_name + || (self.state == TierDeleteDispatchParentState::Completed && self.active_chunk.is_some()) + || self.next_chunk_sequence > self.revision + || self.completed_journal_count < self.next_chunk_sequence + || ((self.next_chunk_sequence == 0) != (self.completed_journal_count == 0)) + { + return Err(Error::other("tier delete dispatch parent binding is invalid")); + } + if let Some(chunk) = &self.active_chunk + && (self.state != TierDeleteDispatchParentState::Active + || chunk.sequence != self.next_chunk_sequence + || chunk.operation_id.is_nil() + || chunk.journal_count == 0 + || chunk.journal_count > max_journal_count + || chunk.manifest_object + != tier_delete_dispatch_chunk_manifest_object_name( + &self.bucket, + self.bucket_incarnation, + &self.prefix, + chunk.operation_id, + ) + || !rustfs_utils::crypto::is_sha256_checksum(&chunk.journal_set_sha256)) + { + return Err(Error::other("tier delete dispatch parent chunk binding is invalid")); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct TierDeleteDispatchParentAdvance { + parent_manifest_object: String, + parent_operation_id: uuid::Uuid, + chunk: TierDeleteDispatchChunkBinding, +} + +enum TierDeleteDispatchRecord { + Manifest(TierDeleteDispatchManifest), + Parent(TierDeleteDispatchParent), +} + fn tier_delete_dispatch_journal_set_digest(names: &[String]) -> String { let mut hasher = Sha256::new(); for name in names { @@ -168,19 +372,46 @@ fn tier_delete_dispatch_journal_set_digest(names: &[String]) -> String { rustfs_utils::crypto::hex(hasher.finalize().as_slice()) } -fn tier_delete_dispatch_manifest_object_name(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { +fn tier_delete_dispatch_manifest_digest(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { let mut hasher = Sha256::new(); hasher.update(bucket.as_bytes()); hasher.update([0]); hasher.update(incarnation.as_bytes()); hasher.update([0]); hasher.update(prefix.as_bytes()); + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +fn tier_delete_dispatch_manifest_object_name(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { format!( "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{}.json", - rustfs_utils::crypto::hex(hasher.finalize().as_slice()) + tier_delete_dispatch_manifest_digest(bucket, incarnation, prefix) ) } +fn tier_delete_dispatch_chunk_manifest_object_name( + bucket: &str, + incarnation: uuid::Uuid, + prefix: &str, + operation_id: uuid::Uuid, +) -> String { + format!( + "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{TIER_DELETE_DISPATCH_CHUNK_PATH}/{}/{}.json", + tier_delete_dispatch_manifest_digest(bucket, incarnation, prefix), + operation_id.simple() + ) +} + +fn tier_delete_dispatch_manifest_path_matches(manifest: &TierDeleteDispatchManifest, object_name: &str) -> bool { + let parent_digest = tier_delete_dispatch_manifest_digest(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix); + object_name == format!("{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{parent_digest}.json") + || object_name + == format!( + "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{TIER_DELETE_DISPATCH_CHUNK_PATH}/{parent_digest}/{}.json", + manifest.operation_id.simple() + ) +} + fn tier_delete_dispatch_operation_lock_name(manifest_object: &str) -> String { format!("{manifest_object}.operation-lock") } @@ -216,6 +447,15 @@ fn encode_tier_delete_dispatch_manifest(manifest: &TierDeleteDispatchManifest) - Ok(data) } +fn encode_tier_delete_dispatch_parent(parent: &TierDeleteDispatchParent) -> Result> { + let data = + serde_json::to_vec(parent).map_err(|err| Error::other_with_context("encode tier delete dispatch parent failed", err))?; + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch parent is too large")); + } + Ok(data) +} + fn decode_tier_delete_dispatch_manifest(data: &[u8], object_name: &str) -> Result { if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { return Err(Error::other("tier delete dispatch manifest is too large")); @@ -226,28 +466,94 @@ fn decode_tier_delete_dispatch_manifest(data: &[u8], object_name: &str) -> Resul Ok(manifest) } +fn decode_tier_delete_dispatch_record(data: &[u8], object_name: &str) -> Result { + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch record is too large")); + } + match decode_tier_delete_dispatch_manifest(data, object_name) { + Ok(manifest) => Ok(TierDeleteDispatchRecord::Manifest(manifest)), + Err(manifest_error) => { + let parent: TierDeleteDispatchParent = serde_json::from_slice(data).map_err(|parent_error| { + Error::other_with_context( + "decode tier delete dispatch record failed", + format!("manifest: {manifest_error}; parent: {parent_error}"), + ) + })?; + parent.validate(object_name)?; + Ok(TierDeleteDispatchRecord::Parent(parent)) + } + } +} + +pub(crate) enum TierDeleteDispatchDurableRecord { + Manifest { + operation_id: uuid::Uuid, + identity_sha256: String, + state: TierDeleteDispatchManifestState, + }, + Parent { + operation_id: uuid::Uuid, + identity_sha256: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + active_chunk_identity_sha256: Option, + completed: bool, + }, +} + pub(crate) fn validate_tier_delete_dispatch_manifest_record( object_name: &str, data: &[u8], -) -> Result<(uuid::Uuid, String, TierDeleteDispatchManifestState)> { - let manifest = decode_tier_delete_dispatch_manifest(data, object_name)?; - let identity = serde_json::to_vec(&( - manifest.version, - manifest.operation_id, - &manifest.bucket, - manifest.bucket_incarnation, - &manifest.prefix, - &manifest.journal_names, - &manifest.journal_set_sha256, - manifest.journal_count, - &manifest.topology_generation, - )) - .map_err(Error::other)?; - Ok(( - manifest.operation_id, - rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), - manifest.state, - )) +) -> Result { + match decode_tier_delete_dispatch_record(data, object_name)? { + TierDeleteDispatchRecord::Manifest(manifest) => { + let identity = serde_json::to_vec(&( + manifest.version, + manifest.operation_id, + &manifest.bucket, + manifest.bucket_incarnation, + &manifest.prefix, + &manifest.journal_names, + &manifest.journal_set_sha256, + manifest.journal_count, + &manifest.topology_generation, + )) + .map_err(Error::other)?; + Ok(TierDeleteDispatchDurableRecord::Manifest { + operation_id: manifest.operation_id, + identity_sha256: rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), + state: manifest.state, + }) + } + TierDeleteDispatchRecord::Parent(parent) => { + let identity = serde_json::to_vec(&( + parent.version, + &parent.record_type, + parent.operation_id, + &parent.bucket, + parent.bucket_incarnation, + &parent.prefix, + &parent.topology_generation, + )) + .map_err(Error::other)?; + let active_chunk_identity_sha256 = parent + .active_chunk + .as_ref() + .map(|chunk| serde_json::to_vec(chunk).map(|data| rustfs_utils::crypto::hex_sha256(&data, ToOwned::to_owned))) + .transpose() + .map_err(Error::other)?; + Ok(TierDeleteDispatchDurableRecord::Parent { + operation_id: parent.operation_id, + identity_sha256: rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), + revision: parent.revision, + next_chunk_sequence: parent.next_chunk_sequence, + completed_journal_count: parent.completed_journal_count, + active_chunk_identity_sha256, + completed: parent.state == TierDeleteDispatchParentState::Completed, + }) + } + } } /// Return the fleet generation durably bound to a v6 manifest or journal. @@ -255,7 +561,12 @@ pub(crate) fn validate_tier_delete_dispatch_manifest_record( /// cleanup compatibility behavior. pub(crate) fn durable_ilm_v6_topology_generation(object_name: &str, data: &[u8]) -> Result> { if object_name.starts_with(TIER_DELETE_DISPATCH_MANIFEST_PREFIX) { - return decode_tier_delete_dispatch_manifest(data, object_name).map(|manifest| Some(manifest.topology_generation)); + return decode_tier_delete_dispatch_record(data, object_name).map(|record| { + Some(match record { + TierDeleteDispatchRecord::Manifest(manifest) => manifest.topology_generation, + TierDeleteDispatchRecord::Parent(parent) => parent.topology_generation, + }) + }); } if object_name.starts_with(TIER_DELETE_JOURNAL_V6_PREFIX) { let entry = decode_tier_delete_journal_entry(data)?; @@ -302,10 +613,12 @@ pub(crate) fn test_tier_delete_dispatch_manifest_record( } struct DispatchedJournalPermit { + manifest_object: String, manifest: TierDeleteDispatchManifest, authorized_etag: String, entries: Vec, fleet_proof: TierDeleteJournalFleetProofToken, + parent_advance: Option, } struct TierDeleteDispatchAuthorizationInner { @@ -423,31 +736,68 @@ pub(crate) struct PreparedTierDeleteDispatch { predecessor_replay_required: bool, } +pub(crate) enum TierDeleteChunkParentInspection { + NoParent, + LegacyManifest, + Ready(String), + Resume(Box), + RetryRequired, +} + pub(crate) struct ActiveTierDeleteDispatch { + manifest_object: String, manifest: TierDeleteDispatchManifest, authorized_etag: String, entries: Arc<[Jentry]>, authorization: TierDeleteDispatchAuthorization, predecessor_replay_required: bool, + parent_advance: Option, } impl PreparedTierDeleteDispatch { + pub(crate) fn entries(&self) -> Result<&[Jentry]> { + self.permit + .as_ref() + .map(|permit| permit.entries.as_slice()) + .ok_or_else(|| Error::other("tier delete dispatch permit was already consumed")) + } + + pub(crate) fn require_exact_predecessor_replay(&mut self) { + self.predecessor_replay_required = true; + } + pub(crate) fn consume(mut self, bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> Result { let permit = self .permit .take() .ok_or_else(|| Error::other("tier delete dispatch permit was already consumed"))?; - let manifest_object = tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix); if permit.manifest.state != TierDeleteDispatchManifestState::DispatchAuthorized || permit.manifest.bucket != bucket || permit.manifest.bucket_incarnation != incarnation || permit.manifest.prefix != prefix - || permit.entries.iter().map(tier_delete_journal_object_name).collect::>() != permit.manifest.journal_names + || permit.manifest.validate(&permit.manifest_object).is_err() + || permit.entries.len() != permit.manifest.journal_names.len() + || !permit + .entries + .iter() + .zip(&permit.manifest.journal_names) + .all(|(entry, name)| tier_delete_journal_object_name(entry) == name.as_str()) || !tier_delete_journal_fleet_proof_matches(&permit.fleet_proof) || tier_delete_journal_topology_generation(&permit.fleet_proof) != permit.manifest.topology_generation + || permit.parent_advance.as_ref().is_some_and(|parent| { + parent.chunk.manifest_object != permit.manifest_object + || parent.chunk.operation_id != permit.manifest.operation_id + || parent.chunk.journal_set_sha256 != permit.manifest.journal_set_sha256 + || parent.chunk.journal_count != permit.manifest.journal_count + }) { return Err(Error::other("tier delete dispatch permit validation failed")); } + if permit.parent_advance.is_none() + && permit.manifest_object != tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix) + { + return Err(Error::other("unbound tier delete child dispatch cannot authorize local mutation")); + } let entries = Arc::<[Jentry]>::from(permit.entries); let journal_entry_indexes = permit .manifest @@ -458,7 +808,7 @@ impl PreparedTierDeleteDispatch { .map(|(index, name)| (name, index)) .collect(); let authorization = TierDeleteDispatchAuthorization(Arc::new(TierDeleteDispatchAuthorizationInner { - manifest_object, + manifest_object: permit.manifest_object.clone(), operation_id: permit.manifest.operation_id, bucket: permit.manifest.bucket.clone(), bucket_incarnation: permit.manifest.bucket_incarnation, @@ -471,11 +821,13 @@ impl PreparedTierDeleteDispatch { mutation_started: AtomicBool::new(false), })); Ok(ActiveTierDeleteDispatch { + manifest_object: permit.manifest_object, manifest: permit.manifest, authorized_etag: permit.authorized_etag, entries, authorization, predecessor_replay_required: self.predecessor_replay_required, + parent_advance: permit.parent_advance, }) } } @@ -492,6 +844,10 @@ impl ActiveTierDeleteDispatch { pub(crate) fn predecessor_replay_required(&self) -> bool { self.predecessor_replay_required } + + pub(crate) fn is_chunked(&self) -> bool { + self.parent_advance.is_some() + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -829,6 +1185,22 @@ async fn read_tier_delete_dispatch_manifest( } } +async fn read_tier_delete_dispatch_record( + api: Arc, + object_name: &str, +) -> Result> { + match config_boundary::read_config_with_metadata(api, object_name, &ObjectOptions::default()).await { + Ok((data, metadata)) => { + let etag = metadata + .etag + .ok_or_else(|| Error::other("tier delete dispatch record has no entity tag"))?; + Ok(Some((decode_tier_delete_dispatch_record(&data, object_name)?, etag))) + } + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => Ok(None), + Err(err) => Err(err), + } +} + async fn read_tier_delete_journal_with_etag(api: Arc, name: &str) -> Result> { match config_boundary::read_config_with_metadata(api, name, &ObjectOptions::default()).await { Ok((data, metadata)) => { @@ -1638,6 +2010,15 @@ async fn record_tier_delete_dispatch_manifest_progress_fenced( record_durable_config_progress_fenced(api, name, &encode_tier_delete_dispatch_manifest(manifest)?, fences_current).await } +async fn record_tier_delete_dispatch_parent_progress_fenced( + api: Arc, + name: &str, + parent: &TierDeleteDispatchParent, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + record_durable_config_progress_fenced(api, name, &encode_tier_delete_dispatch_parent(parent)?, fences_current).await +} + async fn delete_durable_config_if_match( api: Arc, name: &str, @@ -1835,7 +2216,7 @@ fn bind_dispatch_entries( topology_generation: &str, ) -> Result> { let mut entries = entries; - entries.sort_by_key(|entry| tier_delete_journal_v6_object_name(entry, operation_id)); + entries.sort_by_cached_key(|entry| tier_delete_journal_v6_object_name(entry, operation_id)); let mut bound = Vec::with_capacity(entries.len()); let mut last_name: Option = None; for mut entry in entries { @@ -1887,20 +2268,24 @@ fn tier_delete_dispatch_desired_names(entries: &[Jentry], operation_id: uuid::Uu Ok(names) } -fn validate_bound_journal(manifest: &TierDeleteDispatchManifest, name: &str, entry: &Jentry) -> Result<()> { +fn validate_bound_journal( + manifest: &TierDeleteDispatchManifest, + manifest_object: &str, + name: &str, + entry: &Jentry, +) -> Result<()> { + // Callers pass a manifest already validated at its decode/create boundary. + // Keep this member check O(1); rehashing the whole manifest here would be + // quadratic across a maximum-sized batch. + let dispatch_matches = entry.dispatch.as_ref().is_some_and(|binding| { + binding.operation_id == manifest.operation_id + && binding.manifest_object == manifest_object + && binding.journal_set_sha256 == manifest.journal_set_sha256 + && binding.topology_generation == manifest.topology_generation + }); if entry.persisted_version != TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION || tier_delete_journal_object_name(entry) != name - || entry.dispatch.as_ref() - != Some(&TierDeleteDispatchBinding { - operation_id: manifest.operation_id, - manifest_object: tier_delete_dispatch_manifest_object_name( - &manifest.bucket, - manifest.bucket_incarnation, - &manifest.prefix, - ), - journal_set_sha256: manifest.journal_set_sha256.clone(), - topology_generation: manifest.topology_generation.clone(), - }) + || !dispatch_matches { return Err(Error::other("tier delete journal does not match its dispatch manifest")); } @@ -1909,6 +2294,7 @@ fn validate_bound_journal(manifest: &TierDeleteDispatchManifest, name: &str, ent async fn load_complete_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, allowed_states: &[TierDeleteJournalState], fences_current: &impl Fn() -> bool, @@ -1920,7 +2306,7 @@ async fn load_complete_dispatch_journal_set( let (entry, _) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("tier delete dispatch manifest references a missing journal"))?; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !allowed_states.contains(&entry.state) { return Err(Error::other("tier delete dispatch journal has an invalid state")); } @@ -1949,6 +2335,7 @@ async fn load_complete_dispatch_journal_set( async fn persist_prepared_dispatch_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, entry: &Jentry, fences_current: &impl Fn() -> bool, @@ -1956,7 +2343,7 @@ async fn persist_prepared_dispatch_journal( let name = tier_delete_journal_object_name(entry); for _ in 0..4 { if let Some((current, _)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? { - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, entry) { return Err(Error::other("tier delete journal key is occupied by another cleanup identity")); } @@ -1978,6 +2365,7 @@ async fn persist_prepared_dispatch_journal( async fn dispatch_prepared_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, expected: &Jentry, fences_current: &impl Fn() -> bool, @@ -1987,7 +2375,7 @@ async fn dispatch_prepared_journal( let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("prepared tier delete journal disappeared before dispatch"))?; - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, expected) { return Err(Error::other("prepared tier delete journal changed identity before dispatch")); } @@ -2023,6 +2411,7 @@ fn ensure_tier_delete_dispatch_member_scan_fence(fences_current: &impl Fn() -> b async fn validate_staged_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result<()> { @@ -2046,7 +2435,7 @@ async fn validate_staged_dispatch_journal_set( let Some((entry, _)) = observed else { return Ok(()); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); } @@ -2090,6 +2479,7 @@ async fn validate_staged_dispatch_journal_set( async fn delete_staged_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &F, ) -> Result<()> @@ -2099,7 +2489,7 @@ where // Validate the complete immutable set before deleting its first member so // a corrupt binding or impossible Committed state quarantines the whole // operation rather than producing a partial rollback. - validate_staged_dispatch_journal_set(api.clone(), manifest, fences_current).await?; + validate_staged_dispatch_journal_set(api.clone(), manifest_object, manifest, fences_current).await?; // The validation barrier above must drain before this stream is created. // Once deletion starts, stop admitting useful work after the first error @@ -2123,7 +2513,7 @@ where let Some((entry, etag)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? else { return Ok(()); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); } @@ -2206,19 +2596,18 @@ where async fn seal_and_rollback_preparing_dispatch( api: Arc, + manifest_name: &str, expected: &TierDeleteDispatchManifest, fleet_proof: &TierDeleteJournalFleetProofToken, bucket_fence: &crate::object_api::NamespaceLockFence, operation_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result<()> { - let manifest_name = - tier_delete_dispatch_manifest_object_name(&expected.bucket, expected.bucket_incarnation, &expected.prefix); let fences_current = || dispatch_write_fences_current(bucket_fence, operation_guard, fleet_proof, &expected.topology_generation); if !fences_current() { return Err(Error::other("tier delete dispatch rollback fence changed")); } - let Some((mut current, etag)) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? else { + let Some((mut current, etag)) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await? else { return Ok(()); }; if current.operation_id != expected.operation_id @@ -2235,26 +2624,26 @@ async fn seal_and_rollback_preparing_dispatch( } save_config_if_match_fenced( api.clone(), - &manifest_name, + manifest_name, encode_tier_delete_dispatch_manifest(¤t)?, &etag, &fences_current, ) .await?; - let (mut sealed, sealed_etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + let (mut sealed, sealed_etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await? .ok_or_else(|| Error::other("sealed tier delete dispatch manifest disappeared"))?; if sealed.operation_id != expected.operation_id || sealed.state != TierDeleteDispatchManifestState::Aborting { return Err(Error::other("tier delete dispatch rollback seal changed")); } - delete_staged_dispatch_journal_set(api.clone(), &sealed, &fences_current).await?; + delete_staged_dispatch_journal_set(api.clone(), manifest_name, &sealed, &fences_current).await?; sealed.state = TierDeleteDispatchManifestState::Aborted; if !fences_current() { return Err(Error::other("tier delete dispatch rollback fence changed")); } save_config_if_match_fenced( api, - &manifest_name, + manifest_name, encode_tier_delete_dispatch_manifest(&sealed)?, &sealed_etag, &fences_current, @@ -2269,6 +2658,19 @@ async fn authorized_dispatch_permit( bucket_fence: &crate::object_api::NamespaceLockFence, operation_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result { + authorized_dispatch_permit_with_parent(api, manifest_name, fleet_proof, bucket_fence, operation_guard, None, None).await +} + +async fn authorized_dispatch_permit_with_parent( + api: Arc, + manifest_name: &str, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + parent_operation_guard: Option<&rustfs_lock::NamespaceLockGuard>, + parent_advance: Option, +) -> Result { + let is_chunked = parent_advance.is_some(); let (manifest, authorized_etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await? .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared"))?; @@ -2279,11 +2681,14 @@ async fn authorized_dispatch_permit( return Err(Error::other("tier delete dispatch authorization is stale or unconfirmed")); } let entries = { - let fences_current = - || dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation); + let fences_current = || { + dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation) + && parent_operation_guard.is_none_or(|guard| !guard.is_lock_lost()) + }; record_tier_delete_dispatch_manifest_progress_fenced(api.clone(), manifest_name, &manifest, &fences_current).await?; load_complete_dispatch_journal_set( api, + manifest_name, &manifest, &[TierDeleteJournalState::Dispatched, TierDeleteJournalState::Committed], &fences_current, @@ -2292,15 +2697,444 @@ async fn authorized_dispatch_permit( }; Ok(PreparedTierDeleteDispatch { permit: Some(DispatchedJournalPermit { + manifest_object: manifest_name.to_string(), manifest, authorized_etag, entries, fleet_proof, + parent_advance, }), - predecessor_replay_required: false, + predecessor_replay_required: is_chunked, }) } +fn tier_delete_dispatch_child_matches_parent( + parent: &TierDeleteDispatchParent, + binding: &TierDeleteDispatchChunkBinding, + child: &TierDeleteDispatchManifest, +) -> bool { + // Every caller supplies a child decoded against `binding.manifest_object`, + // which already verifies the path, sorted member set, and full digest. + // Keep the parent comparison O(1) for a maximum-sized child. + child.operation_id == binding.operation_id + && child.bucket == parent.bucket + && child.bucket_incarnation == parent.bucket_incarnation + && child.prefix == parent.prefix + && child.topology_generation == parent.topology_generation + && child.journal_set_sha256 == binding.journal_set_sha256 + && child.journal_count == binding.journal_count +} + +fn tier_delete_dispatch_parent_fences_current( + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + parent: &TierDeleteDispatchParent, +) -> bool { + dispatch_write_fences_current(bucket_fence, operation_guard, fleet_proof, &parent.topology_generation) +} + +async fn cas_tier_delete_dispatch_parent( + api: Arc, + object_name: &str, + expected: &TierDeleteDispatchParent, + expected_etag: &str, + next: &TierDeleteDispatchParent, + fences_current: &impl Fn() -> bool, +) -> Result<(TierDeleteDispatchParent, String)> { + expected.validate(object_name)?; + next.validate(object_name)?; + if expected.operation_id != next.operation_id || !fences_current() { + return Err(Error::other("tier delete dispatch parent fence changed before progress")); + } + let write = save_config_if_match_fenced( + api.clone(), + object_name, + encode_tier_delete_dispatch_parent(next)?, + expected_etag, + fences_current, + ) + .await; + if write.as_ref().is_err_and(is_decommission_checkpoint_targets_incomplete) { + return Err(write.expect_err("decommission checkpoint result should remain an error")); + } + let observed = read_tier_delete_dispatch_record(api, object_name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch parent fence changed during progress")); + } + match observed { + Some((TierDeleteDispatchRecord::Parent(observed), etag)) if observed == *next => Ok((observed, etag)), + _ => match write { + Ok(()) | Err(Error::PreconditionFailed) => Err(Error::other("tier delete dispatch parent changed during progress")), + Err(err) => Err(err), + }, + } +} + +async fn tier_delete_dispatch_chunk_journal_namespace_empty(api: Arc, operation_id: uuid::Uuid) -> Result { + let prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", operation_id.simple()); + let list = api + .list_objects_v2(RUSTFS_META_BUCKET, &prefix, None, None, 1, false, None, false) + .await?; + Ok(list.objects.is_empty()) +} + +pub(crate) async fn inspect_tier_delete_chunk_parent( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let Some((record, _)) = read_tier_delete_dispatch_record(api.clone(), &parent_name).await? else { + return Ok(TierDeleteChunkParentInspection::NoParent); + }; + let TierDeleteDispatchRecord::Parent(observed_parent) = record else { + return Ok(TierDeleteChunkParentInspection::LegacyManifest); + }; + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if observed_parent.bucket != bucket + || observed_parent.bucket_incarnation != bucket_incarnation + || observed_parent.prefix != prefix + || tier_delete_journal_topology_generation(&fleet_proof) != observed_parent.topology_generation + { + return Err(Error::other("an incompatible tier delete chunk parent already owns this prefix")); + } + // Lock order: caller-held bucket lifecycle WRITE -> parent operation -> + // child operation (only while reconstructing an Authorized child permit). + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = match read_tier_delete_dispatch_record(api.clone(), &parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("tier delete chunk parent was replaced by a legacy manifest")); + } + None => return Ok(TierDeleteChunkParentInspection::RetryRequired), + }; + if parent != observed_parent + || !tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent) + { + return Err(Error::other("tier delete chunk parent changed during inspection")); + } + { + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &parent_name, &parent, &fences_current).await?; + } + if parent.state == TierDeleteDispatchParentState::Completed { + return Ok(TierDeleteChunkParentInspection::RetryRequired); + } + let Some(binding) = parent.active_chunk.clone() else { + return Ok(TierDeleteChunkParentInspection::Ready(parent.topology_generation.clone())); + }; + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + let child = read_tier_delete_dispatch_manifest(api.clone(), &binding.manifest_object).await?; + let Some((child, _)) = child else { + if !tier_delete_dispatch_chunk_journal_namespace_empty(api.clone(), binding.operation_id).await? { + return Err(Error::other("tier delete chunk parent references a missing child with retained journals")); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &parent_etag, &next, &fences_current).await?; + return Ok(TierDeleteChunkParentInspection::RetryRequired); + }; + if !tier_delete_dispatch_child_matches_parent(&parent, &binding, &child) { + return Err(Error::other("tier delete chunk parent child binding changed")); + } + match child.state { + TierDeleteDispatchManifestState::DispatchAuthorized => { + let child_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&binding.manifest_object)) + .await?; + let child_guard = child_lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let advance = TierDeleteDispatchParentAdvance { + parent_manifest_object: parent_name, + parent_operation_id: parent.operation_id, + chunk: binding, + }; + let child_manifest_object = advance.chunk.manifest_object.clone(); + let prepared = authorized_dispatch_permit_with_parent( + api, + &child_manifest_object, + fleet_proof, + bucket_fence, + &child_guard, + Some(&parent_guard), + Some(advance), + ) + .await?; + Ok(TierDeleteChunkParentInspection::Resume(Box::new(prepared))) + } + TierDeleteDispatchManifestState::Completed => { + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(binding.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &parent_etag, &next, &fences_current).await?; + Ok(TierDeleteChunkParentInspection::RetryRequired) + } + TierDeleteDispatchManifestState::Preparing + | TierDeleteDispatchManifestState::Aborting + | TierDeleteDispatchManifestState::Aborted => Ok(TierDeleteChunkParentInspection::RetryRequired), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn prepare_tier_delete_chunk_dispatch( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + create_parent_if_missing: bool, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + if entries.is_empty() + || entries.len() > tier_delete_dispatch_batch_limit() + || bucket_incarnation.is_nil() + || bucket_fence.is_lock_lost() + || !tier_delete_journal_fleet_proof_matches(&fleet_proof) + { + return Err(Error::other("tier delete chunk dispatch input is invalid or stale")); + } + let topology_generation = tier_delete_journal_topology_generation(&fleet_proof); + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + // Lock order: caller-held bucket lifecycle WRITE -> parent operation -> + // newly created child operation. Completion releases the child lock before + // reacquiring the parent lock, so no child -> parent nesting exists. + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = loop { + let observed = read_tier_delete_dispatch_record(api.clone(), &parent_name).await?; + match observed { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => break (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("a legacy tier delete dispatch already owns this prefix")); + } + None if !create_parent_if_missing => { + return Err(Error::other("tier delete chunk parent disappeared before batch creation")); + } + None => { + let parent = TierDeleteDispatchParent { + version: TIER_DELETE_DISPATCH_PARENT_VERSION, + record_type: TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE.to_string(), + operation_id: uuid::Uuid::new_v4(), + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + topology_generation: topology_generation.clone(), + revision: 0, + next_chunk_sequence: 0, + completed_journal_count: 0, + active_chunk: None, + state: TierDeleteDispatchParentState::Active, + }; + parent.validate(&parent_name)?; + let fences_current = + || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + match save_config_if_none_fenced( + api.clone(), + &parent_name, + encode_tier_delete_dispatch_parent(&parent)?, + &fences_current, + ) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + } + }; + if parent.bucket != bucket + || parent.bucket_incarnation != bucket_incarnation + || parent.prefix != prefix + || parent.topology_generation != topology_generation + || parent.state != TierDeleteDispatchParentState::Active + || parent.active_chunk.is_some() + { + return Err(Error::other("tier delete chunk parent is not ready for a successor batch")); + } + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentPersisted).await; + let child_operation_id = uuid::Uuid::new_v4(); + let journal_names = tier_delete_dispatch_desired_names(&entries, child_operation_id)?; + let journal_set_sha256 = tier_delete_dispatch_journal_set_digest(&journal_names); + let child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, bucket_incarnation, prefix, child_operation_id); + let child = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id: child_operation_id, + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + journal_count: journal_names + .len() + .try_into() + .map_err(|_| Error::other("tier delete chunk journal count is not representable"))?, + journal_names, + journal_set_sha256: journal_set_sha256.clone(), + topology_generation, + state: TierDeleteDispatchManifestState::Preparing, + }; + let child_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&child_name)) + .await?; + let child_guard = child_lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let (child_etag, binding) = { + let write_fences_current = || { + tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent) + && !child_guard.is_lock_lost() + }; + match save_config_if_none_fenced( + api.clone(), + &child_name, + encode_tier_delete_dispatch_manifest(&child)?, + &write_fences_current, + ) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => {} + Err(err) => return Err(err), + } + let (observed_child, child_etag) = read_tier_delete_dispatch_manifest(api.clone(), &child_name) + .await? + .ok_or_else(|| Error::other("tier delete chunk manifest disappeared after creation"))?; + if observed_child != child || !write_fences_current() { + return Err(Error::other("tier delete chunk manifest changed during creation")); + } + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ChildManifestPersisted).await; + let binding = TierDeleteDispatchChunkBinding { + sequence: parent.next_chunk_sequence, + operation_id: child_operation_id, + manifest_object: child_name.clone(), + journal_set_sha256, + journal_count: child.journal_count, + }; + let mut bound_parent = parent.clone(); + bound_parent.revision = bound_parent + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + bound_parent.active_chunk = Some(binding.clone()); + if let Err(err) = cas_tier_delete_dispatch_parent( + api.clone(), + &parent_name, + &parent, + &parent_etag, + &bound_parent, + &write_fences_current, + ) + .await + { + let _ = + seal_and_rollback_preparing_dispatch(api, &child_name, &child, &fleet_proof, bucket_fence, &child_guard).await; + return Err(err); + } + (child_etag, binding) + }; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentBound).await; + let parent_advance = TierDeleteDispatchParentAdvance { + parent_manifest_object: parent_name, + parent_operation_id: parent.operation_id, + chunk: binding, + }; + let prepared = finish_preparing_tier_delete_dispatch( + api, + &child_name, + child, + child_etag, + entries, + fleet_proof, + bucket_fence, + &child_guard, + Some(&parent_guard), + Some(parent_advance), + ) + .await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::DispatchAuthorized).await; + Ok(prepared) +} + +pub(crate) async fn complete_tier_delete_chunk_parent( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + bucket_fence: &crate::object_api::NamespaceLockFence, + fleet_proof: &TierDeleteJournalFleetProofToken, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let Some((record, _)) = read_tier_delete_dispatch_record(api.clone(), &parent_name).await? else { + return Ok(false); + }; + if matches!(record, TierDeleteDispatchRecord::Manifest(_)) { + return Ok(false); + } + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, etag) = match read_tier_delete_dispatch_record(api.clone(), &parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + _ => return Err(Error::other("tier delete chunk parent changed before final completion")), + }; + if parent.bucket != bucket + || parent.bucket_incarnation != bucket_incarnation + || parent.prefix != prefix + || parent.active_chunk.is_some() + || tier_delete_journal_topology_generation(fleet_proof) != parent.topology_generation + { + return Err(Error::other("tier delete chunk parent is not ready for final completion")); + } + if !tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, fleet_proof, &parent) { + return Err(Error::other("tier delete chunk parent completion fence changed")); + } + if parent.state == TierDeleteDispatchParentState::Completed { + return Ok(true); + } + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, fleet_proof, &parent); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &parent_name, &parent, &fences_current).await?; + let mut completed = parent.clone(); + completed.revision = completed + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + completed.state = TierDeleteDispatchParentState::Completed; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &etag, &completed, &fences_current).await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentCompleted).await; + Ok(true) +} + pub(crate) async fn prepare_tier_delete_dispatch( api: Arc, bucket: &str, @@ -2318,10 +3152,34 @@ pub(crate) async fn prepare_tier_delete_dispatch( entries, fleet_proof, bucket_fence, + true, )) .await } +pub(crate) async fn resume_tier_delete_dispatch( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + Box::pin(prepare_tier_delete_dispatch_inner( + api, + bucket, + bucket_incarnation, + prefix, + entries, + fleet_proof, + bucket_fence, + false, + )) + .await +} + +#[allow(clippy::too_many_arguments)] async fn prepare_tier_delete_dispatch_inner( api: Arc, bucket: &str, @@ -2330,6 +3188,7 @@ async fn prepare_tier_delete_dispatch_inner( entries: Vec, fleet_proof: TierDeleteJournalFleetProofToken, bucket_fence: &crate::object_api::NamespaceLockFence, + create_if_missing: bool, ) -> Result { if bucket_incarnation.is_nil() || bucket_fence.is_lock_lost() || !tier_delete_journal_fleet_proof_matches(&fleet_proof) { return Err(Error::other("tier delete journal v6 fleet capability is unavailable")); @@ -2344,7 +3203,7 @@ async fn prepare_tier_delete_dispatch_inner( .await?; ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; - let (mut manifest, manifest_etag) = loop { + let (manifest, manifest_etag) = loop { ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? { Some((existing, etag)) => { @@ -2394,6 +3253,11 @@ async fn prepare_tier_delete_dispatch_inner( } } None => { + if !create_if_missing { + return Err(Error::other( + "legacy tier delete dispatch disappeared; retry to establish a chunk parent if still required", + )); + } let operation_id = uuid::Uuid::new_v4(); let desired_names = tier_delete_dispatch_desired_names(&entries, operation_id)?; let desired_digest = tier_delete_dispatch_journal_set_digest(&desired_names); @@ -2426,18 +3290,48 @@ async fn prepare_tier_delete_dispatch_inner( } }; + finish_preparing_tier_delete_dispatch( + api, + &manifest_name, + manifest, + manifest_etag, + entries, + fleet_proof, + bucket_fence, + &operation_guard, + None, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn finish_preparing_tier_delete_dispatch( + api: Arc, + manifest_name: &str, + mut manifest: TierDeleteDispatchManifest, + manifest_etag: String, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + parent_operation_guard: Option<&rustfs_lock::NamespaceLockGuard>, + parent_advance: Option, +) -> Result { let bound = bind_dispatch_entries( entries, manifest.operation_id, - &manifest_name, + manifest_name, &manifest.journal_set_sha256, &manifest.topology_generation, )?; let attempt = async { - let fences_current = - || dispatch_write_fences_current(bucket_fence, &operation_guard, &fleet_proof, &manifest.topology_generation); + let fences_current = || { + dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation) + && parent_operation_guard.is_none_or(|guard| !guard.is_lock_lost()) + }; let manifest_ref = &manifest; - let operation_guard_ref = &operation_guard; + let operation_guard_ref = operation_guard; let prepare_stopped = Arc::new(AtomicBool::new(false)); let mut prepare_writes = futures::stream::iter((0..bound.len()).map(|index| { let api = api.clone(); @@ -2451,7 +3345,7 @@ async fn prepare_tier_delete_dispatch_inner( return Ok(()); } let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { - Ok(()) => persist_prepared_dispatch_journal(api, manifest, &entry, fences_current).await, + Ok(()) => persist_prepared_dispatch_journal(api, manifest_name, manifest, &entry, fences_current).await, Err(err) => Err(err), }; if result.is_err() { @@ -2491,7 +3385,7 @@ async fn prepare_tier_delete_dispatch_inner( return Ok(()); } let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { - Ok(()) => dispatch_prepared_journal(api, manifest, &entry, fences_current) + Ok(()) => dispatch_prepared_journal(api, manifest_name, manifest, &entry, fences_current) .await .map(|_| ()), Err(err) => Err(err), @@ -2515,7 +3409,7 @@ async fn prepare_tier_delete_dispatch_inner( if let Some(err) = dispatch_error { return Err(err); } - ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + ensure_dispatch_lock_current(bucket_fence, operation_guard)?; if !tier_delete_journal_fleet_proof_matches(&fleet_proof) || tier_delete_journal_topology_generation(&fleet_proof) != manifest.topology_generation { @@ -2523,7 +3417,7 @@ async fn prepare_tier_delete_dispatch_inner( } manifest.state = TierDeleteDispatchManifestState::DispatchAuthorized; let authorized_data = encode_tier_delete_dispatch_manifest(&manifest)?; - match save_config_if_match_fenced(api.clone(), &manifest_name, authorized_data, &manifest_etag, &fences_current).await { + match save_config_if_match_fenced(api.clone(), manifest_name, authorized_data, &manifest_etag, &fences_current).await { Ok(()) => {} Err(Error::PreconditionFailed) => { return Err(Error::other("tier delete dispatch manifest changed before authorization")); @@ -2533,7 +3427,7 @@ async fn prepare_tier_delete_dispatch_inner( // The write may have reached quorum even if the client saw a // timeout. Only a strong read confirming Authorized permits // mutation; every other outcome is retained for recovery. - match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await { + match read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await { Ok(Some((observed, _))) if observed.operation_id == manifest.operation_id && observed.state == TierDeleteDispatchManifestState::DispatchAuthorized => {} @@ -2546,7 +3440,7 @@ async fn prepare_tier_delete_dispatch_inner( .await; if let Err(err) = attempt { - let authorized = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + let authorized = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await .ok() .flatten() @@ -2555,8 +3449,15 @@ async fn prepare_tier_delete_dispatch_inner( && current.state == TierDeleteDispatchManifestState::DispatchAuthorized }); if !authorized - && let Err(rollback_err) = - seal_and_rollback_preparing_dispatch(api.clone(), &manifest, &fleet_proof, bucket_fence, &operation_guard).await + && let Err(rollback_err) = seal_and_rollback_preparing_dispatch( + api.clone(), + manifest_name, + &manifest, + &fleet_proof, + bucket_fence, + operation_guard, + ) + .await { warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, @@ -2572,8 +3473,17 @@ async fn prepare_tier_delete_dispatch_inner( // Re-read the exact Authorized manifest and every bound journal before // constructing the private one-shot permit. - ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; - authorized_dispatch_permit(api, &manifest_name, fleet_proof, bucket_fence, &operation_guard).await + ensure_dispatch_lock_current(bucket_fence, operation_guard)?; + authorized_dispatch_permit_with_parent( + api, + manifest_name, + fleet_proof, + bucket_fence, + operation_guard, + parent_operation_guard, + parent_advance, + ) + .await } async fn commit_dispatched_journal( @@ -2592,7 +3502,7 @@ async fn commit_dispatched_journal( let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("dispatched tier delete journal disappeared before commit"))?; - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, &authorization.0.manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, expected) { return Err(Error::other("dispatched tier delete journal changed identity before commit")); } @@ -2621,6 +3531,75 @@ async fn commit_dispatched_journal( Err(Error::other("tier delete journal changed repeatedly during commit")) } +async fn advance_tier_delete_chunk_parent( + api: Arc, + active: &ActiveTierDeleteDispatch, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result<()> { + let Some(advance) = active.parent_advance.as_ref() else { + return Ok(()); + }; + let parent_lock = api + .new_ns_lock( + RUSTFS_META_BUCKET, + &tier_delete_dispatch_operation_lock_name(&advance.parent_manifest_object), + ) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = match read_tier_delete_dispatch_record(api.clone(), &advance.parent_manifest_object).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + _ => return Err(Error::other("tier delete chunk parent disappeared before batch progress")), + }; + if parent.operation_id != advance.parent_operation_id + || parent.state != TierDeleteDispatchParentState::Active + || parent.active_chunk.as_ref() != Some(&advance.chunk) + || parent.bucket != active.manifest.bucket + || parent.bucket_incarnation != active.manifest.bucket_incarnation + || parent.prefix != active.manifest.prefix + || parent.topology_generation != active.manifest.topology_generation + { + return Err(Error::other("tier delete chunk parent changed before batch progress")); + } + let (child, _) = read_tier_delete_dispatch_manifest(api.clone(), &advance.chunk.manifest_object) + .await? + .ok_or_else(|| Error::other("completed tier delete child manifest disappeared before parent progress"))?; + if child.state != TierDeleteDispatchManifestState::Completed + || !tier_delete_dispatch_child_matches_parent(&parent, &advance.chunk, &child) + { + return Err(Error::other("tier delete child is not durably completed before parent progress")); + } + let fences_current = || { + !bucket_fence.is_lock_lost() + && !parent_guard.is_lock_lost() + && active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) + .is_ok() + }; + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &advance.parent_manifest_object, &parent, &fences_current) + .await?; + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(advance.chunk.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &advance.parent_manifest_object, &parent, &parent_etag, &next, &fences_current).await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentProgressed).await; + Ok(()) +} + pub(crate) async fn complete_tier_delete_dispatch( api: Arc, active: &ActiveTierDeleteDispatch, @@ -2632,113 +3611,127 @@ pub(crate) async fn complete_tier_delete_dispatch( if !active.authorization.mutation_started() { return Err(Error::other("tier delete dispatch cannot commit before its local mutation starts")); } - let manifest_name = tier_delete_dispatch_manifest_object_name( - &active.manifest.bucket, - active.manifest.bucket_incarnation, - &active.manifest.prefix, - ); - let operation_lock = api - .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) - .await?; - let operation_guard = operation_lock - .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) - .await?; - let fences_current = || { - !bucket_fence.is_lock_lost() - && !operation_guard.is_lock_lost() - && active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) - .is_ok() - }; - ensure_durable_write_fence(&fences_current, "before tier delete dispatch completion")?; - let make_commit = |index: usize| { - let api = api.clone(); - let fences_current = &fences_current; - async move { - active.authorization.ensure_current( - &active.manifest.bucket, - active.manifest.bucket_incarnation, - &active.manifest.prefix, - )?; - commit_dispatched_journal(api, &active.manifest, &active.entries[index], &active.authorization, fences_current) - .await - .map(|entry| (index, entry)) - } - }; - let mut next = 0; - let mut commits = futures::stream::FuturesUnordered::new(); - while next < active.entries.len() && commits.len() < TIER_DELETE_DISPATCH_CAS_CONCURRENCY { - commits.push(make_commit(next)); - next += 1; - } - let mut committed_entries = vec![None; active.entries.len()]; - let mut first_error = None; - while let Some(result) = commits.next().await { - match result { - Ok((index, entry)) => committed_entries[index] = Some(entry), - Err(err) if first_error.is_none() => first_error = Some(err), - Err(_) => {} - } - // A failed member CAS leaves the Authorized manifest and every - // already-committed member recoverable. Stop admitting tail work, but - // drain the bounded in-flight set before releasing the operation lock. - if first_error.is_none() && next < active.entries.len() { + // The caller holds bucket lifecycle WRITE. Scope the child operation lock + // so it is released before `advance_tier_delete_chunk_parent` takes the + // parent operation lock. + { + let manifest_name = active.manifest_object.clone(); + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let fences_current = || { + !bucket_fence.is_lock_lost() + && !operation_guard.is_lock_lost() + && active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) + .is_ok() + }; + ensure_durable_write_fence(&fences_current, "before tier delete dispatch completion")?; + let make_commit = |index: usize| { + let api = api.clone(); + let fences_current = &fences_current; + async move { + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + commit_dispatched_journal(api, &active.manifest, &active.entries[index], &active.authorization, fences_current) + .await + .map(|entry| (index, entry)) + } + }; + let mut next = 0; + let mut commits = futures::stream::FuturesUnordered::new(); + while next < active.entries.len() && commits.len() < TIER_DELETE_DISPATCH_CAS_CONCURRENCY { commits.push(make_commit(next)); next += 1; } - } - if let Some(err) = first_error { - return Err(err); - } - let committed_entries = committed_entries - .into_iter() - .collect::>>() - .ok_or_else(|| Error::other("tier delete dispatch completion omitted a journal"))?; + let mut committed_entries = vec![None; active.entries.len()]; + let mut first_error = None; + while let Some(result) = commits.next().await { + match result { + Ok((index, entry)) => committed_entries[index] = Some(entry), + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + // A failed member CAS leaves the Authorized manifest and every + // already-committed member recoverable. Stop admitting tail work, + // but drain the bounded in-flight set before releasing the operation lock. + if first_error.is_none() && next < active.entries.len() { + commits.push(make_commit(next)); + next += 1; + } + } + if let Some(err) = first_error { + return Err(err); + } + let committed_entries = committed_entries + .into_iter() + .collect::>>() + .ok_or_else(|| Error::other("tier delete dispatch completion omitted a journal"))?; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - let (mut current, etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) - .await? - .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared before completion"))?; - if current.operation_id != active.manifest.operation_id - || current.journal_set_sha256 != active.manifest.journal_set_sha256 - || current.state != TierDeleteDispatchManifestState::DispatchAuthorized - || (etag != active.authorized_etag && current != active.manifest) - { - return Err(Error::other("tier delete dispatch manifest changed before completion")); - } - current.state = TierDeleteDispatchManifestState::Completed; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - let completion = save_config_if_match_fenced( - api.clone(), - &manifest_name, - encode_tier_delete_dispatch_manifest(¤t)?, - &etag, - &fences_current, - ) - .await; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - completion?; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + let (mut current, etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + .await? + .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared before completion"))?; + if current.operation_id != active.manifest.operation_id + || current.journal_set_sha256 != active.manifest.journal_set_sha256 + || current.state != TierDeleteDispatchManifestState::DispatchAuthorized + || (etag != active.authorized_etag && current != active.manifest) + { + return Err(Error::other("tier delete dispatch manifest changed before completion")); + } + current.state = TierDeleteDispatchManifestState::Completed; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + let completion = save_config_if_match_fenced( + api.clone(), + &manifest_name, + encode_tier_delete_dispatch_manifest(¤t)?, + &etag, + &fences_current, + ) + .await; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + completion?; - for entry in committed_entries { - if let Err(err) = enqueue_committed_tier_delete_journal_entry(&entry).await { - debug!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - operation_id = %current.operation_id, - error = ?err, - "Committed tier delete dispatch will be picked up by periodic recovery" - ); + #[cfg(all(test, feature = "test-util"))] + if active.is_chunked() { + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ChildCompleted).await; + } + + for entry in committed_entries { + if let Err(err) = enqueue_committed_tier_delete_journal_entry(&entry).await { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + operation_id = %current.operation_id, + error = ?err, + "Committed tier delete dispatch will be picked up by periodic recovery" + ); + } } } + advance_tier_delete_chunk_parent(api, active, bucket_fence).await?; + Ok(()) } @@ -3338,8 +4331,12 @@ async fn load_manifest_for_journal( let (manifest, _) = read_tier_delete_dispatch_manifest(api, &binding.manifest_object) .await? .ok_or_else(|| Error::other("tier delete dispatch manifest is missing"))?; - validate_bound_journal(&manifest, journal_name, journal)?; - if manifest.journal_names.binary_search(&journal_name.to_string()).is_err() { + validate_bound_journal(&manifest, &binding.manifest_object, journal_name, journal)?; + if manifest + .journal_names + .binary_search_by(|name| name.as_str().cmp(journal_name)) + .is_err() + { return Err(Error::other("tier delete dispatch manifest does not contain its journal")); } Ok(manifest) @@ -3582,6 +4579,138 @@ enum TierDeleteDispatchManifestRecoveryOutcome { Retained, } +fn parent_recovery_fences_current( + bucket_guard: &rustfs_lock::NamespaceLockGuard, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + parent: &TierDeleteDispatchParent, + cancel_token: Option<&CancellationToken>, +) -> bool { + cancel_token.is_none_or(|token| !token.is_cancelled()) + && !bucket_guard.is_lock_lost() + && !operation_guard.is_lock_lost() + && tier_delete_journal_fleet_proof_matches(fleet_proof) + && tier_delete_journal_topology_generation(fleet_proof) == parent.topology_generation +} + +async fn delete_tier_delete_dispatch_parent_if_match_confirmed( + api: Arc, + parent_name: &str, + parent: &TierDeleteDispatchParent, + etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + let data = encode_tier_delete_dispatch_parent(parent)?; + let delete = delete_durable_config_if_match(api.clone(), parent_name, &data, etag, fences_current).await; + let observed = read_tier_delete_dispatch_record(api.clone(), parent_name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch parent recovery fence changed during deletion")); + } + match observed { + None => Ok(()), + Some((TierDeleteDispatchRecord::Parent(observed), _)) => { + let observed_data = encode_tier_delete_dispatch_parent(&observed)?; + if api + .durable_ilm_terminal_receipt_covers_active_source(parent_name, &observed_data) + .await? + { + Ok(()) + } else { + Err(Error::other_with_context( + "tier delete dispatch parent changed during deletion", + format!("observed {:?}, delete result {:?}", observed.state, delete.as_ref().err()), + )) + } + } + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + Err(Error::other("tier delete dispatch parent was replaced during deletion")) + } + } +} + +async fn process_tier_delete_dispatch_parent( + api: Arc, + parent_name: &str, + observed_before_lock: &TierDeleteDispatchParent, + cancel_token: Option<&CancellationToken>, +) -> Result { + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if tier_delete_journal_topology_generation(&fleet_proof) != observed_before_lock.topology_generation { + return Err(Error::other("tier delete chunk parent topology generation changed")); + } + // Background lock order matches the request path: bucket lifecycle WRITE + // precedes the parent operation lock. This path never takes a child lock. + let bucket_guard = api.acquire_bucket_lifecycle_write_lock(&observed_before_lock.bucket).await?; + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(parent_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, etag) = match read_tier_delete_dispatch_record(api.clone(), parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("tier delete chunk parent was replaced by a legacy manifest")); + } + None => return Err(Error::ConfigNotFound), + }; + if parent.operation_id != observed_before_lock.operation_id + || parent.bucket != observed_before_lock.bucket + || !parent_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, &parent, cancel_token) + { + return Err(Error::other("tier delete chunk parent recovery fence changed")); + } + let fences_current = || parent_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, &parent, cancel_token); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), parent_name, &parent, &fences_current).await?; + if parent.state == TierDeleteDispatchParentState::Completed { + delete_tier_delete_dispatch_parent_if_match_confirmed(api, parent_name, &parent, &etag, &fences_current).await?; + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Deleted); + } + let Some(binding) = parent.active_chunk.clone() else { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + }; + let child = read_tier_delete_dispatch_manifest(api.clone(), &binding.manifest_object).await?; + let Some((child, _)) = child else { + if !tier_delete_dispatch_chunk_journal_namespace_empty(api.clone(), binding.operation_id).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, parent_name, &parent, &etag, &next, &fences_current).await?; + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced); + }; + if !tier_delete_dispatch_child_matches_parent(&parent, &binding, &child) { + return Err(Error::other("tier delete chunk parent child binding changed during recovery")); + } + if child.state != TierDeleteDispatchManifestState::Completed { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(binding.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, parent_name, &parent, &etag, &next, &fences_current).await?; + Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced) +} + fn manifest_recovery_fences_current( bucket_guard: &rustfs_lock::NamespaceLockGuard, operation_guard: &rustfs_lock::NamespaceLockGuard, @@ -3674,8 +4803,52 @@ async fn delete_tier_delete_dispatch_manifest_if_match_confirmed( } } +/// Child manifests share the dispatch-record namespace with the legacy/root +/// manifest and are therefore returned by the same recovery listing. A child +/// that is still bound by an active chunk parent must remain available until +/// the parent records the child completion and advances its sequence/count. +/// The bucket lifecycle WRITE lock held by the caller serializes this check +/// with request-side parent inspection/progress, so no parent lock nesting is +/// needed here. +async fn child_dispatch_manifest_is_bound_to_active_parent( + api: Arc, + manifest_name: &str, + manifest: &TierDeleteDispatchManifest, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix); + if parent_name == manifest_name { + return Ok(false); + } + let Some((record, _)) = read_tier_delete_dispatch_record(api, &parent_name).await? else { + return Ok(false); + }; + let TierDeleteDispatchRecord::Parent(parent) = record else { + return Err(Error::other("tier delete child exists beside a legacy root manifest")); + }; + if parent.bucket != manifest.bucket + || parent.bucket_incarnation != manifest.bucket_incarnation + || parent.prefix != manifest.prefix + || parent.topology_generation != manifest.topology_generation + { + return Err(Error::other("tier delete child parent identity changed during recovery")); + } + let Some(binding) = parent.active_chunk.as_ref() else { + return Ok(false); + }; + if binding.manifest_object != manifest_name { + return Ok(false); + } + if parent.state != TierDeleteDispatchParentState::Active + || !tier_delete_dispatch_child_matches_parent(&parent, binding, manifest) + { + return Err(Error::other("tier delete child binding is inconsistent during recovery")); + } + Ok(true) +} + async fn authorized_dispatch_all_committed( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result { @@ -3698,7 +4871,7 @@ async fn authorized_dispatch_all_committed( ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; let (entry, _) = observed .ok_or_else(|| Error::other("authorized tier delete dispatch manifest references a missing journal"))?; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; #[cfg(all(test, feature = "test-util"))] tier_delete_dispatch_authorized_progress_test_observed(); record_tier_delete_journal_progress_fenced(api, &name, &entry, fences_current).await?; @@ -3752,6 +4925,7 @@ async fn authorized_dispatch_all_committed( async fn completed_dispatch_has_present_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result { @@ -3776,7 +4950,7 @@ async fn completed_dispatch_has_present_journal( let Some((entry, _)) = observed else { return Ok(None); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if entry.state != TierDeleteJournalState::Committed { return Err(Error::other("completed tier delete dispatch contains an uncommitted journal")); } @@ -3845,6 +5019,8 @@ async fn process_tier_delete_dispatch_manifest( } return Err(Error::other("tier delete dispatch manifest recovery fence changed")); } + let child_bound_to_active_parent = + child_dispatch_manifest_is_bound_to_active_parent(api.clone(), manifest_name, ¤t).await?; { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); @@ -3883,7 +5059,7 @@ async fn process_tier_delete_dispatch_manifest( { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - delete_staged_dispatch_journal_set(api.clone(), ¤t, &fences_current).await?; + delete_staged_dispatch_journal_set(api.clone(), manifest_name, ¤t, &fences_current).await?; } if current.state == TierDeleteDispatchManifestState::Aborting { let next = { @@ -3907,7 +5083,7 @@ async fn process_tier_delete_dispatch_manifest( } return Err(Error::other("tier delete dispatch manifest recovery fence changed")); } - validate_staged_dispatch_journal_set(api.clone(), ¤t, &|| { + validate_staged_dispatch_journal_set(api.clone(), manifest_name, ¤t, &|| { manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) }) .await?; @@ -3920,7 +5096,7 @@ async fn process_tier_delete_dispatch_manifest( if current.state == TierDeleteDispatchManifestState::DispatchAuthorized { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - if !authorized_dispatch_all_committed(api.clone(), ¤t, &fences_current).await? { + if !authorized_dispatch_all_committed(api.clone(), manifest_name, ¤t, &fences_current).await? { return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); } let next = { @@ -3947,7 +5123,14 @@ async fn process_tier_delete_dispatch_manifest( } let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - if completed_dispatch_has_present_journal(api.clone(), ¤t, &fences_current).await? { + if completed_dispatch_has_present_journal(api.clone(), manifest_name, ¤t, &fences_current).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + if child_bound_to_active_parent { + // The parent must observe the completed child and durably add its + // journal count before the child record can be removed. Otherwise a + // concurrent child recovery could erase the only evidence needed to + // advance the parent's sequence. return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); } if !manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) { @@ -4059,8 +5242,8 @@ async fn recover_tier_delete_dispatch_manifest_object( return TierDeleteDispatchManifestScanOutcome::Failed; } }; - let manifest = match decode_tier_delete_dispatch_manifest(&data, &object_name) { - Ok(manifest) => manifest, + let record = match decode_tier_delete_dispatch_record(&data, &object_name) { + Ok(record) => record, Err(err) => { warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, @@ -4068,11 +5251,15 @@ async fn recover_tier_delete_dispatch_manifest_object( subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, error = ?err, - "Invalid tier delete dispatch manifest is quarantined" + "Invalid tier delete dispatch record is quarantined" ); return TierDeleteDispatchManifestScanOutcome::Failed; } }; + let operation_id = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => manifest.operation_id, + TierDeleteDispatchRecord::Parent(parent) => parent.operation_id, + }; match api .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data) .await @@ -4090,14 +5277,25 @@ async fn recover_tier_delete_dispatch_manifest_object( component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, - operation_id = %manifest.operation_id, + operation_id = %operation_id, error = ?err, "Tier delete dispatch terminal source proof will retry later" ); return TierDeleteDispatchManifestScanOutcome::Failed; } } - let result = process_tier_delete_dispatch_manifest(api, &object_name, &manifest, cancel_token.as_ref()).await; + let state = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => format!("{:?}", manifest.state), + TierDeleteDispatchRecord::Parent(parent) => format!("parent::{:?}", parent.state), + }; + let result = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => { + process_tier_delete_dispatch_manifest(api, &object_name, manifest, cancel_token.as_ref()).await + } + TierDeleteDispatchRecord::Parent(parent) => { + process_tier_delete_dispatch_parent(api, &object_name, parent, cancel_token.as_ref()).await + } + }; if cancel_token.as_ref().is_some_and(CancellationToken::is_cancelled) { // Cancellation is cooperative: every admitted mutation above has // already returned while the operation/fleet guards and registry @@ -4116,8 +5314,8 @@ async fn recover_tier_delete_dispatch_manifest_object( component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, - operation_id = %manifest.operation_id, - state = ?manifest.state, + operation_id = %operation_id, + state = %state, error = ?err, "Tier delete dispatch manifest recovery will retry later" ); @@ -4574,11 +5772,17 @@ where #[cfg(test)] mod tests { use super::{ + TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX, - await_tier_delete_journal_recovery, decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, - object_info_references_tier_delete, record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, - same_tier_delete_journal_identity, tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, + TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent, + TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery, + decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest, + encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete, + record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity, + tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name, + tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name, + tier_delete_source_matches_dispatch_scope, }; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -4603,6 +5807,118 @@ mod tests { } } + #[test] + fn chunk_parent_and_child_preserve_the_legacy_fail_closed_fence() { + let legacy_v1_manifest_path_accepts = |data: &[u8], object_name: &str| { + serde_json::from_slice::(data).is_ok_and(|manifest| { + tier_delete_dispatch_manifest_object_name(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix) + == object_name + }) + }; + let bucket = "chunked-bucket"; + let incarnation = uuid::Uuid::new_v4(); + let prefix = "large/"; + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix); + let parent = TierDeleteDispatchParent { + version: TIER_DELETE_DISPATCH_PARENT_VERSION, + record_type: TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE.to_string(), + operation_id: uuid::Uuid::new_v4(), + bucket: bucket.to_string(), + bucket_incarnation: incarnation, + prefix: prefix.to_string(), + topology_generation: "a".repeat(64), + revision: 0, + next_chunk_sequence: 0, + completed_journal_count: 0, + active_chunk: None, + state: TierDeleteDispatchParentState::Active, + }; + let parent_data = encode_tier_delete_dispatch_parent(&parent).expect("chunk parent should encode"); + assert!( + serde_json::from_slice::(&parent_data).is_err(), + "the legacy v1 manifest codec must reject a chunk-parent root sentinel" + ); + assert!(matches!( + decode_tier_delete_dispatch_record(&parent_data, &parent_name).expect("current codec should accept the parent"), + TierDeleteDispatchRecord::Parent(_) + )); + assert!( + decode_tier_delete_dispatch_record(&parent_data, &format!("{parent_name}.other")).is_err(), + "a chunk parent must remain bound to the deterministic legacy root" + ); + let mut impossible_progress = parent.clone(); + impossible_progress.revision = 2; + impossible_progress.next_chunk_sequence = 2; + impossible_progress.completed_journal_count = 1; + let impossible_progress_data = + encode_tier_delete_dispatch_parent(&impossible_progress).expect("invalid test parent should still serialize"); + assert!( + decode_tier_delete_dispatch_record(&impossible_progress_data, &parent_name).is_err(), + "a parent cannot complete fewer journals than its completed child count" + ); + + let child_operation_id = uuid::Uuid::new_v4(); + let child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, incarnation, prefix, child_operation_id); + let journal_names = vec![format!( + "{TIER_DELETE_JOURNAL_V6_PREFIX}{}/{}.json", + child_operation_id.simple(), + "b".repeat(64) + )]; + let journal_set_sha256 = tier_delete_dispatch_journal_set_digest(&journal_names); + let child = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id: child_operation_id, + bucket: bucket.to_string(), + bucket_incarnation: incarnation, + prefix: prefix.to_string(), + journal_count: 1, + journal_names, + journal_set_sha256: journal_set_sha256.clone(), + topology_generation: "a".repeat(64), + state: TierDeleteDispatchManifestState::Completed, + }; + let child_data = encode_tier_delete_dispatch_manifest(&child).expect("chunk child should encode"); + assert_ne!(child_name, parent_name); + assert!( + legacy_v1_manifest_path_accepts(&child_data, &parent_name), + "the unchanged child payload must remain byte-compatible at the legacy root" + ); + assert!( + !legacy_v1_manifest_path_accepts(&child_data, &child_name), + "the legacy root-only path validator must reject an operation-scoped child" + ); + assert!(matches!( + decode_tier_delete_dispatch_record(&child_data, &child_name).expect("current codec should accept the child"), + TierDeleteDispatchRecord::Manifest(_) + )); + let wrong_child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, incarnation, prefix, uuid::Uuid::new_v4()); + assert!( + decode_tier_delete_dispatch_record(&child_data, &wrong_child_name).is_err(), + "a child manifest must remain bound to its exact operation-scoped path" + ); + + let binding = TierDeleteDispatchChunkBinding { + sequence: 0, + operation_id: child_operation_id, + manifest_object: child_name, + journal_set_sha256, + journal_count: 1, + }; + let mut bound_parent = parent; + bound_parent.revision = 1; + bound_parent.active_chunk = Some(binding.clone()); + bound_parent + .validate(&parent_name) + .expect("the exact child binding should produce a valid active parent"); + assert!(tier_delete_dispatch_child_matches_parent(&bound_parent, &binding, &child)); + let mut mismatched_child = child; + mismatched_child.topology_generation = "c".repeat(64); + assert!( + !tier_delete_dispatch_child_matches_parent(&bound_parent, &binding, &mismatched_child), + "a valid child path and payload cannot bypass the exact parent topology binding" + ); + } + fn bound_v6_journal_entry(state: TierDeleteJournalState) -> Jentry { let operation_id = uuid::Uuid::new_v4(); Jentry { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 859415707..00a74906e 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -816,6 +816,7 @@ mod tests { }, tier_delete_journal::{ DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, + TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard, TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage, TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry, install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch, @@ -11624,6 +11625,612 @@ mod tests { assert_eq!(backend.remove_versions().await.len(), objects.len()); } + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() { + run_large_stack_async_test( + "tier-delete-prefix-limit-and-multi-chunk", + tier_delete_prefix_limit_and_multi_chunk_batches_converge_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2); + let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "CHUNKED-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "chunked-prefix-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("chunked prefix source bucket should be created"); + + for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] { + for index in 0..count { + let object = format!("{prefix}/object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("chunked prefix source should be written"); + store + .transition_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("chunked prefix source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("chunked prefix source should transition"); + } + } + backend.set_remove_failure(true); + + store + .delete_object_with_tier_delete_journal( + bucket, + "at-limit/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("exactly one batch must retain the v1 one-shot path"); + + let limit_plus_one_first = store + .delete_object_with_tier_delete_journal( + bucket, + "limit-plus-one/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("limit plus one must start a bounded parent transaction"); + assert!( + limit_plus_one_first.to_string().contains("retry the next durable batch"), + "unexpected first limit-plus-one result: {limit_plus_one_first}" + ); + let active_limit_plus_one_records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 10, + false, + None, + false, + ) + .await + .expect("active limit-plus-one records should be listable"); + let mut active_limit_plus_one_parent_seen = false; + for record in active_limit_plus_one_records + .objects + .iter() + .filter(|record| !record.name.contains("/chunks/")) + { + let data = com::read_config(store.clone(), &record.name) + .await + .expect("an active dispatch root should be readable"); + let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON"); + if value["prefix"] == "limit-plus-one/" { + assert_eq!(value["record_type"], "chunked_parent"); + active_limit_plus_one_parent_seen = true; + } + } + assert!( + active_limit_plus_one_parent_seen, + "limit plus one must establish the fail-closed parent at the legacy root" + ); + + let mut limit_plus_one_completed = false; + let mut limit_plus_one_retries = 1; + for _ in 0..3 { + match store + .delete_object_with_tier_delete_journal( + bucket, + "limit-plus-one/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + limit_plus_one_completed = true; + break; + } + Err(err) if err.to_string().contains("retry the next durable batch") => { + limit_plus_one_retries += 1; + } + Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"), + } + } + assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children"); + assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches"); + + let first_batch = store + .delete_object_with_tier_delete_journal( + bucket, + "multi-chunk/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("the first bounded child must request a successor batch"); + assert!( + first_batch.to_string().contains("retry the next durable batch"), + "unexpected first child result: {first_batch}" + ); + + let newcomer = "multi-chunk/zzz-newcomer.bin"; + let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]); + let newcomer_source = store + .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default()) + .await + .expect("a source created between chunks should be written"); + store + .transition_object( + bucket, + newcomer, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: newcomer_source + .etag + .clone() + .expect("the between-chunks source should have an etag"), + ..Default::default() + }, + mod_time: newcomer_source.mod_time, + ..Default::default() + }, + ) + .await + .expect("a source created between chunks should transition"); + + let mut completed = false; + let mut durable_batch_retries = 1; + for _ in 0..7 { + match store + .delete_object_with_tier_delete_journal( + bucket, + "multi-chunk/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + completed = true; + break; + } + Err(err) + if err.to_string().contains("retry the next durable batch") + || err.to_string().contains("retry the next batch") => + { + durable_batch_retries += 1; + } + Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"), + } + } + assert!(completed, "six entries must converge through three bounded child batches"); + assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 11); + assert_eq!( + backend.object_count().await, + 11, + "remote cleanup must remain durable while the tier is unavailable" + ); + let dispatch_records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 20, + false, + None, + false, + ) + .await + .expect("bounded dispatch records should be listable"); + let mut child_batches = 0; + let mut legacy_at_limit_seen = false; + for record in &dispatch_records.objects { + let data = com::read_config(store.clone(), &record.name) + .await + .expect("a retained dispatch record should be readable"); + let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON"); + if record.name.contains("/chunks/") { + let journal_count = value["journal_count"] + .as_u64() + .expect("a retained child should declare its journal count"); + assert!(journal_count <= 2, "a child batch exceeded the configured resource bound"); + child_batches += 1; + } else if value["prefix"] == "at-limit/" { + assert!( + value.get("record_type").is_none(), + "the exact-limit root must remain a legacy v1 manifest" + ); + assert_eq!(value["journal_count"], 2); + legacy_at_limit_seen = true; + } + } + assert!( + legacy_at_limit_seen, + "the exact-limit dispatch must retain its byte-compatible root shape" + ); + assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children"); + + backend.set_remove_failure(false); + drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await; + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!( + backend.remove_versions().await.len(), + 11, + "each remote version must be removed exactly once" + ); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() { + run_large_stack_async_test( + "tier-delete-chunk-crash-boundaries", + tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); + let stages = [ + TierDeleteChunkTestStage::ParentPersisted, + TierDeleteChunkTestStage::ChildManifestPersisted, + TierDeleteChunkTestStage::ParentBound, + TierDeleteChunkTestStage::DispatchAuthorized, + TierDeleteChunkTestStage::LocalReplayCompleted, + TierDeleteChunkTestStage::ChildCompleted, + TierDeleteChunkTestStage::ParentProgressed, + TierDeleteChunkTestStage::FinalLocalDeletionCompleted, + TierDeleteChunkTestStage::ParentCompleted, + ]; + for (case, stage) in stages.into_iter().enumerate() { + let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir"); + let initial_name = format!("chunk-crash-boundary-{case}"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}"); + let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await; + backend.set_remove_failure(true); + let bucket = format!("chunk-crash-boundary-{case}-bucket"); + let prefix = "prefix/"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("chunk crash-boundary bucket should be created"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]); + let source = store + .put_object(&bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("chunk crash-boundary source should be written"); + store + .transition_object( + &bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("chunk crash-boundary source should transition"); + } + let local_only = format!("{prefix}local-only.bin"); + let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]); + store + .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default()) + .await + .expect("chunk crash-boundary local-only object should be written"); + let tier_config = ctx + .tier_config_mgr() + .read() + .await + .tiers + .get(&tier_name) + .expect("chunk crash-boundary tier config should remain available for restart") + .clone_with_credentials(); + + if matches!( + stage, + TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted + ) { + for _ in 0..2 { + let retry = store + .delete_object_with_tier_delete_journal( + &bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("each bounded child must complete before the final crash boundary"); + assert!( + retry.to_string().contains("retry the next durable batch"), + "unexpected bounded-child result before {stage:?}: {retry}" + ); + } + } + + let barrier = TierDeleteChunkTestBarrier::install(stage); + let worker_store = store.clone(); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .delete_object_with_tier_delete_journal( + &worker_bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}")); + worker.abort(); + let _ = worker.await; + drop(barrier); + let released_bucket_guard = + tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket)) + .await + .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock")) + .unwrap_or_else(|err| { + panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}") + }); + drop(released_bucket_guard); + shutdown.cancel(); + drop(store); + drop(ctx); + + let restarted_name = format!("chunk-crash-boundary-{case}-restart"); + let (restarted_ctx, restarted_store, restarted_shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await; + { + let tier_config_mgr = restarted_ctx.tier_config_mgr(); + let mut manager = tier_config_mgr.write().await; + manager.tiers.insert(tier_name.clone(), tier_config); + manager + .install_test_driver(&tier_name, Box::new(backend.clone())) + .expect("the exact chunk crash-boundary tier driver should reinstall after restart"); + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + + let mut completed = false; + for _ in 0..20 { + let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None) + .await + .expect("chunk crash-boundary manifest recovery should remain readable"); + assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary"); + match restarted_store + .delete_object_with_tier_delete_journal( + &bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + completed = true; + break; + } + Err(err) + if err.to_string().contains("retry") + || err.to_string().contains("rollback") + || err.to_string().contains("durable cleanup") => {} + Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"), + } + } + assert!(completed, "chunk state must converge after cancellation at {stage:?}"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + assert!( + restarted_store.pools[0] + .get_disks_by_key(&object) + .load_file_info_versions_exact(&bucket, &object) + .await + .expect("chunk crash-boundary source lookup should succeed") + .is_none(), + "source {object} must not be skipped after cancellation at {stage:?}" + ); + } + assert!( + restarted_store.pools[0] + .get_disks_by_key(&local_only) + .load_file_info_versions_exact(&bucket, &local_only) + .await + .expect("chunk crash-boundary local-only source lookup should succeed") + .is_none(), + "the final raw delete must remove the local-only source after cancellation at {stage:?}" + ); + assert_eq!(backend.object_count().await, 2); + backend.set_remove_failure(false); + drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await; + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!( + backend.remove_versions().await.len(), + 2, + "each crash case must retain exactly one cleanup owner per remote version" + ); + restarted_shutdown.cancel(); + } + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_chunk_missing_child_with_journals_fails_closed() { + run_large_stack_async_test( + "tier-delete-chunk-missing-child", + tier_delete_chunk_missing_child_with_journals_fails_closed_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); + let temp_dir = tempfile::tempdir().expect("create missing-child store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "CHUNK-MISSING-CHILD"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_remove_failure(true); + let bucket = "chunk-missing-child-bucket"; + let prefix = "prefix/"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("missing-child bucket should be created"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("missing-child source should be written"); + store + .transition_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("missing-child source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("missing-child source should transition"); + } + + let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized); + let worker_store = store.clone(); + let worker = tokio::spawn(async move { + worker_store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("chunk delete should persist its authorization before corruption injection"); + worker.abort(); + let _ = worker.await; + drop(barrier); + + let records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 10, + false, + None, + false, + ) + .await + .expect("parent and child records should be listable"); + let child = records + .objects + .iter() + .find(|object| object.name.contains("/chunks/")) + .expect("the bound child manifest should exist") + .name + .clone(); + com::delete_config(store.clone(), &child) + .await + .expect("the test should remove only the child manifest"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 1); + + let error = store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a missing child with retained journals must quarantine the parent"); + assert!( + error.to_string().contains("missing child with retained journals"), + "unexpected missing-child result: {error}" + ); + assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version"); + shutdown.cancel(); + } + #[cfg(feature = "test-util")] #[test] #[serial_test::serial(storage_class_env)] @@ -11647,6 +12254,7 @@ mod tests { #[cfg(feature = "test-util")] async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir"); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4])) @@ -11659,8 +12267,8 @@ mod tests { .expect("authorized replay tier lease should resolve"); let bucket = "authorized-prefix-predecessor-replay-bucket"; let prefix = "prefix/"; - let predecessor = "prefix/predecessor.bin"; - let newcomer = "prefix/newcomer.bin"; + let predecessor = "prefix/000-predecessor.bin"; + let newcomer = "prefix/zzz-newcomer.bin"; store .make_bucket(bucket, &MakeBucketOptions::default()) .await diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index c3e150964..f231e738a 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -18,7 +18,9 @@ use crate::bucket::lifecycle::{ get_expiry_configs, tier_delete_journal::{ ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE, - complete_tier_delete_dispatch, prepare_tier_delete_dispatch, record_tier_delete_journal_backend_identity, + TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch, + inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch, + record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit, tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, }, tier_sweeper::{ @@ -41,7 +43,10 @@ use crate::object_api::{ NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState, TierFreeVersionReceiptSink, }; -use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof; +use crate::services::notification_sys::{ + TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, + tier_delete_journal_topology_generation, +}; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::{ SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, @@ -52,6 +57,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _}, }; +use futures::StreamExt as _; use parking_lot::Mutex as ParkingMutex; use rustfs_filemeta::ObjectPartInfo; use rustfs_io_metrics::{ @@ -73,6 +79,7 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000; const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2; const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks"; const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50); +const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16; fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option { if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix { @@ -147,14 +154,96 @@ async fn prepare_prefix_tier_delete_journal_entries( Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await } +type TierDeleteLeaseReference = (String, Option); + +fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool { + truncated && limit_cancellation && matches!(error, Error::OperationCanceled) +} + +fn combine_tier_delete_walk_results(results: impl IntoIterator>) -> Result<()> { + let mut cancelled = false; + for result in results { + match result { + Ok(()) => {} + Err(Error::OperationCanceled) => cancelled = true, + Err(err) => return Err(err), + } + } + if cancelled { Err(Error::OperationCanceled) } else { Ok(()) } +} + +async fn acquire_prefix_tier_delete_reference_leases( + api: &Arc, + tier_references: &std::collections::HashSet, +) -> Result> { + let mut tier_references = tier_references.iter().cloned().collect::>(); + tier_references.sort_unstable(); + let mut leases = Vec::with_capacity(tier_references.len()); + for (tier_name, backend_identity) in tier_references { + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, + } + .map_err(Error::other)?; + leases.push(lease); + } + Ok(leases) +} + +async fn acquire_prefix_tier_delete_leases(api: &Arc, entries: &[Jentry]) -> Result> { + let tier_references = entries + .iter() + .map(|entry| (entry.tier_name.clone(), entry.backend_identity)) + .collect::>(); + acquire_prefix_tier_delete_reference_leases(api, &tier_references).await +} + async fn prepare_prefix_tier_delete_journal_entries_inner( api: &Arc, bucket: &str, prefix: &str, opts: &ObjectOptions, ) -> Result { + let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) { + (false, false, None) + } else { + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + match Box::pin(inspect_tier_delete_chunk_parent( + Arc::clone(api), + bucket, + bucket_incarnation, + prefix, + bucket_fence, + )) + .await? + { + TierDeleteChunkParentInspection::NoParent => (false, false, None), + TierDeleteChunkParentInspection::LegacyManifest => (false, true, None), + TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)), + TierDeleteChunkParentInspection::Resume(dispatch) => { + let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?; + return Ok(PreparedPrefixTierDelete { + dispatch: Some(*dispatch), + chunk_parent_active: true, + chunk_parent_fleet_proof: None, + _leases: leases, + }); + } + TierDeleteChunkParentInspection::RetryRequired => { + return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch")); + } + } + }; let mut tier_references = std::collections::HashSet::<(String, Option)>::new(); let mut entries_by_name = std::collections::BTreeMap::new(); + let batch_limit = tier_delete_dispatch_batch_limit(); let logical_prefix = decode_dir_object(prefix); let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone()); let physical_sets = api @@ -164,12 +253,11 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( .collect::>(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100); let cancellation = tokio_util::sync::CancellationToken::new(); + let limit_cancellation = Arc::new(AtomicBool::new(false)); let walk_cancel = cancellation.clone(); let bucket_owned = bucket.to_string(); let prefix_owned = prefix.to_string(); let walk = async move { - use futures::StreamExt as _; - let results = futures::stream::iter(physical_sets.into_iter().map(|set| { let tx = tx.clone(); let cancellation = walk_cancel.clone(); @@ -199,14 +287,28 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( .collect::>() .await; drop(tx); - results.into_iter().collect::>>().map(|_| ()) + combine_tier_delete_walk_results(results) }; + let collect_limit_cancellation = limit_cancellation.clone(); let collect = async { + let mut truncated = false; while let Some(result) = rx.recv().await { if let Some(err) = result.err { + // Once limit + 1 has been observed this request can authorize + // only the exact retained batch; it cannot infer prefix + // absence or run the raw delete. Drain only the explicit + // cancellation fallout; a real walker error must still fail + // the request even when another set reached the limit first. + if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err) + { + continue; + } cancellation.cancel(); return Err(err); } + if truncated { + continue; + } let Some(source) = result.item else { continue; }; @@ -223,62 +325,125 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( "recursive prefix delete cannot discard an existing tier free-version cleanup obligation", )); } - if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE { + let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE { let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?; - tier_references.insert((source.transitioned_object.tier.clone(), backend_identity)); - } + Some((source.transitioned_object.tier.clone(), backend_identity)) + } else { + None + }; if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? { - entries_by_name - .entry(tier_delete_journal_object_name(&entry)) - .or_insert(entry); + let name = tier_delete_journal_object_name(&entry); + let at_limit = entries_by_name.len() == batch_limit; + match entries_by_name.entry(name) { + std::collections::btree_map::Entry::Occupied(_) => {} + std::collections::btree_map::Entry::Vacant(_) if at_limit => { + truncated = true; + collect_limit_cancellation.store(true, Ordering::Release); + cancellation.cancel(); + } + std::collections::btree_map::Entry::Vacant(slot) => { + if let Some(tier_reference) = tier_reference { + tier_references.insert(tier_reference); + } + slot.insert(entry); + } + } + } else if let Some(tier_reference) = tier_reference { + tier_references.insert(tier_reference); } } - Ok(()) + Ok(truncated) }; let (walk_result, collect_result) = tokio::join!(walk, collect); - collect_result?; - walk_result?; - let entries = entries_by_name.into_values().collect::>(); - - let mut tier_references = tier_references.into_iter().collect::>(); - tier_references.sort_unstable(); - let mut leases = Vec::with_capacity(tier_references.len()); - for (tier_name, backend_identity) in tier_references { - let lease = match backend_identity { - Some(backend_identity) => { - TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity) - .await - } - None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, - } - .map_err(Error::other)?; - leases.push(lease); + let truncated = collect_result?; + // A truncated walk normally reports OperationCanceled from the physical + // walkers. That cancellation is expected; any other result is a genuine + // scan failure and cannot be hidden by the bounded batch. + if let Err(err) = walk_result + && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err) + { + return Err(err); } + let entries = entries_by_name.into_values().collect::>(); + let mut leased_tier_references = tier_references; + let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?; if entries.is_empty() { + let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() { + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology { + return Err(Error::other("tier delete chunk parent topology changed during final source scan")); + } + Some(fleet_proof) + } else { + None + }; return Ok(PreparedPrefixTierDelete { dispatch: None, + chunk_parent_active, + chunk_parent_fleet_proof, _leases: leases, }); } let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; - let fleet_proof = acquire_tier_delete_journal_fleet_proof() - .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?; let bucket_fence = opts .bucket_lifecycle_lock_fence .as_ref() .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; - let dispatch = + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?; + if chunk_parent_topology_generation + .as_deref() + .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected) + { + return Err(Error::other("tier delete chunk parent topology changed during source scan")); + } + let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) { + Box::pin(prepare_tier_delete_chunk_dispatch( + Arc::clone(api), + bucket, + bucket_incarnation, + prefix, + entries, + truncated && !chunk_parent_active, + fleet_proof, + bucket_fence, + )) + .await? + } else if legacy_manifest_active && truncated { + resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence) + .await? + } else { prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence) - .await?; + .await? + }; + // A resumed legacy authorization may own predecessors that are absent + // from this bounded scan. Pin every backend generation in the actual + // permit before any local mutation, while avoiding duplicate leases for + // entries already covered by the scan. + let additional_tier_references = dispatch + .entries()? + .iter() + .map(|entry| (entry.tier_name.clone(), entry.backend_identity)) + .filter(|reference| leased_tier_references.insert(reference.clone())) + .collect::>(); + leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?); + if legacy_manifest_active && truncated { + dispatch.require_exact_predecessor_replay(); + } Ok(PreparedPrefixTierDelete { dispatch: Some(dispatch), + chunk_parent_active: chunk_parent_active || truncated, + chunk_parent_fleet_proof: None, _leases: leases, }) } struct PreparedPrefixTierDelete { dispatch: Option, + chunk_parent_active: bool, + chunk_parent_fleet_proof: Option, _leases: Vec, } @@ -365,12 +530,16 @@ async fn delete_prefix_with_tier_delete_journal( let Some(api) = tier_journal_api else { return store.delete_prefix(bucket, object, opts).await; }; - let PreparedPrefixTierDelete { dispatch, _leases } = - prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?; + let PreparedPrefixTierDelete { + dispatch, + chunk_parent_active, + chunk_parent_fleet_proof, + _leases, + } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?; let Some(dispatch) = dispatch else { - // There is no remote-cleanup candidate, so no v6 manifest or fleet - // proof is required. Keep any compatibility-path tier leases alive - // until the local delete has committed. + // There is no new remote-cleanup candidate. Keep compatibility-path + // tier leases and, for a chunked final pass, the matching parent fleet + // proof alive until local deletion and parent completion both commit. let _tier_leases = _leases; let mut operation_opts = opts.clone(); // `tier_delete_journal_api` means a v6 dispatch authorization must be @@ -379,7 +548,42 @@ async fn delete_prefix_with_tier_delete_journal( // transitioned metadata retains its FreeVersion fallback. operation_opts.tier_delete_journal_api = None; operation_opts.tier_delete_dispatch_authorization = None; - return store.delete_prefix(bucket, object, &operation_opts).await; + let parent_fleet_proof = if chunk_parent_active { + Some( + chunk_parent_fleet_proof + .as_ref() + .filter(|proof| tier_delete_journal_fleet_proof_matches(proof)) + .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?, + ) + } else { + None + }; + store.delete_prefix(bucket, object, &operation_opts).await?; + if let Some(parent_fleet_proof) = parent_fleet_proof { + #[cfg(all(test, feature = "test-util"))] + crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause( + crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted, + ) + .await; + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + if !Box::pin(complete_tier_delete_chunk_parent( + Arc::clone(api), + bucket, + bucket_incarnation, + object, + bucket_fence, + parent_fleet_proof, + )) + .await? + { + return Err(Error::other("tier delete chunk parent disappeared after final local deletion")); + } + } + return Ok(()); }; let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; let bucket_fence = opts @@ -396,9 +600,19 @@ async fn delete_prefix_with_tier_delete_journal( // Keep every backend generation lease until the whole local operation has // either committed its journal set or returned an ambiguous mutation. let _tier_leases = _leases; - if active.predecessor_replay_required() { + if active.predecessor_replay_required() || active.is_chunked() { replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?; + #[cfg(all(test, feature = "test-util"))] + if active.is_chunked() { + crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause( + crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted, + ) + .await; + } complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?; + if active.is_chunked() { + return Err(Error::other("tier delete chunk completed; retry the next durable batch")); + } return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch")); } let result = store.delete_prefix(bucket, object, &operation_opts).await; @@ -443,7 +657,7 @@ async fn replay_authorized_tier_delete_sources( let authorization = active.authorization(); authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?; - let mut source_objects = std::collections::BTreeSet::new(); + let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len()); for entry in active.entries() { let source = entry .source @@ -453,61 +667,130 @@ async fn replay_authorized_tier_delete_sources( if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) { return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope")); } - source_objects.insert(source.object.clone()); + source_objects.insert(source.object.as_str()); } - let mut deleted = 0; - for object in source_objects { - if bucket_fence.is_lock_lost() { - return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + if let Some(scope) = publication_scope { + if scope.state() == ScannerPublicationCommitState::Admitted { + scope + .try_begin() + .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?; } - let encoded_object = encode_dir_object(&object); - let guards = if opts.delete_prefix_object { - store - .acquire_remaining_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) - .await? - } else { - store - .acquire_all_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) - .await? - }; - authorization.ensure_current(bucket, bucket_incarnation, prefix)?; - if let Some(scope) = publication_scope { - if scope.state() == ScannerPublicationCommitState::Admitted { - scope - .try_begin() - .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?; - } - if !scope.can_commit() { - let _ = scope.mark_indeterminate(); - return Err(StorageError::OperationCanceled); - } - } - let mut replay_opts = opts.clone(); - replay_opts.no_lock = true; - replay_opts.delete_prefix = false; - replay_opts.delete_prefix_object = false; - for guard in &guards { - guard.add_namespace_lock_fence(&mut replay_opts); - } - for pool in &store.pools { - for set in &pool.disk_set { - authorization.ensure_current(bucket, bucket_incarnation, prefix)?; - deleted += set - .replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts) - .await?; - } - } - if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { - return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + if !scope.can_commit() { + let _ = scope.mark_indeterminate(); + return Err(StorageError::OperationCanceled); } } - if let Some(scope) = publication_scope { - let _ = scope.mark_committed(); + + let stopped = Arc::new(AtomicBool::new(false)); + // The caller holds bucket lifecycle WRITE. Each bounded future acquires + // only one logical object's physical lock set and releases it before + // completion; no future nests locks for two object keys. + let make_replay = |object: String| { + let stopped = stopped.clone(); + let authorization = authorization.clone(); + async move { + if stopped.load(Ordering::Acquire) { + return Ok::<_, Error>(0usize); + } + let result = async { + if bucket_fence.is_lock_lost() { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + } + let encoded_object = encode_dir_object(&object); + let guards = if opts.delete_prefix_object { + store + .acquire_remaining_physical_object_write_locks( + "tier_delete_dispatch_predecessor_replay", + bucket, + &encoded_object, + ) + .await? + } else { + store + .acquire_all_physical_object_write_locks( + "tier_delete_dispatch_predecessor_replay", + bucket, + &encoded_object, + ) + .await? + }; + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + if publication_scope.is_some_and(|scope| !scope.can_commit()) { + return Err(StorageError::OperationCanceled); + } + let mut replay_opts = opts.clone(); + replay_opts.no_lock = true; + replay_opts.delete_prefix = false; + replay_opts.delete_prefix_object = false; + for guard in &guards { + guard.add_namespace_lock_fence(&mut replay_opts); + } + let mut deleted = 0usize; + for pool in &store.pools { + for set in &pool.disk_set { + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + deleted = deleted + .checked_add( + set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts) + .await?, + ) + .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?; + } + } + if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + } + Ok(deleted) + } + .await; + if result.is_err() { + stopped.store(true, Ordering::Release); + } + result + } + }; + let mut objects = source_objects.into_iter(); + let mut replays = futures::stream::FuturesUnordered::new(); + for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY { + let Some(object) = objects.next().map(ToOwned::to_owned) else { + break; + }; + replays.push(make_replay(object)); + } + let mut deleted = 0usize; + let mut first_error = None; + while let Some(result) = replays.next().await { + match result { + Ok(count) => { + deleted = deleted + .checked_add(count) + .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?; + } + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + if first_error.is_none() + && !stopped.load(Ordering::Acquire) + && let Some(object) = objects.next().map(ToOwned::to_owned) + { + replays.push(make_replay(object)); + } } if deleted > 0 { super::list_objects::observe_list_objects_mutation(store, bucket).await; } + if let Some(err) = first_error { + if publication_scope.is_some_and(|scope| !scope.can_commit()) + && let Some(scope) = publication_scope + { + let _ = scope.mark_indeterminate(); + } + return Err(err); + } + if let Some(scope) = publication_scope { + let _ = scope.mark_committed(); + } Ok(()) } @@ -6518,6 +6801,28 @@ mod tests { ); } + #[test] + fn tier_delete_walk_only_accepts_explicit_limit_cancellation() { + let cancelled = Error::OperationCanceled; + assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed"))); + } + + #[test] + fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() { + let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)]) + .expect_err("a real walk error must not be hidden by earlier cancellation"); + assert_eq!(err, StorageError::FileAccessDenied); + + assert_eq!( + combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)]) + .expect_err("cancellation must remain visible when there is no real error"), + Error::OperationCanceled + ); + } + impl Drop for BodyCacheHookGuard { fn drop(&mut self) { clear_get_object_body_cache_hook(); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 1e10a226c..6472481a0 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,7 @@ ## Open Items - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. +- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources. diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index a1b3c9fb3..ccf3d9995 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -30,7 +30,7 @@ These are approved-target invariants. A protocol's explicitly labeled current ex | Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence | | Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Current recovery finds the transaction's logical bucket/object/version and checks `TRANSITION_COMPLETE` plus the same remote object, tier, and remote version. It does not compare the recorded data directory, modification time, size, or ETag; the approved target adds that full source comparison | | An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version | -| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable dispatch manifest | Yes, but only after manifest completion and all-pool absence proof | `DispatchAuthorized`, local destructive mutation, every journal `Committed`, then manifest `Completed` | +| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion | | Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners | An old journal and a free-version can coexist during compatibility recovery. That coexistence is evidence of multiple possible owners, not permission to choose one: the journal path must retain its record until the version-specific recovery rule proves which owner is authoritative. @@ -50,11 +50,12 @@ All keys below are objects in the internal metadata bucket. The table gives the | Manual worker result | `rustfs-manual-transition-worker-result-v1` | `ilm/manual-transition/results///.json` | The worker persists it after an actual result; no current GC owner | Immutable job/task key and outcome/reason | Append-only create with `If-None-Match: *` and maximum parity | | Legacy tier-delete journal | Versions 1 through 5 | `ilm/tier-delete-journal/.json` | The deleting path creates it; version-specific journal recovery cleans it | Remote tuple; v2 adds backend identity, v3 exact version, v4 version state, v5 stable source and transaction state | v5 state changes use ETag CAS; v3/v4 recovery rereads and conditionally cleans. Initial legacy-compatible writes can still be unconditional | | Sole-owner tier-delete journal | Version 6 | `ilm/tier-delete-journal-v6//.json` | The manifest coordinator creates/dispatches it; the journal worker deletes the remote object and cleans the record | Exact remote/source/backend identity plus manifest/operation/topology binding; mutable state `Prepared`/`Dispatched`/`Committed` | Create-only and fenced ETag CAS. Record cleanup writes a terminal receipt first only while a decommission run is active; ordinary recovery without one conditionally deletes the exact ETag directly | -| Tier-delete dispatch manifest | Version 1 | `ilm/tier-delete-dispatch-manifests/.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback | +| Tier-delete dispatch manifest | Version 1 | Single dispatch: `ilm/tier-delete-dispatch-manifests/.json`; chunk child: `ilm/tier-delete-dispatch-manifests/chunks//.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback. A child cannot authorize local mutation without the exact active parent binding | +| Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion | | Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts////.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity | | Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal | -`durable_namespace.rs` registers exactly the two tier-journal namespaces, dispatch manifest, transaction, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object. +`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object. ## Durable fences and write primitives @@ -89,8 +90,8 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n | Path | Current acquisition order | Operations allowed while held | Operations forbidden while held | |---|---|---|---| | Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation | -| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; synthetic manifest-operation WRITE lock | Build and write the immutable journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes | -| v6 manifest recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; synthetic manifest-operation WRITE lock | Read/write manifest and journal metadata, verify exact set/digest, authorize, converge, or roll back local records | Remote tier DELETE; per-object worker cleanup; rollback after authorization | +| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition | +| v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery | | v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version | | v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version | | Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes | @@ -242,7 +243,9 @@ The lease interval is 60 seconds. CAS behavior is phase-specific: cancellation t | v5 | Adds stable source identity and transaction state. Recovery proves source/free-version presence across physical sets before deciding abort, retain, or commit | | v6 | Sole-owner record bound to immutable operation/manifest/topology. It is the only new journal format for destructive prefix dispatch | -The v1 manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are: +For a complete source set at or below 200,000 journals, the byte-compatible v1 single manifest remains at the deterministic scope-digest root. For a larger source set, that same root instead contains a strict `chunked_parent` sentinel, and each bounded child uses the unchanged v1 manifest payload at `chunks//.json`. A pre-chunking reader rejects the parent schema and the non-root child path, so it cannot start a competing single dispatch while chunking is active. + +A v1 single/child manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are: ```text Preparing -> DispatchAuthorized -> Completed @@ -251,6 +254,16 @@ Preparing -> DispatchAuthorized -> Completed The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinator owns the whole `Prepared` set and is the only actor that may roll it back or complete the manifest. A per-journal worker cannot remove one prepared member. +The chunk parent stores only monotonic O(1) progress and binds at most one child: + +```text +Active(no child) -> Active(bound Preparing child) +Active(bound Completed child) -> Active(no child, next sequence/count) +Active(no child, final empty rescan and local delete complete) -> Completed +``` + +Child creation is ordered `Preparing` child create, parent binding CAS, journal preparation/dispatch, then child `DispatchAuthorized`. One request exactly replays one bounded child under source-object locks, commits every child journal, marks the child `Completed`, advances the parent, and returns retry-required. A successor request rescans from the prefix start; no listing cursor crosses bucket-lock lifetimes. New or changed source identities are therefore admitted only by a fresh child. Final success requires an error-free, non-truncated scan with no v6 candidate, local prefix deletion under the same bucket fence, and the parent `Completed` CAS. + ### Journal recovery decisions | Record/state and evidence | Unique current owner | Current recovery decision | Remote DELETE admission | @@ -278,11 +291,24 @@ The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinato | `Completed` | Journal workers own member cleanup; coordinator owns final manifest cleanup | Wait for all member records to disappear, then conditionally delete manifest | Journal workers are the remote-delete owners | | Missing member, set/digest mismatch, wrong incarnation/topology, corrupt state, scan ambiguity, or cancellation | No actor acquires new destructive authority | Retain | Fail closed; an authorized operation never rolls back | -Manifest preparation is bounded by 200,000 journals and a 32 MiB record. Journal recovery scans bounded batches with per-entry timeouts and limited concurrency. Those are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely. +### Chunk-parent recovery decisions + +| Parent/child state and evidence | Unique current owner | Current recovery decision | Authority | +|---|---|---|---| +| Active parent with no child | A later prefix-delete retry under bucket WRITE | Retain the parent and rescan from the prefix start | No local or remote deletion | +| Active parent with exact bound `Preparing`/`Aborting`/`Aborted` child | Child manifest coordinator | Retain parent while child recovery rolls back and removes the child | Never authorize or advance that child | +| Active parent with exact bound `DispatchAuthorized` child | The bound child permit or journal recovery | Resume exact-source replay on request; otherwise retain until all journals become `Committed` and the child becomes `Completed` | Only the exact parent-bound child may authorize local replay | +| Active parent with exact bound `Completed` child | Parent coordinator | CAS the next sequence/count and clear the binding | Parent progress only; remote DELETE remains owned by committed child journals | +| Bound child missing and its exact operation journal namespace is non-empty or unreadable | No actor can prove safe abandonment | Retain and fail closed | Never clear the binding | +| Bound child missing and its exact operation journal namespace is proven empty | Parent coordinator | CAS-clear the stale binding and retry from the prefix start | No deletion; the fresh scan reconstructs any remaining source work | +| Completed parent with no active child | Parent recovery | Record terminal decommission evidence when applicable, then conditionally delete the exact parent ETag | Metadata cleanup only | +| Parent identity, child binding, topology, incarnation, sequence/count, CAS generation, or fence mismatches | No actor acquires progress authority | Retain | Fail closed | + +Single and child manifest preparation is bounded by 200,000 journals and a 32 MiB record. On the first unique candidate beyond the bound, the physical walks are cancelled and only the retained exact batch can proceed; cancellation fallout is not absence proof. The parent never accumulates child names, and exact local replay uses bounded concurrency. Journal and manifest recovery retain their existing bounded pages, per-entry timeouts, and concurrency. These are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely. ### Approved target and open design -- New destructive prefix paths use only v6 plus a manifest. No new v1-v5 sole-owner records may be created. +- New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created. - Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE. - Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous. - **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof. @@ -309,9 +335,9 @@ How historical objects without RustFS transition-version-state can be upgraded s ### Current contract -Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads. +Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads. -The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. +The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target. @@ -371,7 +397,7 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr | Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled | | Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof | | Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization | -| Journal v6 and dispatch manifest v1 | v6-aware writers/readers require immutable manifest membership and topology; v5-and-older readers reject and retain v6 | Gate v6 writers on fleet capability. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation | +| Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation | | Decommission receipt v2 and expected manifest v1 | Current decommission readers validate exact schema/checksum/path/checkpoint and fail completion on unknown input | No ignore path. Mixed-version decommission must not complete unless every participant preserves the registered durable namespace; broader downgrade negotiation is open | ## Reconcile, observability, and retention From ff3c5a4989efde7734a6239e0a11311920d32a39 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 18:46:19 +0800 Subject: [PATCH 11/19] fix(ilm): drain tier-delete recovery pages (#7133) --- .../bucket/lifecycle/tier_delete_journal.rs | 320 ++++++++++++++---- crates/ecstore/src/store/init.rs | 225 +++++++++++- 2 files changed, 467 insertions(+), 78 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index ed502e570..3a6a7e451 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -61,12 +61,13 @@ pub(crate) const LOG_COMPONENT_ECSTORE: &str = "ecstore"; pub(crate) const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; pub(crate) const EVENT_LIFECYCLE_TIER_DELETE_JOURNAL: &str = "lifecycle_tier_delete_journal"; -// Keep one background pass small enough that a slow remote tier cannot hold -// the shared recovery worker for minutes. Subsequent passes resume from the -// returned marker, so this bounds latency without reducing eventual coverage. +// Keep one page small enough that slow remote tiers remain bounded by per-entry +// deadlines and worker concurrency. A production pass may consume multiple +// successful pages while its wall-clock budget remains. pub const DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT: usize = 8; const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60); const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300); +const TIER_DELETE_JOURNAL_RECOVERY_PASS_BUDGET: Duration = Duration::from_secs(240); const TIER_DELETE_REMOTE_DEADLINE: Duration = Duration::from_secs(30); const TIER_DELETE_JOURNAL_ENTRY_RECOVERY_TIMEOUT: Duration = Duration::from_secs(90); const TIER_DELETE_JOURNAL_RECOVERY_CONCURRENCY: usize = 4; @@ -1085,7 +1086,7 @@ fn validate_version_state( Ok(()) } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct TierDeleteJournalRecoveryStats { pub scanned: usize, pub deleted: usize, @@ -1094,7 +1095,7 @@ pub struct TierDeleteJournalRecoveryStats { pub truncated: bool, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct TierDeleteDispatchManifestRecoveryStats { pub scanned: usize, pub advanced: usize, @@ -1105,6 +1106,18 @@ pub struct TierDeleteDispatchManifestRecoveryStats { pub truncated: bool, } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct TierDeleteJournalRecoveryPassStats { + pub manifest_pages: usize, + pub journal_pages: usize, + pub manifest_errors: usize, + pub journal_errors: usize, + pub deadline_exhausted: bool, + pub canceled: bool, + pub manifests: TierDeleteDispatchManifestRecoveryStats, + pub journals: TierDeleteJournalRecoveryStats, +} + pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String { let digest = tier_delete_journal_identity_digest(je); if let Some(dispatch) = &je.dispatch { @@ -5660,6 +5673,201 @@ pub async fn recover_tier_delete_journal_entries( Ok(stats) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TierDeleteJournalRecoveryPageStep { + stop_queue: bool, + made_progress: bool, +} + +fn remaining_recovery_pass_budget( + deadline: tokio::time::Instant, + stats: &mut TierDeleteJournalRecoveryPassStats, +) -> Option { + match deadline.checked_duration_since(tokio::time::Instant::now()) { + Some(remaining) if !remaining.is_zero() => Some(remaining), + _ => { + stats.deadline_exhausted = true; + None + } + } +} + +fn accumulate_dispatch_manifest_recovery_stats( + total: &mut TierDeleteDispatchManifestRecoveryStats, + page: TierDeleteDispatchManifestRecoveryStats, +) { + total.scanned += page.scanned; + total.advanced += page.advanced; + total.deleted += page.deleted; + total.retained += page.retained; + total.failed += page.failed; + total.next_marker = page.next_marker; + total.truncated = page.truncated; +} + +fn accumulate_tier_delete_journal_recovery_stats( + total: &mut TierDeleteJournalRecoveryStats, + page: TierDeleteJournalRecoveryStats, +) { + total.scanned += page.scanned; + total.deleted += page.deleted; + total.failed += page.failed; + total.next_marker = page.next_marker; + total.truncated = page.truncated; +} + +async fn recover_tier_delete_dispatch_manifest_pass_page( + api: Arc, + cancel_token: &CancellationToken, + marker: &mut Option, + deadline: tokio::time::Instant, + stats: &mut TierDeleteJournalRecoveryPassStats, +) -> Option { + if cancel_token.is_cancelled() { + stats.canceled = true; + return None; + } + let remaining = remaining_recovery_pass_budget(deadline, stats)?; + let page_marker = marker.clone(); + let recovery = recover_tier_delete_dispatch_manifests(api, DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, page_marker.clone()); + let page = + await_tier_delete_journal_recovery(cancel_token, remaining.min(TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT), recovery).await; + let Some(page) = page else { + stats.canceled = true; + return None; + }; + match page { + Ok(page) => { + stats.manifest_pages += 1; + let made_progress = page.advanced > 0 || page.deleted > 0; + let stop_queue = !page.truncated || !made_progress; + *marker = page.next_marker.clone(); + accumulate_dispatch_manifest_recovery_stats(&mut stats.manifests, page); + Some(TierDeleteJournalRecoveryPageStep { + stop_queue, + made_progress, + }) + } + Err(err) => { + stats.manifest_errors += 1; + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + next_marker = ?page_marker, + error = ?err, + "Failed to recover tier delete dispatch manifest page" + ); + Some(TierDeleteJournalRecoveryPageStep { + stop_queue: true, + made_progress: false, + }) + } + } +} + +async fn recover_tier_delete_journal_pass_page( + api: Arc, + cancel_token: &CancellationToken, + marker: &mut Option, + deadline: tokio::time::Instant, + stats: &mut TierDeleteJournalRecoveryPassStats, +) -> Option { + if cancel_token.is_cancelled() { + stats.canceled = true; + return None; + } + let remaining = remaining_recovery_pass_budget(deadline, stats)?; + let page_marker = marker.clone(); + let recovery = recover_tier_delete_journal_entries(api, DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, page_marker.clone()); + let page = + await_tier_delete_journal_recovery(cancel_token, remaining.min(TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT), recovery).await; + let Some(page) = page else { + stats.canceled = true; + return None; + }; + match page { + Ok(page) => { + stats.journal_pages += 1; + let made_progress = page.deleted > 0; + let stop_queue = !page.truncated || !made_progress; + *marker = page.next_marker.clone(); + accumulate_tier_delete_journal_recovery_stats(&mut stats.journals, page); + Some(TierDeleteJournalRecoveryPageStep { + stop_queue, + made_progress, + }) + } + Err(err) => { + stats.journal_errors += 1; + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + next_marker = ?page_marker, + error = ?err, + "Failed to recover tier delete journal page" + ); + Some(TierDeleteJournalRecoveryPageStep { + stop_queue: true, + made_progress: false, + }) + } + } +} + +async fn recover_tier_delete_journal_pass( + api: Arc, + cancel_token: &CancellationToken, + manifest_marker: &mut Option, + journal_marker: &mut Option, + budget: Duration, +) -> TierDeleteJournalRecoveryPassStats { + let deadline = tokio::time::Instant::now() + budget; + let mut stats = TierDeleteJournalRecoveryPassStats::default(); + let mut manifest_stopped = false; + let mut journal_stopped = false; + + while !manifest_stopped || !journal_stopped { + let mut made_progress = false; + if !manifest_stopped { + let Some(step) = + recover_tier_delete_dispatch_manifest_pass_page(api.clone(), cancel_token, manifest_marker, deadline, &mut stats) + .await + else { + return stats; + }; + manifest_stopped = step.stop_queue; + made_progress |= step.made_progress; + } + if !journal_stopped { + let Some(step) = + recover_tier_delete_journal_pass_page(api.clone(), cancel_token, journal_marker, deadline, &mut stats).await + else { + return stats; + }; + journal_stopped = step.stop_queue; + made_progress |= step.made_progress; + } + if !made_progress { + break; + } + } + + stats +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn recover_test_tier_delete_journal_pass_with_budget( + api: Arc, + cancel_token: &CancellationToken, + manifest_marker: &mut Option, + journal_marker: &mut Option, + budget: Duration, +) -> TierDeleteJournalRecoveryPassStats { + recover_tier_delete_journal_pass(api, cancel_token, manifest_marker, journal_marker, budget).await +} + pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_token: CancellationToken) { let mut interval = tokio::time::interval(TIER_DELETE_JOURNAL_RECOVERY_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -5674,78 +5882,40 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_tok _ = api.ctx.wait_for_tier_delete_journal_recovery() => {}, } - let manifest_recovery = recover_tier_delete_dispatch_manifests( + let stats = recover_tier_delete_journal_pass( api.clone(), - DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, - manifest_marker.clone(), + &cancel_token, + &mut manifest_marker, + &mut marker, + TIER_DELETE_JOURNAL_RECOVERY_PASS_BUDGET, + ) + .await; + if stats.canceled { + return; + } + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_pages = stats.manifest_pages, + manifest_scanned = stats.manifests.scanned, + manifest_advanced = stats.manifests.advanced, + manifest_deleted = stats.manifests.deleted, + manifest_retained = stats.manifests.retained, + manifest_failed = stats.manifests.failed, + manifest_errors = stats.manifest_errors, + manifest_truncated = stats.manifests.truncated, + manifest_next_marker = ?manifest_marker, + journal_pages = stats.journal_pages, + journal_scanned = stats.journals.scanned, + journal_deleted = stats.journals.deleted, + journal_failed = stats.journals.failed, + journal_errors = stats.journal_errors, + journal_truncated = stats.journals.truncated, + journal_next_marker = ?marker, + deadline_exhausted = stats.deadline_exhausted, + "Recovered tier delete journal pass" ); - let Some(manifest_result) = - await_tier_delete_journal_recovery(&cancel_token, TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT, manifest_recovery).await - else { - return; - }; - match manifest_result { - Ok(stats) => { - manifest_marker = stats.next_marker; - debug!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - scanned = stats.scanned, - advanced = stats.advanced, - deleted = stats.deleted, - retained = stats.retained, - failed = stats.failed, - truncated = stats.truncated, - next_marker = ?manifest_marker, - "Reconciled tier delete dispatch manifests" - ); - } - Err(err) => { - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - next_marker = ?manifest_marker, - error = ?err, - "Failed to recover tier delete dispatch manifests" - ); - } - } - - let recovery = - recover_tier_delete_journal_entries(api.clone(), DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, marker.clone()); - let Some(result) = - await_tier_delete_journal_recovery(&cancel_token, TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT, recovery).await - else { - return; - }; - match result { - Ok(stats) => { - marker = stats.next_marker; - debug!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - scanned = stats.scanned, - deleted = stats.deleted, - failed = stats.failed, - truncated = stats.truncated, - next_marker = ?marker, - "Recovered tier delete journal tasks" - ); - } - Err(err) => { - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - next_marker = ?marker, - error = ?err, - "Failed to recover tier delete journal tasks" - ); - } - } } } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 00a74906e..a89ef05f1 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -821,9 +821,9 @@ mod tests { TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry, install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch, recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget, - recover_tier_delete_dispatch_manifests, recover_tier_delete_journal_entries, - test_tier_delete_dispatch_manifest_checkpoint, test_tier_delete_dispatch_manifest_state, - tier_delete_dispatch_manifest_operation_lock_held_for_test, + recover_test_tier_delete_journal_pass_with_budget, recover_tier_delete_dispatch_manifests, + recover_tier_delete_journal_entries, test_tier_delete_dispatch_manifest_checkpoint, + test_tier_delete_dispatch_manifest_state, tier_delete_dispatch_manifest_operation_lock_held_for_test, tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test, tier_delete_journal_object_name, }, @@ -8697,6 +8697,35 @@ mod tests { .len() } + #[cfg(feature = "test-util")] + async fn install_committed_tier_delete_journals( + store: Arc, + tier_name: &str, + backend_identity: [u8; 32], + count: usize, + ) -> Vec { + let mut entries = Vec::with_capacity(count); + for index in 0..count { + let entry = Jentry { + persisted_version: 0, + obj_name: format!("remote/pass-drain-{index:06}.bin"), + version_id: uuid::Uuid::new_v4().to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + dispatch: None, + }; + persist_tier_delete_journal_entry(store.clone(), &entry) + .await + .expect("committed tier-delete journal fixture should persist"); + entries.push(entry); + } + entries + } + #[cfg(feature = "test-util")] fn synthetic_v6_dispatch_entry( bucket: &str, @@ -8793,6 +8822,196 @@ mod tests { .expect("same-disk restart recovery should converge without retained dispatch state"); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_delete_recovery_pass_drains_multiple_fast_manifest_pages() { + const MANIFEST_COUNT: usize = 10; + + let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-manifest-pass", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "tier-delete-fast-manifest-pass-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fast manifest pass bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("fast manifest pass bucket incarnation should resolve"); + let tier_name = "FAST-MANIFEST-PASS"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("fast manifest pass tier lease should resolve") + .backend_identity(); + for index in 0..MANIFEST_COUNT { + install_aborting_dispatch_fixture( + store.clone(), + bucket, + incarnation, + &format!("manifest-page-{index:06}/"), + tier_name, + backend_identity, + 1, + ) + .await; + } + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT); + assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT); + + let cancel = CancellationToken::new(); + let mut manifest_marker = None; + let mut journal_marker = None; + let stats = recover_test_tier_delete_journal_pass_with_budget( + store.clone(), + &cancel, + &mut manifest_marker, + &mut journal_marker, + Duration::from_secs(30), + ) + .await; + + assert!(!stats.canceled); + assert!(!stats.deadline_exhausted); + assert!( + stats.manifest_pages > 1, + "one production pass must cross the default eight-manifest page limit" + ); + assert_eq!(stats.manifests.scanned, MANIFEST_COUNT); + assert_eq!(stats.manifests.deleted, MANIFEST_COUNT); + assert_eq!(stats.manifests.failed, 0); + assert_eq!(manifest_marker, None); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_delete_recovery_pass_drains_multiple_fast_journal_pages() { + const JOURNAL_COUNT: usize = 25; + + let temp_dir = tempfile::tempdir().expect("create fast pass recovery store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-pass", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "FAST-PASS-RECOVERY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("fast pass tier lease should resolve") + .backend_identity(); + let entries = install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await; + assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT); + + let cancel = CancellationToken::new(); + let mut manifest_marker = None; + let mut journal_marker = None; + let stats = recover_test_tier_delete_journal_pass_with_budget( + store.clone(), + &cancel, + &mut manifest_marker, + &mut journal_marker, + Duration::from_secs(30), + ) + .await; + + assert!(!stats.canceled); + assert!(!stats.deadline_exhausted); + assert!( + stats.journal_pages > 1, + "one production pass must cross the default eight-record page limit" + ); + assert_eq!(stats.journals.scanned, JOURNAL_COUNT); + assert_eq!(stats.journals.deleted, JOURNAL_COUNT); + assert_eq!(stats.journals.failed, 0); + assert_eq!(journal_marker, None); + assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!(backend.exact_remove_count(), JOURNAL_COUNT); + let mut removed = backend.remove_versions().await; + removed.sort(); + let mut expected = entries + .into_iter() + .map(|entry| (entry.obj_name, entry.version_id)) + .collect::>(); + expected.sort(); + assert_eq!(removed, expected); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_delete_recovery_pass_rotates_failed_journal_pages_without_busy_loop() { + const JOURNAL_COUNT: usize = 9; + + let temp_dir = tempfile::tempdir().expect("create failed pass recovery store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-failed-pass", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "FAILED-PASS-RECOVERY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("failed pass tier lease should resolve") + .backend_identity(); + install_committed_tier_delete_journals(store.clone(), tier_name, backend_identity, JOURNAL_COUNT).await; + backend.set_remove_failure(true); + + let cancel = CancellationToken::new(); + let mut manifest_marker = None; + let mut journal_marker = None; + let first = recover_test_tier_delete_journal_pass_with_budget( + store.clone(), + &cancel, + &mut manifest_marker, + &mut journal_marker, + Duration::from_secs(30), + ) + .await; + assert_eq!(first.journal_pages, 1, "a no-progress failed page must stop the pass"); + assert_eq!(first.journals.scanned, 8); + assert_eq!(first.journals.deleted, 0); + assert_eq!(first.journals.failed, 8); + assert!(journal_marker.is_some(), "a truncated failed page must retain its continuation marker"); + assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT); + + let second = recover_test_tier_delete_journal_pass_with_budget( + store.clone(), + &cancel, + &mut manifest_marker, + &mut journal_marker, + Duration::from_secs(30), + ) + .await; + assert_eq!(second.journal_pages, 1); + assert_eq!(second.journals.scanned, 1); + assert_eq!(second.journals.deleted, 0); + assert_eq!(second.journals.failed, 1); + assert_eq!( + journal_marker, None, + "end-of-list rotation must revisit the failed prefix on a later pass" + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT); + + backend.set_remove_failure(false); + let third = recover_test_tier_delete_journal_pass_with_budget( + store.clone(), + &cancel, + &mut manifest_marker, + &mut journal_marker, + Duration::from_secs(30), + ) + .await; + assert_eq!(third.journals.scanned, JOURNAL_COUNT); + assert_eq!(third.journals.deleted, JOURNAL_COUNT); + assert_eq!(third.journals.failed, 0); + assert_eq!(tier_delete_journal_count(store).await, 0); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] From 81014fd23358a0baaab265bfb530b4109f7f9a6a Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 19:53:36 +0800 Subject: [PATCH 12/19] docs(ilm): approve legacy tier state reconciliation (#7142) --- .../ilm-tiering-persistence-contracts.md | 84 ++++++++++++++++++- docs/operations/tier-ilm-debugging.md | 41 +++++++-- 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index ccf3d9995..a7427ae74 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -327,9 +327,89 @@ Recursive prefix/delete-all is the exception because physical directory removal - Every code path that removes or overwrites transitioned metadata either atomically leaves an exact free-version owner or enters an already-authorized v6 dispatch. - If both a journal and free-version are observable, recovery preserves the remote object until version-specific authority and all-pool absence prove a single owner. -- Missing `transition-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID. +- Missing `transitioned-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID. -How historical objects without RustFS transition-version-state can be upgraded safely is an **open design**. It requires a read/repair rule with rollback behavior before destructive cleanup may consume those records. +The following single-record protocol approves how historical objects without RustFS `transitioned-version-state` can be upgraded. It does not approve a bulk scanner or allow destructive cleanup to consume an unproven record. + +## Legacy transitioned-version-state reconciliation + +### Current + +An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. The current GET and free-version cleanup paths reject that state rather than interpreting an empty remote version as unversioned. There is no admin route that repairs this field in `xl.meta`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation. + +An explicitly persisted `unknown`, a malformed state, conflicting RustFS/MinIO compatibility keys, an invalid or nil version identifier, and a partial transition tuple are not legacy absence. They remain invalid or ambiguous and fail closed. + +### Approved single-record control surface + +The approved target is one synchronous, exact logical-version operation. It does not list a bucket or prefix and does not create a durable job: + +```text +GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +``` + +`versionId` is required; the literal `null` is the explicit selector for a locally unversioned object, while an omitted or empty selector is invalid. GET requires `admin:ListTier`. It performs an authoritative all-pool metadata read and a bounded server-side live backend probe, but it never writes metadata or mutates the remote tier. Its response includes the canonical immutable source tuple, every original per-set missing-state representation, the proposed target tuple when one is provable, an opaque reconciliation digest over all three, fleet/topology and tier-generation readiness, and whether POST is ready to attempt migration. GET never labels an unmodified legacy record `migrated`. + +POST requires `admin:SetTier`. Its body contains `confirm: true`, the complete source and target tuples, every original per-set representation, and their reconciliation digest returned by a recent GET. The server does not trust a client-supplied state or external assertion: it rereads the object, requires every immutable source field to match exactly after canonicalization, repeats the bounded live probe, and derives the target state itself. For the mutable state/version/destination fields, each set must equal either its original missing-state representation bound by the digest or the exact newly proven target; this is the only accepted partial-retry shape. A missing confirmation, any other stale tuple or digest, or a widened selector is rejected before any write. + +The expected tuple binds all evidence whose change could redirect the repair: + +- bucket name and incarnation; exact object name, local version ID, data directory, modification time, size, and ETag; +- transition completion status, tier name, canonical remote object name, raw remote-version key presence/value, and raw state-key presence/value under both compatibility prefixes; +- tier-config generation and the `transition-tier-destination-id` binding, including backend type, endpoint/bucket/prefix identity, and the credential-independent backend fingerprint; +- topology generation and the generation/digest of every authoritative `xl.meta` copy found across pools and sets. + +The only allowed state derivations are: + +| Live proof | Persisted state | Persisted remote version | Remote request meaning | +|---|---|---|---| +| Provider proves versioning disabled and the candidate is present | `KnownDisabled` | Absent/empty | Send no `versionId` | +| Provider proves suspended-version null semantics and the exact candidate is present | `SuspendedNull` | Literal `null` | Send the provider's null-version form | +| Provider proves one exact, nonempty, non-`null` opaque version | `Exact` | That exact opaque identifier | Send that exact `versionId` | + +Missing, multiple, changing, or unsupported probe results do not select a state. In particular, a preexisting empty remote-version field is not evidence for `KnownDisabled`, and a client may not nominate `Exact` or `SuspendedNull`. + +The POST may write only the derived `transitioned-version-state`, its corresponding `transitioned-versionID` value when the proven model requires one, and the exact `transition-tier-destination-id` binding under both compatibility prefixes in the matching `xl.meta` version. It does not issue remote GET beyond the proof probe, remote PUT, remote DELETE, local object DELETE, free-version cleanup, transaction/journal cleanup, tier-config mutation, restore, or source-payload rewrite. Reconciliation establishes metadata meaning; a later ordinary owner may perform cleanup under its own destructive protocol. + +### Outcome contract + +POST returns exactly one of the following outcomes and whether it changed bytes. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged: + +| Outcome | Meaning and permitted effect | +|---|---| +| `migrated` | All authoritative copies already contain, or were monotonically advanced to, the same proven state and destination identity. Only this outcome makes the record eligible for later ordinary read/delete semantics. | +| `retained-ambiguous` | The tuple is structurally legacy-compatible, but the live probe is missing, multiple, changing, unsupported, or otherwise cannot prove exactly one state. No metadata or remote object is changed. | +| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established, and nothing is changed. | +| `backend-unavailable` | The bound tier generation/destination cannot be acquired, the bounded probe fails, or a metadata quorum/strong readback needed to complete the operation is unavailable. Any already-persisted monotonic subset is retained for an idempotent retry; it is never rolled back. | + +HTTP failure detail may distinguish a stale expected tuple, lost fence, timeout, or unavailable quorum, but it must preserve one of these machine-readable outcomes. Logs and audit events include request identity, object identity, tier, generations, outcome, and whether bytes changed; they never include credentials or raw credential-derived configuration. + +### Fence, write, and retry order + +The approved POST executes the following order. A step that cannot be proven stops the operation without remote mutation: + +1. Authenticate `admin:SetTier`, validate the exact single-record selector, `confirm: true`, expected tuple, and digest. +2. Prove every required node advertises the reconciliation format and destination-identity capability; capture the fleet and topology generation. Unknown or unsupported nodes block the writer. +3. Acquire the bucket-lifecycle WRITE fence and validate the bucket incarnation. +4. Acquire the exact tier-config generation lease bound to the expected destination identity. +5. Acquire exact object-version WRITE locks for every owning physical pool/set in stable pool/set order. +6. Perform an authoritative all-pool read, reject duplicate/conflicting ownership, validate every compatibility key, and require each set to match the immutable source tuple plus either its digest-bound original missing-state representation or the exact proposed target. +7. Run one bounded, cancellation-aware live probe through the leased backend. No client or cached probe result is authority. +8. Before writing, revalidate the fleet/topology generation, bucket incarnation and lifecycle fence, tier lease/destination identity, physical owner set, and complete metadata tuple. +9. Write the same derived state and destination identity to each authoritative set with that set's metadata quorum and conditional generation. A timeout or response loss is resolved only by a strong read of that exact set. +10. Strongly reread every authoritative set and revalidate the full tuple, state, destination identity, and topology before returning `migrated`. Release locks and leases in reverse order. + +Cross-pool and cross-set partial success is monotonic. The only legal repair edge is `missing state -> one proven {state, remote version, destination identity}`. A retry may accept an already-written subset only when every known copy equals the newly proven target, every remaining copy equals its original missing-state representation captured by the reconciliation digest, and all immutable source fields still match; it then fills only the missing copies. This exact target-plus-original subset is neither stale nor corrupt. The retry never clears a known state, rewrites it to another state, changes destination identity, or rolls a successful set back to missing/`Unknown`. Any other divergent value produces `corrupt`; an unavailable set/readback produces `backend-unavailable`, and destructive cleanup remains blocked until a later strong all-pool read proves complete convergence. + +GET takes the same fleet/topology snapshot and authoritative all-pool read but no write locks that imply mutation authority. Because GET is advisory, POST always repeats every fence, read, and live proof rather than promoting the GET result. + +### Mixed-version and future batch work + +The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. Downgrade is blocked while reconciled records could be rewritten by readers that erase or misinterpret those fields. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient. + +Explicit `Unknown`, corruption, and ambiguity remain fail closed for reads that cannot prove non-destructive semantics and for every destructive path. A migrated record becomes ordinary explicit metadata, but reconciliation itself never transfers remote DELETE ownership. + +A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a separate durable job identity, create-only admission, lease/CAS checkpoint, bounded pages, per-record expected tuples and outcomes, cancellation/restart semantics, retention, fleet rollout negotiation, and status counters. Implementations must not approximate that protocol by adding a list selector or background loop to the synchronous route. ## Durable namespace receipts during decommission diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 44b9de648..5aa59c316 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -39,11 +39,11 @@ Internal metadata is stored under both `x-rustfs-internal-` and `x-minio |--------|---------| | `transition-status` | `"complete"` when tiered | | `transitioned-object` | tier key path (stored without the tier prefix; `get_dest` adds it) | -| `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) | +| `transitioned-versionID` | Provider version identifier: current exact UTF-8 text, legacy RustFS raw UUID bytes, MinIO's empty unversioned value, or absent for some historical unversioned records. Interpret it only with `transitioned-version-state` or a live compatibility probe. | | `transition-tier` | tier name | | `tier-free-versionID` | delete-marker version for free-version sweep | -Reading binary values must reject empty, malformed, and nil values (regression covered in `crates/filemeta/src/filemeta/version.rs` tests): +Legacy raw UUID values must reject empty, malformed, and nil UUIDs (regression covered in `crates/filemeta/src/filemeta/version.rs` tests): ```rust get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) @@ -52,7 +52,7 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) // None for: absent key, wrong-length bytes, nil UUID ``` -`transition_version_id == None` means the tier bucket is unversioned; the GET/DELETE against the tier must then send no `versionId` parameter. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. +`transition_version_id == None` means only that no usable legacy UUID projection exists; it does not prove the remote bucket's versioning model. Only an explicit `KnownDisabled` state authorizes ordinary GET/DELETE to omit `versionId`. A missing state with an absent or empty version key remains `Unknown` and requires the bounded compatibility probe or the approved reconcile workflow; it never directly authorizes cleanup. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. ## Inspect xl.meta directly @@ -64,8 +64,8 @@ cargo build -p rustfs-filemeta --example dump_fileinfo | Output | Meaning | |---|---| -| `transition_ver_id: ` | No versionId will be sent to the tier (correct for a non-versioned tier bucket). | -| `transition_ver_id: ` | That UUID will be sent as `?versionId=`. | +| `transition_ver_id: ` | No usable legacy UUID projection exists. Inspect `transitioned-version-state` and the raw compatibility keys; do not infer unversioned semantics. | +| `transition_ver_id: ` | A legacy UUID representation decoded successfully. It is not destructive authority unless the persisted state or reconcile proof establishes the exact remote model. | There is one `xl.meta` per erasure shard disk (`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should be identical. `dump_versions` (same crate) lists every version in a file. @@ -80,7 +80,7 @@ RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs ... | `fetching transitioned object from tier` | DEBUG | Emitted before the tier request. | | `tier GET failed` | ERROR | Includes `tier_version_id`. | -If both `x-rustfs-internal-transitioned-versionID` and `x-minio-internal-transitioned-versionID` are the empty string, the object was transitioned to a non-versioned tier bucket and no versionId must be sent. +If the version keys are empty while `transitioned-version-state` is absent, the record has the historical MinIO unversioned shape but still remains `Unknown`; only the compatibility probe or reconcile protocol may prove `KnownDisabled`. If state is explicitly `KnownDisabled`, no `versionId` is sent. ## Manual transition run @@ -154,6 +154,35 @@ Historical transition transactions in `upload_outcome_unknown` state can use an `finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically. +## Reconcile legacy transition-version metadata + +This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version. + +The approved interface is synchronous and accepts exactly one bucket/object/local-version tuple: + +```text +GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +``` + +`versionId` is required; use the literal `null` for a locally unversioned object. An omitted or empty selector is invalid. GET requires `admin:ListTier`. It reports the authoritative all-pool tuple, destination identity, fleet/topology readiness, live probe classification, opaque expected-tuple digest, and a machine-readable diagnosis. It returns `ready-to-migrate`, not `migrated`, when a missing state is provable because GET is read-only. + +POST requires `admin:SetTier`, `confirm: true`, and the complete immutable source tuple, original per-set missing-state representations, proposed target, and reconciliation digest returned by GET. The server rereads every authoritative copy and repeats the bounded live backend probe; provider console output or an operator-supplied state is diagnostic evidence only, never write authority. A retry accepts only copies that still match their digest-bound original representation or already equal the exact proven target; any other divergence is stale or corrupt. The server may persist only one of these exact state/version pairs, together with the bound destination identity: + +| Proven remote model | State | Version value | +|---|---|---| +| Versioning disabled | `KnownDisabled` | Empty/absent; later requests omit `versionId` | +| Versioning suspended null object | `SuspendedNull` | Literal `null` | +| One exact version | `Exact` | Exact nonempty, non-`null` opaque identifier | + +The response outcome is `migrated`, `retained-ambiguous`, `corrupt`, or `backend-unavailable`. `migrated` means strong all-pool readback proved the same state and destination identity on every authoritative copy; it can be idempotent with `changed=false`. Ambiguous/missing/multiple probe results are retained, and explicit `Unknown`, malformed or conflicting dual keys, nil identifiers, partial tuples, or copies outside the exact `{original missing representation, proven target}` retry subset fail closed. An unavailable backend, tier generation, metadata quorum, or required strong readback reports `backend-unavailable`; a monotonic partial write is retained for retry and never rolled back. + +The POST does not issue remote DELETE or PUT, remove local data, create a free-version, clean a transaction/journal, or change tier configuration. It holds the approved fleet/topology, bucket-lifecycle, exact tier-generation/destination, and stable all-pool object-version fences across authoritative reread and the bounded probe; it rechecks them before quorum writes and after strong readback. A fleet containing a node that cannot preserve the explicit state/destination binding is inspect-only, and a cross-pool first match is never enough. + +There is intentionally no bucket, prefix, or fleet selector. Batch repair requires a separate durable, resumable job protocol and remains future work. Until the single-record route is implemented, retain affected metadata, use external inspection only for diagnosis, and never hand-edit `xl.meta` or enable remote cleanup by assuming that an empty version field means an unversioned tier. + +The full approved fence, quorum, cross-set retry, destination-binding, and mixed-version contract is specified in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#legacy-transitioned-version-state-reconciliation). + ## Invariant: local-first expiry ordering `expire_transitioned_object` deletes local metadata first (making the object unreachable) and leaves a persisted free-version for remote-tier cleanup. `tier_free_version_recovery.rs` scans and re-enqueues that record; the lifecycle worker's `cleanup_free_version_exact` in `bucket_lifecycle_ops.rs` performs the fenced remote delete, local-marker cleanup, and rescan. Never remove a remote tier version while live local metadata still points at it: doing so lets a concurrent GET read a stored version_id whose remote version is already gone and fail with `NoSuchVersion`. From 507447da12a56cb8b1aec2c203b7c9e88d8bdc1e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 20:04:28 +0800 Subject: [PATCH 13/19] fix(restore): reject SELECT restore and keep typed S3 errors (#7113) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(restore): reject SELECT restore and keep typed S3 errors RestoreObject accepted `Type=SELECT` requests, but the restore path can only write the retrieved bytes back to the source key: `put_restore_opts` built SELECT output options and `restore_transitioned_object` then PUT them over the source bucket/object. On an unversioned bucket that dropped `x-amz-restore`, user metadata and tags from the live object; on a versioned bucket it published a bogus latest version. Nothing was ever written to `OutputLocation.S3`, yet the response still carried a fabricated `x-amz-restore-output-path`. Reject SELECT at the API boundary with a typed NotImplemented, before any guard or metadata write, and fail closed in `put_restore_opts` as the backstop for any other caller. Every other RestoreObject failure was collapsed into a `Custom` error code, which serializes as a generic retryable 500: a missing key or version, a malformed version-id, an object that was never transitioned, an illegal `Days`, and authorization or storage failures all looked the same to a client. Map them to their S3 identities instead — NoSuchKey, NoSuchVersion, InvalidArgument, InvalidObjectState, InvalidRequest, MalformedXML — by preserving `StorageError` through `post_restore_opts` and letting `ApiError` do the mapping. The intentional 409 RestoreAlreadyInProgress and 503 SlowDown behaviour is unchanged, and request validation now runs before any lock is taken. backlog#1341, backlog#2205 * test(restore): give the typed-error regression the ecstore test stack `execute_restore_object_maps_failures_to_typed_s3_errors` builds a real ECStore fixture, and under nextest each test runs in a spawned thread with libtest's 2 MiB stack. On Linux CI that overflowed: the test aborted with SIGABRT / "fatal runtime error: stack overflow" while every other test in the run passed. Add it to the `ecstore-base-stack` filter in both the default and ci profiles, alongside the other `package(rustfs)` tests that drive the same store fixture. 4 MiB matches what the deeper multipart and access roundtrips already use. --- .config/nextest.toml | 4 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 78 ++---- rustfs/src/app/object/mod.rs | 10 +- rustfs/src/app/object/restore.rs | 223 +++++++++++++----- rustfs/src/app/storage_api.rs | 3 +- 5 files changed, 199 insertions(+), 119 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index c6eaf1908..bc57b8b8f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.default.scripts]] @@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.ci.scripts]] diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 7a8b65f75..7649e3ac9 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -87,16 +87,12 @@ use rustfs_filemeta::{ use rustfs_scanner_metrics::metrics::{ IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics, }; -use rustfs_utils::{ - get_env_i64, get_env_usize, - path::encode_dir_object, - string::{parse_bool, strings_has_prefix_fold}, -}; +use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool}; use s3s::dto::{ BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, Timestamp, }; -use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; +use s3s::header::X_AMZ_RESTORE; use sha2::{Digest, Sha256}; use std::any::Any; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" )] pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; -pub const AMZ_ENCRYPTION_AES: &str = "AES256"; #[allow( dead_code, reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" @@ -4802,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation reader } -pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result { +/// Resolve the RestoreObject request options. +/// +/// Returns the typed [`StorageError`]: flattening these into an opaque +/// `io::Error` string erased the identity the S3 layer needs to answer +/// InvalidArgument instead of a generic 500 (backlog#2205). +pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result { let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await; let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await; let vid = version_id.trim(); if !vid.is_empty() && vid != NULL_VERSION_ID { if let Err(_err) = Uuid::parse_str(vid) { - return Err(std::io::Error::other( - StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(), - )); + return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string())); } if !versioned && !version_suspended { - return Err(std::io::Error::other( - StorageError::InvalidArgument( - bucket.to_string(), - object.to_string(), - format!("version-id specified {} but versioning is not enabled on {}", vid, bucket), - ) - .to_string(), + return Err(StorageError::InvalidArgument( + bucket.to_string(), + object.to_string(), + format!("version-id specified {vid} but versioning is not enabled on {bucket}"), )); } } @@ -4872,43 +4867,18 @@ pub async fn put_restore_opts( } meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/ - if let Some(type_) = &rreq.type_ - && type_.as_str() == RestoreRequestType::SELECT + // A SELECT restore must never reach the restore writer: the caller writes + // the retrieved bytes back to the source bucket/object, so building + // SELECT output options here produced a source overwrite carrying only + // the OutputLocation metadata instead of a write to `OutputLocation.S3` + // (backlog#1341). RestoreObject rejects SELECT at the API boundary; this + // is the fail-closed backstop for any other caller. + if rreq + .type_ + .as_ref() + .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT) { - let Some(s3) = select_restore_s3_location(rreq)? else { - return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests")); - }; - if let Some(user_metadata) = s3.user_metadata.as_ref() { - for metadata in user_metadata { - let name = metadata - .name - .as_deref() - .ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?; - let value = metadata.value.clone().unwrap_or_default(); - if strings_has_prefix_fold(name, "x-amz-meta") { - meta.insert(name.to_string(), value); - } else { - meta.insert(format!("x-amz-meta-{name}"), value); - } - } - } - if let Some(tags) = &s3.tagging { - meta.insert( - AMZ_OBJECT_TAGGING.to_string(), - serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()), - ); - } - if let Some(encryption) = &s3.encryption - && encryption.encryption_type.as_str() != "" - { - meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string()); - } - return Ok(ObjectOptions { - versioned: BucketVersioningSys::prefix_enabled(bucket, object).await, - version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await, - user_defined: meta, - ..Default::default() - }); + return Err(std::io::Error::other("SELECT restore requests are not supported")); } for (k, v) in oi.user_defined.iter() { meta.insert(k.to_string(), v.clone()); diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index 5360948e3..d5751d5c4 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -170,12 +170,12 @@ use s3s::dto::{ DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts, GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput, - PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm, - SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption, - ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, - WebsiteRedirectLocation, + PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus, + SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, + ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, + Timestamp, TimestampFormat, WebsiteRedirectLocation, }; -use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; +use s3s::header::X_AMZ_RESTORE; use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; diff --git a/rustfs/src/app/object/restore.rs b/rustfs/src/app/object/restore.rs index 56319265d..96201a670 100644 --- a/rustfs/src/app/object/restore.rs +++ b/rustfs/src/app/object/restore.rs @@ -165,18 +165,49 @@ impl DefaultObjectUsecase { validate_table_catalog_object_mutation(&bucket, &object).await?; - let rreq = rreq.ok_or_else(|| { - S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required") - })?; + // Typed S3 errors on every RestoreObject failure (backlog#2205): a + // `Custom` code serializes as a generic 500, which makes SDK clients + // retry client errors and conflicts alike. + let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?; + + // SELECT-type restore is not supported (backlog#1341). The restore + // path can only write the retrieved bytes back to the source key, so + // honouring a SELECT request overwrote the source object with + // SELECT-only metadata (dropping `x-amz-restore`, user metadata and + // tags on an unversioned bucket, or publishing a bogus latest version + // on a versioned one) while never writing anything to + // `OutputLocation.S3`. Reject before any guard, metadata write or + // fabricated `x-amz-restore-output-path` response header. + if rreq + .type_ + .as_ref() + .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT) + { + return Err(S3Error::with_message( + S3ErrorCode::NotImplemented, + "SELECT restore requests are not supported.", + )); + } let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + // Validate the request shape before taking any lock or reading the + // object: a malformed request or an illegal `Days` value is a client + // error, and the validator messages are static — they carry no + // backend or credential detail. + if let Err(e) = validate_restore_request(&rreq, store.clone()) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Restore object validation failed: {e}"), + )); + } + let version_id_str = version_id.clone().unwrap_or_default(); let mut opts = post_restore_opts(&version_id_str, &bucket, &object) .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?; + .map_err(ApiError::from)?; apply_bucket_generation_guard(&req, &bucket, &mut opts)?; // `apply_bucket_generation_guard` deliberately tolerates a missing guard // (only the S3 access layer installs one), so this must not hard-require @@ -192,11 +223,7 @@ impl DefaultObjectUsecase { } }; - // SELECT-type restores skip both the ongoing check and the metadata - // write below, so the accept guard would protect nothing for them — - // they keep the plain (read-locked) accept path. - let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT"); - let restore_operation_id = (!is_select).then(Uuid::new_v4); + let restore_operation_id = Some(Uuid::new_v4()); let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id { Some( store @@ -210,10 +237,10 @@ impl DefaultObjectUsecase { // Hold the restore-accept guard across the restore-status read, the // ongoing/already-restored decision, and the metadata write below, so - // two concurrent (non-SELECT) POST ?restore cannot both observe - // ongoing=false and both start a copy-back (backlog#1304). Reads and - // writes inside this scope run with no_lock; the guard is dropped - // before the copy-back is spawned so it never blocks readers. + // two concurrent POST ?restore cannot both observe ongoing=false and + // both start a copy-back (backlog#1304). Reads and writes inside this + // scope run with no_lock; the guard is dropped before the copy-back is + // spawned so it never blocks readers. // Contention on the accept guard (e.g. a concurrent accept or an // in-flight commit on the same object) is transient — answer 503 // SlowDown so SDK clients back off and retry instead of treating it @@ -222,9 +249,7 @@ impl DefaultObjectUsecase { if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id { return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into()); } - let mut accept_guard = if is_select { - None - } else { + let mut accept_guard = { let guard = store .acquire_restore_accept_guard(&bucket, &object) .await @@ -233,24 +258,17 @@ impl DefaultObjectUsecase { Some(guard) }; - let mut obj_info = store - .get_object_info(&bucket, &object, &opts) - .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?; + // A missing key or version must stay NoSuchKey / NoSuchVersion, and an + // authorization or storage failure must keep its own identity, so map + // the storage error instead of flattening it (backlog#2205). + let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; - // Check if object is in a transitioned state + // Restoring an object that was never transitioned is the S3 + // InvalidObjectState case, not an internal error. if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrInvalidTransitionedState".into()), - "restore object failed.", - )); - } - - // Validate restore request - if let Err(e) = validate_restore_request(&rreq, store.clone()) { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrValidRestoreObject".into()), - format!("Restore object validation failed: {}", e), + S3ErrorCode::InvalidObjectState, + "The operation is not valid for the object's storage class.", )); } @@ -260,7 +278,7 @@ impl DefaultObjectUsecase { // would create an ABBA cycle. If the probe succeeds, reacquire and // re-read the object before replacing the exact orphan generation. let mut superseded_worker_guard = None; - if obj_info.restore_ongoing && !is_select { + if obj_info.restore_ongoing { match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) { OngoingRestoreRecovery::ActiveOrUnsafe => { return Err(S3Error::with_message( @@ -293,13 +311,11 @@ impl DefaultObjectUsecase { .map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?, ); opts.no_lock = true; - obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| { - S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.") - })?; + obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrInvalidTransitionedState".into()), - "restore object failed.", + S3ErrorCode::InvalidObjectState, + "The operation is not valid for the object's storage class.", )); } if obj_info.restore_ongoing { @@ -327,11 +343,11 @@ impl DefaultObjectUsecase { remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID); remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK); - let mut header = HeaderMap::new(); - let event_object_info = obj_info.clone(); let obj_info_ = obj_info.clone(); - if !is_select { + // Scopes the accept-guarded metadata write: everything below runs + // inside the accept critical section, which is released right after. + { obj_info.metadata_only = true; metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string()); let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| { @@ -403,7 +419,7 @@ impl DefaultObjectUsecase { &restore_dst_opts, ) .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?; + .map_err(ApiError::from)?; rustfs_scanner::record_dirty_usage_bucket(&bucket); #[cfg(test)] maybe_pause_after_restore_status_commit(&bucket, &object).await; @@ -429,17 +445,6 @@ impl DefaultObjectUsecase { drop(accept_guard); drop(restore_bucket_lifecycle_guard); - // Handle output location for SELECT requests - if let Some(output_location) = &rreq.output_location - && let Some(s3) = &output_location.s3 - && !s3.bucket_name.is_empty() - { - let restore_object = Uuid::new_v4().to_string(); - if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() { - header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value); - } - } - // Spawn restoration task in the background. Pin the copy-back to the // version the accept resolved and flagged: with a versionless request // on a versioned bucket, a PUT landing between the accept and the @@ -499,7 +504,7 @@ impl DefaultObjectUsecase { restore_output_path: None, }; helper = helper.object(event_object_info).version_id(version_id_str); - let result = Ok(S3Response::with_headers(output, header)); + let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); result } @@ -629,6 +634,28 @@ mod tests { assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe); } + fn restore_request(days: Option) -> RestoreRequest { + RestoreRequest { + days, + description: None, + glacier_job_parameters: None, + output_location: None, + select_parameters: None, + tier: None, + type_: None, + } + } + + fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput { + RestoreObjectInput::builder() + .bucket(bucket.to_string()) + .key(key.to_string()) + .restore_request(Some(rreq)) + .build() + .expect("restore input should build") + } + + /// backlog#2205: a missing restore body is a client error, not a 500. #[tokio::test] async fn execute_restore_object_rejects_missing_restore_request() { let input = RestoreObjectInput::builder() @@ -641,10 +668,92 @@ mod tests { let usecase = DefaultObjectUsecase::without_context(); let err = usecase.execute_restore_object(req).await.unwrap_err(); - match err.code() { - S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"), - code => panic!("unexpected error code: {:?}", code), - } + assert_eq!(err.code(), &S3ErrorCode::MalformedXML); + } + + /// backlog#1341: a SELECT restore must be rejected outright — the restore + /// path can only write back to the source key, never to + /// `OutputLocation.S3`. Rejection happens before the store is resolved, so + /// an uninitialized usecase still answers NotImplemented rather than the + /// InternalError every request that gets past this point returns. + #[tokio::test] + async fn execute_restore_object_rejects_select_type() { + let mut rreq = restore_request(None); + rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT)); + + let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST); + let usecase = DefaultObjectUsecase::without_context(); + + let err = usecase.execute_restore_object(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + /// backlog#2205: every RestoreObject failure that reaches storage must + /// keep its typed S3 identity. Before this, a missing key, a malformed + /// version-id, an illegal `Days` and an object that was never transitioned + /// all collapsed into `Custom(...)` codes, which serialize as a retryable + /// HTTP 500. + #[tokio::test] + #[serial_test::serial] + async fn execute_restore_object_maps_failures_to_typed_s3_errors() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let context = crate::app::gating_test_env::shared_gating_ambient().await; + let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create restore test bucket"); + let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec()); + store + .put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default()) + .await + .expect("put untransitioned test object"); + + let usecase = DefaultObjectUsecase::with_context(Some(context)); + + // An illegal `Days` is a client error, rejected before any lock or + // object read. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "local-object", restore_request(Some(0))), + Method::POST, + )) + .await + .expect_err("days=0 must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + // A malformed version-id keeps InvalidArgument instead of being + // flattened inside `post_restore_opts`. + let mut input = restore_input(&bucket, "local-object", restore_request(Some(1))); + input.version_id = Some("not-a-uuid".to_string()); + let err = usecase + .execute_restore_object(build_request(input, Method::POST)) + .await + .expect_err("malformed version-id must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + + // A missing key stays NoSuchKey. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "missing-object", restore_request(Some(1))), + Method::POST, + )) + .await + .expect_err("missing key must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchKey); + + // Restoring an object that was never transitioned is the S3 + // InvalidObjectState case, not an internal error. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "local-object", restore_request(Some(1))), + Method::POST, + )) + .await + .expect_err("untransitioned object must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState); } #[tokio::test] diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index e90f9d9ae..19548aa66 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -368,7 +368,8 @@ pub(crate) mod bucket { version_id: &str, bucket: &str, object: &str, - ) -> Result { + ) -> Result + { crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts( version_id, bucket, object, ) From da28f8c8430834d6a3788b5e014ce343e99533bb Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 4 Sep 2026 20:10:16 +0800 Subject: [PATCH 14/19] chore(deps): refresh hotpath, mimalloc, and kafka (#7135) * chore(deps): refresh hotpath and mimalloc Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): bump rustfs kafka crates Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 --- Cargo.lock | 119 +++++++++++++++++---------------------------- Cargo.toml | 12 ++--- rustfs/src/main.rs | 8 ++- 3 files changed, 57 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2aba26999..e9e7d9fe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,9 +1328,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -1368,9 +1368,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.2" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -5103,9 +5103,9 @@ dependencies = [ [[package]] name = "hickory-net" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1" dependencies = [ "async-trait", "cfg-if", @@ -5127,9 +5127,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d" dependencies = [ "data-encoding", "idna", @@ -5147,9 +5147,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f" dependencies = [ "cfg-if", "futures-util", @@ -5226,9 +5226,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737" +checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592" dependencies = [ "arc-swap", "async-channel", @@ -5240,7 +5240,6 @@ dependencies = [ "futures-util", "hdrhistogram", "hotpath-macros", - "hotpath-meta", "http 1.5.0", "libc", "object 0.36.7", @@ -5260,30 +5259,15 @@ dependencies = [ [[package]] name = "hotpath-macros" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1" +checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] -[[package]] -name = "hotpath-macros-meta" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b" - -[[package]] -name = "hotpath-meta" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b" -dependencies = [ - "hotpath-macros-meta", -] - [[package]] name = "htmlescape" version = "0.3.1" @@ -5642,18 +5626,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "internal-russh-num-bigint" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" -dependencies = [ - "num-integer", - "num-traits", - "rand 0.10.2", - "rand_core 0.10.1", -] - [[package]] name = "io-uring" version = "0.7.14" @@ -5904,17 +5876,20 @@ dependencies = [ [[package]] name = "kafka-protocol" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2" +checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda" dependencies = [ "anyhow", "bytes", "crc", "crc32c", + "flate2", "indexmap 2.14.1", - "paste", + "lz4", + "snap", "uuid", + "zstd", ] [[package]] @@ -6923,6 +6898,8 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", + "rand 0.10.2", + "rand_core 0.10.1", ] [[package]] @@ -7515,9 +7492,9 @@ dependencies = [ [[package]] name = "pageant" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791" +checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce" dependencies = [ "base16ct 1.0.0", "byteorder", @@ -7606,12 +7583,6 @@ dependencies = [ "phc", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "path-absolutize" version = "4.0.1" @@ -9250,9 +9221,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.63.1" +version = "0.63.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031" +checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd" dependencies = [ "aes 0.9.3", "aws-lc-rs", @@ -9281,13 +9252,12 @@ dependencies = [ "hex-literal", "hmac 0.13.0", "inout 0.2.2", - "internal-russh-num-bigint", "keccak", "log", "md5", "ml-kem", "module-lattice", - "num-bigint 0.4.8", + "num-bigint 0.5.1", "p256 0.14.0", "p384 0.14.0", "p521", @@ -9978,11 +9948,11 @@ dependencies = [ [[package]] name = "rustfs-kafka" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96" +checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "fnv", "hmac 0.13.0", @@ -9992,22 +9962,22 @@ dependencies = [ "pbkdf2 0.13.0", "rand 0.10.2", "rustls", - "rustls-native-certs", "sha2 0.11.0", "socket2", "thiserror 2.0.20", "tracing", "twox-hash", + "uuid", "webpki-roots 1.0.9", ] [[package]] name = "rustfs-kafka-async" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630" +checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "hmac 0.13.0", "kafka-protocol", @@ -10016,11 +9986,11 @@ dependencies = [ "rand 0.10.2", "rustfs-kafka", "rustls", - "rustls-native-certs", "sha2 0.11.0", "tokio", "tokio-rustls", "tracing", + "uuid", "webpki-roots 1.0.9", ] @@ -10194,18 +10164,18 @@ dependencies = [ [[package]] name = "rustfs-mimalloc" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15" +checksum = "46a7b69356718defa4060de3059e609c9da6619c35924fefef55d902ccd94958" dependencies = [ "rustfs-mimalloc-sys", ] [[package]] name = "rustfs-mimalloc-sys" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692" +checksum = "1dd3ec9b7e7b9fed453acd8b4e32713ddc0e01e1aee31a3deec5b8025c0880c3" dependencies = [ "cc", ] @@ -12301,7 +12271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -12485,9 +12455,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -13652,13 +13622,14 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "wnaf" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" dependencies = [ "ff 0.14.0", "group 0.14.0", "hybrid-array", + "primefield", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 644e5b08a..05ddc1110 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,7 +165,7 @@ http-body = "1.1.0" http-body-util = "0.1.5" minlz = "1.2.3" reqwest = "0.13.4" -rustfs-kafka-async = { version = "1.2.0" } +rustfs-kafka-async = { version = "1.3.1" } socket2 = { version = "0.6.5" } tokio = { version = "1.53.1" } tokio-rustls = { default-features = false, version = "0.26.4" } @@ -244,8 +244,8 @@ aws-sdk-kms = { default-features = false, version = "1.117.0" } aws-sdk-s3 = { default-features = false, version = "1.144.0" } aws-sdk-sts = { default-features = false, version = "1.113.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } -aws-smithy-runtime-api = { version = "1.15.0" } -aws-smithy-types = { version = "1.6.2" } +aws-smithy-runtime-api = { version = "1.16.0" } +aws-smithy-types = { version = "1.6.3" } base64-simd = "0.8.0" brotli = "9.0.0" clap = { version = "4.6.6" } @@ -359,15 +359,15 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "11.0.0" } rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.63.1" } +russh = { version = "0.63.2" } russh-sftp = "2.4.0" # WebDAV dav-server = "0.11.0" # Performance Analysis and Memory Profiling -rustfs-mimalloc = { version = "0.5.2" } -hotpath = { version = "0.24.0", default-features = false } +rustfs-mimalloc = { version = "0.5.3" } +hotpath = { version = "0.25.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 51aef9ef5..c2f3325c9 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -14,6 +14,8 @@ #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] use std::alloc::{GlobalAlloc, Layout}; +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] +use std::ptr::NonNull; #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] #[derive(Default)] @@ -35,8 +37,10 @@ unsafe impl GlobalAlloc for MiMallocAllocator { } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - // SAFETY: ptr came from this allocator and layout.size() is the original allocation size. - unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) } + // SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's + // dealloc contract, and layout.size() is the original allocation size. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { From 20e4fd7de64a8d7dc5f905625bc2eba1bb5340f0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 20:11:49 +0800 Subject: [PATCH 15/19] feat(ecstore): add the sealed remote credential seam (#7137) * feat(ecstore): add the sealed remote credential seam Replication targets, remote tiers and on-demand migration sources will all seal their stored secrets through one envelope rather than three (rustfs/backlog#2168, design in docs/architecture/remote-credential-sealing-adr.md). Adds the versioned envelope, the seal scope that binds a ciphertext to the store, owner and field it belongs to, the sealer registration point, and the fail-closed error type. ECStore still has no rustfs-kms dependency: the binary installs a sealer the way it installs the event dispatch hook. Nothing is wired to a consumer yet, so no stored format changes. * docs(ecstore): name the event dispatch hook by module, not by symbol The architecture guard keeps EVENT_DISPATCH_HOOK references inside the event-notification owner module; the module doc cited the symbol only as an example of the hook shape, so cite its file instead. --- crates/ecstore/src/api/mod.rs | 7 + crates/ecstore/src/bucket/mod.rs | 1 + .../ecstore/src/bucket/sealed_credentials.rs | 351 ++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100644 crates/ecstore/src/bucket/sealed_credentials.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index aa04369a6..7ce3523fd 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -252,6 +252,13 @@ pub mod bucket { }; } + pub mod sealed_credentials { + pub use crate::bucket::sealed_credentials::{ + CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError, + SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret, + }; + } + pub mod replication { pub use crate::bucket::replication::replication_pool::{ DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary, diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index 48fce4e02..e93419cf2 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -31,6 +31,7 @@ pub mod policy_sys; pub mod quota; pub mod remote_s3_client; pub mod replication; +pub mod sealed_credentials; pub mod tagging; pub mod target; pub mod utils; diff --git a/crates/ecstore/src/bucket/sealed_credentials.rs b/crates/ecstore/src/bucket/sealed_credentials.rs new file mode 100644 index 000000000..392e40dd1 --- /dev/null +++ b/crates/ecstore/src/bucket/sealed_credentials.rs @@ -0,0 +1,351 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Sealed remote credentials shared by the three stores that persist one +//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote +//! tiers (`tier-config.bin`) and on-demand migration sources +//! (`on-demand-migration.json`). +//! +//! The design record is `docs/architecture/remote-credential-sealing-adr.md`. +//! What this module owns: the versioned envelope, the encryption context that +//! binds a ciphertext to the record owning it, the sealer registration point, +//! and the fail-closed error type. What it deliberately does not own: any KMS +//! call (ECStore does not depend on `rustfs-kms`; the binary installs a +//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event +//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and +//! any decision about which stored field a consumer writes. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, OnceLock}; + +/// Envelope format this build writes. A reader accepts only versions it +/// knows; an unknown version is a typed error, never a fallback. +pub const SEALED_CREDENTIAL_VERSION: u8 = 1; + +/// Which store a sealed value belongs to. Part of the encryption context, so +/// a ciphertext cannot be replayed into a different store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SealedCredentialStore { + /// `bucket-targets.json` (replication and other bucket targets). + BucketTargets, + /// `tier-config.bin` (remote tiers). + TierConfig, + /// `on-demand-migration.json` (migration sources). + OnDemandMigration, +} + +impl SealedCredentialStore { + pub fn as_str(self) -> &'static str { + match self { + SealedCredentialStore::BucketTargets => "bucket-targets", + SealedCredentialStore::TierConfig => "tier-config", + SealedCredentialStore::OnDemandMigration => "on-demand-migration", + } + } +} + +/// Identity of the record a secret belongs to: the store, its owner (bucket +/// name, tier name, or target ARN) and the field name. Rendered into the KMS +/// encryption context so a ciphertext moved between buckets, tiers or fields +/// fails to decrypt instead of silently authorizing a different remote. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SealScope { + pub store: SealedCredentialStore, + pub owner: String, + pub field: &'static str, +} + +impl SealScope { + pub fn new(store: SealedCredentialStore, owner: impl Into, field: &'static str) -> Self { + Self { + store, + owner: owner.into(), + field, + } + } + + /// The encryption context handed to the sealer. Keys are stable: they are + /// part of the on-disk contract, because a ciphertext only decrypts under + /// the same context. + pub fn encryption_context(&self) -> HashMap { + HashMap::from([ + ("rustfs:store".to_string(), self.store.as_str().to_string()), + ("rustfs:owner".to_string(), self.owner.clone()), + ("rustfs:field".to_string(), self.field.to_string()), + ]) + } +} + +/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value +/// is not a secret, but it is noise in a log line and an operator reading one +/// should see the key it is wrapped under, not the bytes. +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SealedCredential { + /// Envelope version; see [`SEALED_CREDENTIAL_VERSION`]. + pub v: u8, + /// KMS master key id the data key is wrapped under. + pub key_id: String, + /// Master key version, when the backend reports one. Carried so the KMS + /// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell + /// stale envelopes apart; nothing here rotates on its own. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key_version: Option, + /// Algorithm label reported by the sealer, for forensics and migration. + pub alg: String, + /// Ciphertext blob as produced by the sealer, base64 (standard, padded) + /// in the JSON stores and raw inside the tier msgpack payload. + pub ct: String, +} + +impl fmt::Debug for SealedCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SealedCredential") + .field("v", &self.v) + .field("key_id", &self.key_id) + .field("key_version", &self.key_version) + .field("alg", &self.alg) + .field("ct", &format_args!("<{} bytes sealed>", self.ct.len())) + .finish() + } +} + +impl SealedCredential { + /// Rejects an envelope this build cannot read. Called before every + /// unseal so an unknown version fails here rather than inside a backend. + pub fn check_version(&self) -> Result<(), SealedCredentialError> { + if self.v == SEALED_CREDENTIAL_VERSION { + Ok(()) + } else { + Err(SealedCredentialError::UnsupportedVersion(self.v)) + } + } +} + +/// Why a seal or unseal did not produce a usable value. Every variant is +/// terminal for the record that carried it: a caller reports the remote as +/// unusable, and never substitutes a default or empty credential. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum SealedCredentialError { + /// No sealer is installed: KMS is not configured, or the process has not + /// finished startup. Reading a sealed record is impossible here. + #[error("no credential sealer is installed")] + NoSealer, + /// The stored envelope is from a newer (or otherwise unknown) format. + #[error("unsupported sealed credential envelope version {0}")] + UnsupportedVersion(u8), + /// The stored bytes are not a well-formed envelope. + #[error("malformed sealed credential: {0}")] + Malformed(String), + /// The sealer refused: wrong encryption context, missing key, revoked + /// access, or a failed authentication tag. + #[error("sealed credential could not be unsealed: {0}")] + Kms(String), +} + +/// The KMS-backed half, installed by the binary at startup. +#[async_trait] +pub trait CredentialSealer: Send + Sync + 'static { + /// Wraps `plaintext` under the scope's encryption context. + async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result; + + /// Unwraps a stored envelope. Must fail when the envelope was sealed + /// under a different scope. + async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result; +} + +static CREDENTIAL_SEALER: OnceLock> = OnceLock::new(); + +/// Installs the process-wide sealer. Returns `false` when one is already +/// installed, matching the other ECStore hooks. +pub fn install_credential_sealer(sealer: Arc) -> bool { + CREDENTIAL_SEALER.set(sealer).is_ok() +} + +/// The installed sealer, or `None` when KMS is not wired. Callers that only +/// need to know whether sealing is possible use this; callers that must have +/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error. +pub fn credential_sealer() -> Option> { + CREDENTIAL_SEALER.get().cloned() +} + +/// Seals one secret field. Fails closed: without a sealer the caller must +/// reject the write rather than persist the secret in clear text after the +/// operator asked for sealing. +pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result { + let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?; + sealer.seal(plaintext, scope).await +} + +/// Unseals one secret field, rejecting an unknown envelope version first. +pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result { + sealed.check_version()?; + let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?; + sealer.unseal(sealed, scope).await +} + +#[cfg(test)] +mod tests { + use super::*; + use parking_lot::Mutex; + + /// Stands in for the KMS-backed sealer: records the context it was called + /// with, and refuses a ciphertext presented under a different one. + #[derive(Default)] + struct FakeSealer { + sealed_contexts: Mutex>>, + } + + #[async_trait] + impl CredentialSealer for FakeSealer { + async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result { + let context = scope.encryption_context(); + self.sealed_contexts.lock().push(context.clone()); + let mut bound = serde_json::to_string(&context).expect("context serializes"); + bound.push('|'); + bound.push_str(plaintext); + Ok(SealedCredential { + v: SEALED_CREDENTIAL_VERSION, + key_id: "key-1".to_string(), + key_version: Some("3".to_string()), + alg: "AES-256-GCM".to_string(), + ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()), + }) + } + + async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result { + let raw = base64_simd::STANDARD + .decode_to_vec(sealed.ct.as_bytes()) + .map_err(|err| SealedCredentialError::Malformed(err.to_string()))?; + let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?; + let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes"); + bound + .strip_prefix(&expected) + .and_then(|rest| rest.strip_prefix('|')) + .map(str::to_string) + .ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string())) + } + } + + fn scope(owner: &str) -> SealScope { + SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key") + } + + #[tokio::test] + async fn seal_round_trips_and_binds_the_scope() { + let sealer = Arc::new(FakeSealer::default()); + let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal"); + assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION); + assert_eq!(sealed.key_version.as_deref(), Some("3")); + assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret"); + + // The same ciphertext under another bucket must not unseal. + let err = sealer + .unseal(&sealed, &scope("other-bucket")) + .await + .expect_err("a ciphertext must not move between owners"); + assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}"); + + // Nor under another field of the same record. + let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token"); + let err = sealer + .unseal(&sealed, &other_field) + .await + .expect_err("a ciphertext must not move between fields"); + assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}"); + + let contexts = sealer.sealed_contexts.lock(); + assert_eq!(contexts.len(), 1); + assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration"); + assert_eq!(contexts[0]["rustfs:owner"], "photos"); + assert_eq!(contexts[0]["rustfs:field"], "secret_key"); + } + + #[tokio::test] + async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() { + let sealed = SealedCredential { + v: SEALED_CREDENTIAL_VERSION + 1, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + assert_eq!( + sealed.check_version().expect_err("a newer envelope must not be read"), + SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1) + ); + // The global helper reports the version, not "no sealer", even in a + // process where none is installed. + assert_eq!( + unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"), + SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1) + ); + } + + #[tokio::test] + async fn without_a_sealer_both_directions_fail_closed() { + // This test binary installs no sealer, so the global helpers must + // report NoSealer rather than fall back to clear text. + assert!(credential_sealer().is_none(), "no sealer is installed in unit tests"); + assert_eq!( + seal_secret("super-secret", &scope("photos")).await.expect_err("seal"), + SealedCredentialError::NoSealer + ); + let sealed = SealedCredential { + v: SEALED_CREDENTIAL_VERSION, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + assert_eq!( + unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"), + SealedCredentialError::NoSealer + ); + } + + #[test] + fn debug_and_serde_keep_the_on_disk_shape_stable() { + let sealed = SealedCredential { + v: 1, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + // key_version is omitted when absent, so an envelope from a backend + // without version history stays compact. + assert_eq!( + serde_json::to_string(&sealed).expect("serialize"), + r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"# + ); + let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#) + .expect("an envelope without key_version parses"); + assert_eq!(parsed, sealed); + + let rendered = format!("{sealed:?}"); + assert!(rendered.contains("key-1"), "{rendered}"); + assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}"); + } + + #[test] + fn a_malformed_envelope_is_a_typed_error() { + let err = serde_json::from_str::(r#"{"v":1,"key_id":"key-1"}"#) + .map_err(|err| SealedCredentialError::Malformed(err.to_string())) + .expect_err("a truncated envelope must not parse"); + assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}"); + } +} From 3da3bcd9eed2869e2d1db5e8b3a09d9c8de982cb Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 20:24:02 +0800 Subject: [PATCH 16/19] fix(tier): preserve credentials during backend validation (#7127) * fix(tier): preserve credentials during backend validation * test(tier): make clone redaction assertion clippy-clean --- crates/ecstore/src/services/tier/tier.rs | 2146 +++++++++++++++-- .../ecstore/src/services/tier/tier_admin.rs | 196 +- .../ecstore/src/services/tier/tier_config.rs | 418 +++- .../ecstore/src/services/tier/warm_backend.rs | 601 ++++- .../src/services/tier/warm_backend_aliyun.rs | 6 +- .../src/services/tier/warm_backend_azure.rs | 110 +- .../src/services/tier/warm_backend_gcs.rs | 505 +++- .../services/tier/warm_backend_huaweicloud.rs | 6 +- .../src/services/tier/warm_backend_s3.rs | 310 ++- .../src/services/tier/warm_backend_tencent.rs | 6 +- .../src/services/tier/warm_backend_wasabi.rs | 6 +- crates/s3-client/src/api_error_response.rs | 23 +- crates/s3-client/src/api_get_object.rs | 249 +- rustfs/src/admin/handlers/tier.rs | 190 +- 14 files changed, 4303 insertions(+), 469 deletions(-) diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 377f6ba06..6f48db424 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -50,9 +50,12 @@ use tracing::{debug, error, info, warn}; use crate::error::{Error, Result, StorageError, stable_io_error}; use crate::services::tier::{ tier_admin::TierCreds, - tier_config::{TierConfig, TierType, TierWasabi}, + tier_config::{TIER_CREDENTIAL_REDACTED, TierConfig, TierType, TierWasabi}, tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME}, - warm_backend::{TransitionCandidateProbe, WarmBackend, check_warm_backend, new_warm_backend}, + warm_backend::{ + TransitionCandidateProbe, WARM_BACKEND_PROBE_TIMEOUT, WarmBackend, check_warm_backend, check_warm_backend_until, + new_warm_backend, + }, }; use crate::storage_api_contracts::{ bucket::BucketOperations, @@ -136,6 +139,14 @@ struct TierDriverBuildBarrier { #[cfg(test)] static TIER_DRIVER_BUILD_BARRIER: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); +#[cfg(test)] +type TierDriverTestFactory = Arc std::result::Result + Send + Sync + 'static>; + +#[cfg(test)] +tokio::task_local! { + static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory; +} + #[cfg(test)] struct TierDriverBuildBarrierGuard; @@ -157,12 +168,18 @@ fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc std::result::Result { +fn tier_validation_timeout(message: impl Into) -> AdminError { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = message.into(); + err +} + +#[cfg(test)] +async fn wait_for_tier_driver_build_barrier(tier_name: &str) { #[cfg(test)] let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() }; - #[cfg(test)] if let Some(barrier) = test_barrier - && barrier.tier_name == tier.name + && barrier.tier_name == tier_name { barrier.arrived.notify_one(); barrier @@ -172,7 +189,52 @@ async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Resu .expect("tier driver build test barrier should stay open") .forget(); } - new_warm_backend(tier, probe).await +} + +async fn construct_warm_backend(tier: &TierConfig) -> std::result::Result { + #[cfg(test)] + if let Ok(result) = TIER_DRIVER_TEST_FACTORY.try_with(|factory| factory(tier)) { + return result; + } + new_warm_backend(tier, false).await +} + +async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result { + build_warm_backend_with_deadline(tier, probe, None).await +} + +async fn build_warm_backend_with_deadline( + tier: &TierConfig, + probe: bool, + deadline: Option, +) -> std::result::Result { + #[cfg(test)] + { + let wait = wait_for_tier_driver_build_barrier(&tier.name); + if let Some(deadline) = deadline { + timeout_at(deadline, wait) + .await + .map_err(|_| tier_validation_timeout("Timed out preparing the remote tier backend"))?; + } else { + wait.await; + } + } + + let driver = if let Some(deadline) = deadline { + timeout_at(deadline, construct_warm_backend(tier)) + .await + .map_err(|_| tier_validation_timeout("Timed out preparing the remote tier backend"))?? + } else { + construct_warm_backend(tier).await? + }; + if probe { + if let Some(deadline) = deadline { + check_warm_backend_until(Some(&driver), deadline).await?; + } else { + check_warm_backend(Some(&driver)).await?; + } + } + Ok(driver) } const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json"; @@ -219,6 +281,189 @@ lazy_static! { }; } +fn tier_invalid_config(message: impl Into) -> AdminError { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = message.into(); + err +} + +fn normalize_add_tier_name_fields( + canonical_name: &mut String, + nested_name: &mut String, + provider: &str, +) -> std::result::Result<(), AdminError> { + if !canonical_name.is_empty() && !nested_name.is_empty() && canonical_name.as_str() != nested_name.as_str() { + return Err(tier_invalid_config(format!( + "TierConfig.Name conflicts with the legacy {provider}.name field" + ))); + } + let resolved_name = if canonical_name.is_empty() { + nested_name.clone() + } else { + canonical_name.clone() + }; + if resolved_name.is_empty() { + return Err(tier_invalid_config("Remote tier name is empty")); + } + canonical_name.clone_from(&resolved_name); + nested_name.clone_from(&resolved_name); + Ok(()) +} + +fn normalize_s3_gcs_add_tier_name(config: &mut TierConfig) -> std::result::Result<(), AdminError> { + match config.tier_type { + TierType::S3 => { + if let Some(s3) = config.s3.as_mut() { + normalize_add_tier_name_fields(&mut config.name, &mut s3.name, "S3")?; + } + } + TierType::GCS => { + if let Some(gcs) = config.gcs.as_mut() { + normalize_add_tier_name_fields(&mut config.name, &mut gcs.name, "GCS")?; + } + } + _ => {} + } + Ok(()) +} + +fn credential_is_redacted(value: &str) -> bool { + value.trim() == TIER_CREDENTIAL_REDACTED +} + +fn validate_static_tier_credentials(access_key: &str, secret_key: &str) -> std::result::Result<(), AdminError> { + if access_key.is_empty() || secret_key.is_empty() || credential_is_redacted(access_key) || credential_is_redacted(secret_key) + { + return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + } + Ok(()) +} + +fn tier_creds_request_aws_role(creds: &TierCreds) -> bool { + creds.aws_role || !creds.aws_role_web_identity_token_file.is_empty() || !creds.aws_role_arn.is_empty() +} + +fn s3_config_requests_aws_role(config: &crate::services::tier::tier_config::TierS3) -> bool { + config.aws_role + || !config.aws_role_web_identity_token_file.is_empty() + || !config.aws_role_arn.is_empty() + || !config.aws_role_session_name.is_empty() + || config.aws_role_duration_seconds != 0 +} + +fn reject_unsupported_aws_role() -> AdminError { + tier_invalid_config("AWS role and web identity credentials are not supported for remote tiers") +} + +fn validate_gcs_credentials_json(credentials: &str) -> std::result::Result<(), AdminError> { + if credentials.is_empty() || credential_is_redacted(credentials) { + return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + } + let value: serde_json::Value = serde_json::from_str(credentials) + .map_err(|_| tier_invalid_config("GCS credentials must be valid service account JSON"))?; + if value.get("type").and_then(serde_json::Value::as_str) != Some("service_account") { + return Err(tier_invalid_config("GCS credentials must describe a service account")); + } + Ok(()) +} + +fn validate_azure_supported_options( + azure: &crate::services::tier::tier_config::TierAzure, +) -> std::result::Result<(), AdminError> { + let sp_auth_set = + !azure.sp_auth.tenant_id.is_empty() || !azure.sp_auth.client_id.is_empty() || !azure.sp_auth.client_secret.is_empty(); + if !azure.storage_class.is_empty() || sp_auth_set { + return Err(tier_invalid_config( + "Azure remote tiers do not support storageClass or spAuth yet; leave both unset", + )); + } + Ok(()) +} + +fn validate_tier_config_credentials(config: &TierConfig) -> std::result::Result<(), AdminError> { + match config.tier_type { + TierType::S3 => { + if let Some(s3) = config.s3.as_ref() { + if s3_config_requests_aws_role(s3) { + return Err(reject_unsupported_aws_role()); + } + validate_static_tier_credentials(&s3.access_key, &s3.secret_key)?; + } + } + TierType::Wasabi => { + if let Some(backend) = config.wasabi.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::RustFS => { + if let Some(backend) = config.rustfs.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::MinIO => { + if let Some(backend) = config.minio.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Aliyun => { + if let Some(backend) = config.aliyun.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Tencent => { + if let Some(backend) = config.tencent.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Huaweicloud => { + if let Some(backend) = config.huaweicloud.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Azure => { + if let Some(backend) = config.azure.as_ref() { + validate_azure_supported_options(backend)?; + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::GCS => { + if let Some(gcs) = config.gcs.as_ref() { + validate_gcs_credentials_json(&gcs.creds)?; + } + } + TierType::R2 => { + if let Some(backend) = config.r2.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Unsupported => {} + } + Ok(()) +} + +fn merge_static_tier_credentials( + access_key: &mut String, + secret_key: &mut String, + creds: &TierCreds, +) -> std::result::Result<(), AdminError> { + if tier_creds_request_aws_role(creds) { + return Err(reject_unsupported_aws_role()); + } + if !creds.creds_json.is_empty() { + return Err(tier_invalid_config("GCS credentials cannot be used with this remote tier type")); + } + match (creds.access_key.is_empty(), creds.secret_key.is_empty()) { + (true, true) => {} + (false, false) => { + validate_static_tier_credentials(&creds.access_key, &creds.secret_key)?; + access_key.clone_from(&creds.access_key); + secret_key.clone_from(&creds.secret_key); + } + _ => return Err(ERR_TIER_MISSING_CREDENTIALS.clone()), + } + validate_static_tier_credentials(access_key, secret_key) +} + #[derive(Serialize, Deserialize)] pub struct TierConfigMgr { #[serde(skip)] @@ -323,7 +568,11 @@ struct PreparedTierDriver { impl TierPublishTransition { async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> { - let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT; + self.wait_for_active_leases_until(Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT) + .await + } + + async fn wait_for_active_leases_until(&self, deadline: Instant) -> std::result::Result<(), AdminError> { for generation in self.revoked.values() { if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); @@ -542,6 +791,18 @@ enum TierCandidateMutation { } impl TierCandidateMutation { + fn add(mut config: TierConfig, force: bool) -> std::result::Result { + normalize_s3_gcs_add_tier_name(&mut config)?; + Ok(Self::Add(config, force)) + } + + fn normalize_add_tier_name(&mut self) -> std::result::Result<(), AdminError> { + if let Self::Add(config, _) = self { + normalize_s3_gcs_add_tier_name(config)?; + } + Ok(()) + } + fn intent_kind(&self) -> TierMutationIntentKind { match self { Self::Add(_, _) => TierMutationIntentKind::Add, @@ -605,15 +866,19 @@ impl TierCandidateMutation { ) } - async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result, AdminError> { + async fn apply( + self, + candidate: &mut TierConfigMgr, + deadline: Option, + ) -> std::result::Result, AdminError> { match self { Self::Add(config, force) => { let tier_name = config.name.clone(); - candidate.add(config, force).await?; + candidate.add_with_deadline(config, force, deadline).await?; Ok(Some(tier_name)) } Self::Edit(tier_name, credentials) => { - candidate.edit(&tier_name, credentials).await?; + candidate.edit_with_deadline(&tier_name, credentials, deadline).await?; Ok(Some(tier_name)) } Self::Remove(tier_name, force) => { @@ -802,7 +1067,7 @@ where { let targets = affected_targets .iter() - .filter(|target| target.old_backend_identity.is_some()) + .filter(|target| target.old_backend_identity.is_some() && target.old_backend_identity != target.new_backend_identity) .cloned() .collect::>(); if targets.is_empty() { @@ -1572,7 +1837,12 @@ async fn apply_tier_candidate_mutation( candidate: &mut TierConfigMgr, deadline: Instant, ) -> std::result::Result, AdminError> { - match timeout_at(deadline, mutation.apply(candidate)).await { + if matches!(&mutation, TierCandidateMutation::Add(_, _) | TierCandidateMutation::Edit(_, _)) { + // Add/Edit validation performs its own deadline-aware cleanup. Do not + // wrap it in an outer timeout that would cancel an uncertain probe. + return mutation.apply(candidate, Some(deadline)).await; + } + match timeout_at(deadline, mutation.apply(candidate, None)).await { Ok(result) => result, Err(_) => { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); @@ -2157,6 +2427,22 @@ struct ExternalTierCompatible { region: String, } +fn decode_external_gcs_credentials(credentials: &str) -> io::Result { + if serde_json::from_str::(credentials).is_ok() { + return Ok(credentials.to_string()); + } + // MinIO config and madmin AddTier payloads carry URL-safe-base64 credentials, + // while the existing RustFS v2 disk format stores raw JSON for rolling upgrades. + let decoded = base64_simd::STANDARD + .decode_to_vec(credentials.as_bytes()) + .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(credentials.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(credentials.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(credentials.as_bytes())) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "tier config contains invalid GCS credentials encoding"))?; + String::from_utf8(decoded) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "tier config contains non-UTF-8 GCS credentials JSON")) +} + fn tier_config_path(file: &str) -> String { format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file) } @@ -2577,7 +2863,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul cfg.gcs = Some(crate::services::tier::tier_config::TierGCS { name: cfg.name.clone(), endpoint: gcs.endpoint.clone(), - creds: gcs.creds.clone(), + creds: decode_external_gcs_credentials(&gcs.creds)?, bucket: gcs.bucket.clone(), prefix: gcs.prefix.clone(), region: gcs.region.clone(), @@ -2765,7 +3051,17 @@ impl TierConfigMgr { (TierType::Unsupported, false) } - pub async fn add(&mut self, mut tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + pub async fn add(&mut self, tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + self.add_with_deadline(tier_config, force, None).await + } + + async fn add_with_deadline( + &mut self, + mut tier_config: TierConfig, + force: bool, + deadline: Option, + ) -> std::result::Result<(), AdminError> { + normalize_s3_gcs_add_tier_name(&mut tier_config)?; self.ensure_generation_is_idle(&tier_config.name)?; let tier_name = tier_config.name.clone(); if tier_name != tier_name.to_uppercase() { @@ -2797,28 +3093,28 @@ impl TierConfigMgr { })?; } - // The Azure warm backend goes through the same S3-compatible TransitionClient as every - // other provider (backlog#2055): it has no Azure Blob-native client and no Azure AD - // dependency, so `storage_class` and `sp_auth` cannot be honored today even though the - // config type carries them. Reject them explicitly here instead of silently accepting - // and then dropping them at the WarmBackendAzure construction boundary. - if matches!(&tier_config.tier_type, TierType::Azure) - && let Some(azure) = tier_config.azure.as_ref() + if matches!(&tier_config.tier_type, TierType::GCS) + && let Some(gcs) = tier_config.gcs.as_mut() { - let sp_auth_set = !azure.sp_auth.tenant_id.is_empty() - || !azure.sp_auth.client_id.is_empty() - || !azure.sp_auth.client_secret.is_empty(); - if !azure.storage_class.is_empty() || sp_auth_set { - let mut err = ERR_TIER_INVALID_CONFIG.clone(); - err.message = "Azure remote tiers do not support storageClass or spAuth yet; leave both unset".to_string(); - return Err(err); - } + gcs.creds = decode_external_gcs_credentials(&gcs.creds).map_err(|source| tier_invalid_config(source.to_string()))?; } - let d = new_warm_backend(&tier_config, true).await?; + validate_tier_config_credentials(&tier_config)?; + let d = match deadline { + Some(deadline) => build_warm_backend_with_deadline(&tier_config, true, Some(deadline)).await?, + None => build_warm_backend(&tier_config, true).await?, + }; if !force { - let in_use = d.in_use().await; + let in_use = match deadline { + Some(deadline) => timeout_at(deadline, d.in_use()).await, + None => tokio::time::timeout(WARM_BACKEND_PROBE_TIMEOUT, d.in_use()).await, + } + .map_err(|_| { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out checking whether the remote tier is in use".to_string(); + err + })?; match in_use { Ok(b) => { if b { @@ -2910,7 +3206,7 @@ impl TierConfigMgr { pub fn list_tiers(&self) -> Vec { let mut tier_cfgs = Vec::::new(); for (_, tier) in self.tiers.iter() { - let tier = tier.clone(); + let tier = tier.redacted(); tier_cfgs.push(tier); } tier_cfgs @@ -2919,120 +3215,108 @@ impl TierConfigMgr { pub fn get(&self, tier_name: &str) -> Option { for (tier_name2, tier) in self.tiers.iter() { if tier_name == tier_name2 { - return Some(tier.clone()); + return Some(tier.redacted()); } } None } pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> { + self.edit_with_deadline(tier_name, creds, None).await + } + + async fn edit_with_deadline( + &mut self, + tier_name: &str, + creds: TierCreds, + deadline: Option, + ) -> std::result::Result<(), AdminError> { self.ensure_generation_is_idle(tier_name)?; let (tier_type, exists) = self.is_tier_name_in_use(tier_name); if !exists { return Err(ERR_TIER_NOT_FOUND.clone()); } + if !creds.azure_service_principal.is_empty() { + return Err(tier_invalid_config( + "Azure service principal credentials are not supported for remote tier edits", + )); + } - let mut tier_config = self.tiers[tier_name].clone(); + let mut tier_config = self.tiers[tier_name].clone_with_credentials(); match tier_type { TierType::S3 => { if let Some(s3) = tier_config.s3.as_mut() { - if creds.aws_role { - s3.aws_role = true - } - if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" { - s3.aws_role_arn = creds.aws_role_arn; - s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file; - } - if creds.access_key != "" && creds.secret_key != "" { - s3.access_key = creds.access_key; - s3.secret_key = creds.secret_key; - } + merge_static_tier_credentials(&mut s3.access_key, &mut s3.secret_key, &creds)?; } } TierType::Wasabi => { if let Some(wasabi) = tier_config.wasabi.as_mut() { - if creds.access_key.is_empty() || creds.secret_key.is_empty() { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - wasabi.access_key = creds.access_key; - wasabi.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut wasabi.access_key, &mut wasabi.secret_key, &creds)?; } } TierType::RustFS => { if let Some(rustfs) = tier_config.rustfs.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - rustfs.access_key = creds.access_key; - rustfs.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut rustfs.access_key, &mut rustfs.secret_key, &creds)?; } } TierType::MinIO => { if let Some(compatible_backend) = tier_config.minio.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - compatible_backend.access_key = creds.access_key; - compatible_backend.secret_key = creds.secret_key; + merge_static_tier_credentials( + &mut compatible_backend.access_key, + &mut compatible_backend.secret_key, + &creds, + )?; } } TierType::Aliyun => { if let Some(aliyun) = tier_config.aliyun.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - aliyun.access_key = creds.access_key; - aliyun.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut aliyun.access_key, &mut aliyun.secret_key, &creds)?; } } TierType::Tencent => { if let Some(tencent) = tier_config.tencent.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - tencent.access_key = creds.access_key; - tencent.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut tencent.access_key, &mut tencent.secret_key, &creds)?; } } TierType::Huaweicloud => { if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - huaweicloud.access_key = creds.access_key; - huaweicloud.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut huaweicloud.access_key, &mut huaweicloud.secret_key, &creds)?; } } TierType::Azure => { if let Some(azure) = tier_config.azure.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - azure.access_key = creds.access_key; - azure.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut azure.access_key, &mut azure.secret_key, &creds)?; } } TierType::GCS => { if let Some(gcs) = tier_config.gcs.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + if tier_creds_request_aws_role(&creds) { + return Err(reject_unsupported_aws_role()); + } + if !creds.access_key.is_empty() || !creds.secret_key.is_empty() { + return Err(tier_invalid_config("Static access and secret keys cannot be used with a GCS tier")); + } + if !creds.creds_json.is_empty() { + let credentials = std::str::from_utf8(&creds.creds_json) + .map_err(|_| tier_invalid_config("GCS credentials must be UTF-8 service account JSON"))?; + validate_gcs_credentials_json(credentials)?; + gcs.creds = credentials.to_string(); } - gcs.creds = creds.access_key; //creds.creds_json } } TierType::R2 => { if let Some(r2) = tier_config.r2.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - r2.access_key = creds.access_key; - r2.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut r2.access_key, &mut r2.secret_key, &creds)?; } } _ => (), } - let d = new_warm_backend(&tier_config, true).await?; + validate_tier_config_credentials(&tier_config)?; + let d = match deadline { + Some(deadline) => build_warm_backend_with_deadline(&tier_config, true, Some(deadline)).await?, + None => build_warm_backend(&tier_config, true).await?, + }; self.revoke_driver(tier_name); self.tiers.insert(tier_name.to_string(), tier_config); self.replace_driver(tier_name, d)?; @@ -3266,11 +3550,12 @@ impl TierConfigMgr { async fn update_candidate_with_config_lock( handle: &Arc>, api: Arc, - mutation: TierCandidateMutation, + mut mutation: TierCandidateMutation, ) -> std::result::Result<(), TierConfigUpdateError> where S: TierReferenceProofStore + NamespaceLocking + 'static, { + mutation.normalize_add_tier_name().map_err(TierConfigUpdateError::Mutation)?; let config_lock = Self::acquire_tier_config_write_lock(api.clone()).await?; Self::reject_pending_mutation_recovery_before_update(handle, api.clone()).await?; let update = Self::admin_update_lock(handle).await; @@ -3452,7 +3737,7 @@ impl TierConfigMgr { let exact_get_delete = tier_exact_get_delete(config); Some(PreparedTierDriver { tier_name: tier_name.to_string(), - tier_config: config.clone(), + tier_config: config.clone_with_credentials(), config_fingerprint, backend_identity, exact_get_delete, @@ -3493,7 +3778,7 @@ impl TierConfigMgr { })?; let entry = Arc::new(TierDriverGeneration { tier_name: Arc::from(prepared.tier_name.as_str()), - tier_config: prepared.tier_config.clone(), + tier_config: prepared.tier_config.clone_with_credentials(), generation, config_fingerprint: prepared.config_fingerprint, backend_identity: prepared.backend_identity, @@ -3596,6 +3881,8 @@ impl TierConfigMgr { let handle = handle.clone(); #[cfg(test)] let test_peers = TIER_MUTATION_TEST_PEERS.try_with(|peers| peers.clone()).ok(); + #[cfg(test)] + let test_driver_factory = TIER_DRIVER_TEST_FACTORY.try_with(|factory| factory.clone()).ok(); tokio::spawn(async move { let update_task = async move { match AssertUnwindSafe(async move { @@ -3609,6 +3896,7 @@ impl TierConfigMgr { } let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string); let mutation_force = mutation.force(); + let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; let current_for_targets = TierConfigMgr { driver_cache: HashMap::new(), tiers: candidate @@ -3625,13 +3913,12 @@ impl TierConfigMgr { .map_err(TierConfigUpdateError::Publish)? }; transition - .wait_for_active_leases() + .wait_for_active_leases_until(validation_deadline) .await .map_err(TierConfigUpdateError::Publish)?; - let driver_tier = - apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT) - .await - .map_err(TierConfigUpdateError::Mutation)?; + let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline) + .await + .map_err(TierConfigUpdateError::Mutation)?; let proof_targets = tier_mutation_proof_targets( mutation_kind, explicit_tier_name.as_deref(), @@ -3830,10 +4117,15 @@ impl TierConfigMgr { }; #[cfg(test)] { - if let Some(peers) = test_peers { - TIER_MUTATION_TEST_PEERS.scope(peers, update_task).await - } else { - update_task.await + match (test_peers, test_driver_factory) { + (Some(peers), Some(factory)) => { + TIER_DRIVER_TEST_FACTORY + .scope(factory, TIER_MUTATION_TEST_PEERS.scope(peers, update_task)) + .await + } + (Some(peers), None) => TIER_MUTATION_TEST_PEERS.scope(peers, update_task).await, + (None, Some(factory)) => TIER_DRIVER_TEST_FACTORY.scope(factory, update_task).await, + (None, None) => update_task.await, } } #[cfg(not(test))] @@ -4923,7 +5215,8 @@ impl TierConfigMgr { tier_config: TierConfig, force: bool, ) -> std::result::Result<(), TierConfigUpdateError> { - Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Add(tier_config, force)).await + let mutation = TierCandidateMutation::add(tier_config, force).map_err(TierConfigUpdateError::Mutation)?; + Self::update_candidate_with_config_lock(handle, api, mutation).await } pub async fn edit_and_save( @@ -4936,6 +5229,32 @@ impl TierConfigMgr { .await } + #[cfg(test)] + async fn edit_and_save_with( + handle: &Arc>, + api: Arc, + tier_name: &str, + credentials: TierCreds, + ) -> std::result::Result<(), TierConfigUpdateError> + where + S: TierReferenceProofStore + 'static, + { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned( + handle, + api, + candidate, + version, + TierCandidateMutation::Edit(tier_name.to_string(), credentials), + update, + None, + ) + .await + } + pub async fn remove_and_save( handle: &Arc>, api: Arc, @@ -4999,8 +5318,14 @@ impl TierConfigMgr { let lease = Self::acquire_operation_lease(handle, tier_name) .await .map_err(io::Error::other)?; - let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone())); - check_warm_backend(Some(&driver)).await.map_err(io::Error::other) + tokio::spawn(async move { + let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone())); + let result = check_warm_backend(Some(&driver)).await.map_err(io::Error::other); + drop(lease); + result + }) + .await + .map_err(|_| io::Error::other("remote tier verification task failed"))? } pub(crate) async fn acquire_operation_lease_for_backend_identity( @@ -5060,7 +5385,7 @@ impl TierConfigMgr { let driver: SharedWarmBackend = Arc::from(driver); let entry = Arc::new(TierDriverGeneration { tier_name: Arc::from(tier_name), - tier_config: config.clone(), + tier_config: config.clone_with_credentials(), generation, config_fingerprint, backend_identity, @@ -6379,7 +6704,7 @@ mod tests { // // These tests must not reach a real remote tier. Two techniques keep them // hermetic: - // * error paths that return *before* `new_warm_backend` constructs a + // * error paths that return *before* `build_warm_backend` constructs a // client (name validation, duplicate detection, unsupported type, // missing backend payload, missing credentials); // * a `MockWarmBackend` injected directly into `driver_cache`, so @@ -6387,6 +6712,7 @@ mod tests { // lets us drive `remove`/`verify` through every branch. // --------------------------------------------------------------------- + use crate::services::tier::test_util::{MockWarmBackend as RecordingWarmBackend, MockWarmOp}; use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts}; use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; @@ -6417,6 +6743,24 @@ mod tests { } } + fn build_gcs_tier(name: &str, credentials: &str) -> TierConfig { + TierConfig { + version: "v1".to_string(), + tier_type: TierType::GCS, + name: name.to_string(), + gcs: Some(crate::services::tier::tier_config::TierGCS { + name: name.to_string(), + endpoint: "https://storage.googleapis.com".to_string(), + creds: credentials.to_string(), + bucket: "bucket-gcs".to_string(), + prefix: "prefix-gcs".to_string(), + region: String::new(), + storage_class: String::new(), + }), + ..Default::default() + } + } + #[derive(Debug)] struct LockingTierConfigStore { locks: Mutex>, @@ -6766,6 +7110,51 @@ mod tests { in_use_value: Option, /// When false, put/get/remove all fail (drives `verify` error paths). healthy: bool, + probe_present: AtomicBool, + } + + #[derive(Default)] + struct HangingInUseBackend { + probe_present: AtomicBool, + } + + #[async_trait::async_trait] + impl WarmBackend for HangingInUseBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result { + self.probe_present.store(true, Ordering::SeqCst); + Ok("probe-version".to_string()) + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> io::Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result { + Ok(BufReader::new(Cursor::new(b"RustFS".to_vec()))) + } + + async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> { + self.probe_present.store(false, Ordering::SeqCst); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> io::Result { + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent("probe-version".to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> io::Result { + std::future::pending().await + } } #[async_trait::async_trait] @@ -6788,6 +7177,7 @@ mod tests { async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result { if self.healthy { + self.probe_present.store(true, Ordering::SeqCst); Ok("mock-version".to_string()) } else { Err(std::io::Error::other("mock put failed")) @@ -6819,6 +7209,7 @@ mod tests { async fn remove(&self, _object: &str, _rv: &str) -> std::result::Result<(), std::io::Error> { if self.healthy { + self.probe_present.store(false, Ordering::SeqCst); Ok(()) } else { Err(std::io::Error::other("mock remove failed")) @@ -6832,6 +7223,20 @@ mod tests { self.remove(object, rv).await } + async fn probe_transition_candidate( + &self, + _object: &str, + ) -> std::result::Result { + if !self.healthy { + return Err(std::io::Error::other("mock candidate probe failed")); + } + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent("mock-version".to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> std::result::Result { match self.in_use_value { Some(b) => Ok(b), @@ -6845,8 +7250,37 @@ mod tests { mgr.driver_cache.insert(name.to_string(), Box::new(mock)); } + fn recording_driver_factory( + backend: RecordingWarmBackend, + observed_configs: Arc>>, + ) -> TierDriverTestFactory { + Arc::new(move |config| { + lock_unpoisoned(&observed_configs).push(config.clone_with_credentials()); + Ok(Box::new(backend.clone())) + }) + } + + fn healthy_driver_factory() -> TierDriverTestFactory { + recording_driver_factory(RecordingWarmBackend::new(), Arc::new(Mutex::new(Vec::new()))) + } + // ---- add ------------------------------------------------------------ + #[test] + fn test_add_name_normalization_uses_canonical_name_and_supports_legacy_nested_name() { + let mut canonical_s3 = build_s3_tier("COLD-CANONICAL"); + canonical_s3.s3.as_mut().expect("S3 payload should exist").name.clear(); + normalize_s3_gcs_add_tier_name(&mut canonical_s3).expect("canonical S3 name should normalize"); + assert_eq!(canonical_s3.name, "COLD-CANONICAL"); + assert_eq!(canonical_s3.s3.as_ref().expect("S3 payload should remain").name, "COLD-CANONICAL"); + + let mut legacy_gcs = build_gcs_tier("COLD-LEGACY", r#"{"type":"service_account"}"#); + legacy_gcs.name.clear(); + normalize_s3_gcs_add_tier_name(&mut legacy_gcs).expect("legacy nested GCS name should normalize"); + assert_eq!(legacy_gcs.name, "COLD-LEGACY"); + assert_eq!(legacy_gcs.gcs.as_ref().expect("GCS payload should remain").name, "COLD-LEGACY"); + } + #[tokio::test] async fn test_add_rejects_non_uppercase_name() { let mut mgr = empty_mgr(); @@ -6952,24 +7386,102 @@ mod tests { assert!(mgr.tiers.is_empty()); } + #[tokio::test] + async fn test_add_rejects_canonical_azure_sp_auth_wire_before_backend_setup() { + let tier: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "azure", + "Name": "COLD-AZURE", + "azure": { + "name": "COLD-AZURE", + "endpoint": "https://azure.example.invalid", + "accessKey": "account", + "secretKey": "key", + "bucket": "archive", + "spAuth": { + "TenantID": "tenant" + } + } + })) + .expect("mixed RustFS/madmin Azure payload should decode"); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let mut mgr = empty_mgr(); + + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(tier, true)) + .await + .expect_err("canonical Azure service-principal fields must fail closed"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("spAuth"), "{}", err.message); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0); + assert!(mgr.tiers.is_empty()); + } + #[tokio::test] async fn test_add_does_not_reject_azure_config_without_storage_class_or_sp_auth() { // A plain Azure config (the common case: static access/secret key, no - // storageClass, no spAuth) must sail past the new gate. `new_warm_backend` - // builds the S3-compatible client lazily (no eager DNS/connect), so with - // `force: true` (which also skips the `in_use` probe) this succeeds even - // against a fake endpoint — the point here is only that the gate itself - // does not fire. + // storageClass, no spAuth) must sail past the provider-specific gate. let mut mgr = empty_mgr(); let tier = build_azure_tier("account-a"); let tier_name = tier.name.clone(); - mgr.add(tier, true) + TIER_DRIVER_TEST_FACTORY + .scope(healthy_driver_factory(), mgr.add(tier, true)) .await .expect("a config with no storageClass/spAuth must not trip the new gate"); assert!(mgr.tiers.contains_key(&tier_name)); } + #[tokio::test] + async fn test_add_force_does_not_bypass_bad_credentials_or_unreachable_probe() { + for unreachable in [false, true] { + let mut mgr = empty_mgr(); + let backend = RecordingWarmBackend::new(); + if unreachable { + backend.set_unreachable(true).await; + } else { + backend.set_reject_credentials(true).await; + } + let observed = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, observed.clone()), + mgr.add(build_s3_tier("COLD-A"), true), + ) + .await + .expect_err("force must not bypass the backend read/write/delete probe"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(lock_unpoisoned(&observed).len(), 1); + assert!(mgr.tiers.is_empty()); + } + } + + #[tokio::test] + async fn test_add_rejects_unsupported_s3_role_before_backend_setup() { + let mut mgr = empty_mgr(); + let mut tier = build_s3_tier("COLD-A"); + let s3 = tier.s3.as_mut().expect("S3 payload should exist"); + s3.access_key.clear(); + s3.secret_key.clear(); + s3.aws_role = true; + + let err = mgr + .add(tier, true) + .await + .expect_err("unsupported role credentials must fail before backend setup"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + assert!(mgr.tiers.is_empty()); + } + #[tokio::test] async fn test_add_rejects_reserved_names() { // Supersedes the former `test_add_does_not_reserve_standard_name_regression_anchor` @@ -6998,6 +7510,240 @@ mod tests { } } + #[tokio::test] + async fn test_add_rejects_canonical_s3_role_fields_even_with_static_credentials() { + let mut tier: TierConfig = serde_json::from_value(serde_json::json!({ + "Version": "v1", + "Type": "s3", + "Name": "COLD-A", + "S3": { + "Endpoint": "https://s3.example.invalid", + "AccessKey": "static-access", + "SecretKey": "static-secret", + "Bucket": "archive", + "Prefix": "objects", + "Region": "us-east-1", + "StorageClass": "STANDARD", + "AWSRole": true, + "AWSRoleWebIdentityTokenFile": "/var/run/private-token", + "AWSRoleARN": "arn:aws:iam::123456789012:role/archive", + "AWSRoleSessionName": "archive-session", + "AWSRoleDurationSeconds": 900 + } + })) + .expect("canonical madmin S3 role payload should decode"); + assert_eq!(tier.name, "COLD-A"); + let s3 = tier.s3.as_ref().expect("S3 payload should decode"); + assert!(s3.name.is_empty(), "canonical madmin S3 has no nested Name field"); + assert_eq!(s3.endpoint, "https://s3.example.invalid"); + assert!(!s3.access_key.is_empty()); + assert!(!s3.secret_key.is_empty()); + assert_eq!(s3.bucket, "archive"); + assert_eq!(s3.prefix, "objects"); + assert_eq!(s3.region, "us-east-1"); + assert_eq!(s3.storage_class, "STANDARD"); + assert!(s3.aws_role); + assert_eq!(s3.aws_role_web_identity_token_file, "/var/run/private-token"); + assert_eq!(s3.aws_role_duration_seconds, 900); + normalize_s3_gcs_add_tier_name(&mut tier).expect("canonical madmin Name should normalize before validation"); + assert_eq!(tier.name, "COLD-A"); + assert_eq!(tier.s3.as_ref().expect("S3 payload should remain").name, "COLD-A"); + let serialized = serde_json::to_value(&tier).expect("S3 config should serialize"); + assert_eq!(serialized["type"], "s3"); + assert!(serialized.get("Type").is_none()); + assert!(serialized.get("S3").is_none()); + assert!(serialized.get("Name").is_none()); + assert!(serialized["s3"].get("AccessKey").is_none()); + assert!(serialized["s3"].get("accessKey").is_some()); + assert!(serialized["s3"].get("AWSRole").is_none()); + assert!(serialized["s3"].get("AWSRoleWebIdentityTokenFile").is_none()); + + let mut mgr = empty_mgr(); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_backend_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_backend_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(tier, true)) + .await + .expect_err("unsupported role fields must be rejected before backend setup"); + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0); + assert!(mgr.tiers.is_empty()); + } + + #[tokio::test] + async fn test_add_rejects_conflicting_canonical_and_legacy_s3_gcs_names_before_backend_setup() { + let fixtures = [ + ( + "S3", + serde_json::json!({ + "Type": "S3", + "Name": "COLD-CANONICAL", + "S3": { + "name": "COLD-LEGACY", + "Endpoint": "https://s3.example.invalid", + "AccessKey": "access", + "SecretKey": "secret", + "Bucket": "archive" + } + }), + ), + ( + "GCS", + serde_json::json!({ + "Type": "GCS", + "Name": "COLD-CANONICAL", + "GCS": { + "name": "COLD-LEGACY", + "Endpoint": "https://storage.googleapis.com/", + "Creds": "e30=", + "Bucket": "archive" + } + }), + ), + ]; + + for (provider, fixture) in fixtures { + let tier: TierConfig = serde_json::from_value(fixture) + .unwrap_or_else(|err| panic!("{provider} aliases should decode before name validation: {err}")); + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_backend_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_backend_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + factory, + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::update_candidate_with_config_lock(&manager, store, TierCandidateMutation::Add(tier, true)), + ), + ) + .await + .expect_err("conflicting canonical and legacy tier names must fail closed"); + let TierConfigUpdateError::Mutation(err) = err else { + panic!("conflicting {provider} names should fail as a typed mutation error") + }; + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("conflicts"), "{}", err.message); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0, "{provider} backend must not be prepared"); + assert!(lock_unpoisoned(&peer_calls).is_empty(), "{provider} conflict must not prepare peers"); + assert!(manager.read().await.tiers.is_empty()); + } + } + + #[tokio::test(start_paused = true)] + async fn test_add_times_out_a_hanging_in_use_check_after_probe_cleanup() { + let mut mgr = empty_mgr(); + let factory: TierDriverTestFactory = Arc::new(|_| Ok(Box::new(HangingInUseBackend::default()))); + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(build_rustfs_tier("COLD-A"), false)) + .await + .expect_err("a hanging in-use check must not retain the admin lock forever"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out checking")); + assert!(mgr.tiers.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn candidate_add_honors_the_caller_validation_deadline() { + let mut candidate = empty_mgr(); + let factory: TierDriverTestFactory = Arc::new(|_| Ok(Box::new(HangingInUseBackend::default()))); + let deadline = Instant::now() + Duration::from_secs(1); + let err = { + let add = TIER_DRIVER_TEST_FACTORY.scope( + factory, + apply_tier_candidate_mutation( + TierCandidateMutation::Add(build_rustfs_tier("COLD-DEADLINE"), false), + &mut candidate, + deadline, + ), + ); + tokio::pin!(add); + tokio::task::yield_now().await; + + let result = tokio::time::timeout(Duration::from_secs(2), &mut add); + tokio::pin!(result); + tokio::time::advance(Duration::from_secs(2) + Duration::from_millis(1)).await; + result + .await + .expect("the add mutation must finish within its caller deadline") + .expect_err("a hanging in-use check must fail the add mutation") + }; + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out checking")); + assert!(candidate.tiers.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn candidate_edit_honors_the_caller_validation_deadline() { + let mut candidate = empty_mgr(); + candidate + .tiers + .insert("COLD-DEADLINE".to_string(), build_rustfs_tier("COLD-DEADLINE")); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + let factory = recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))); + let deadline = Instant::now() + Duration::from_secs(1); + let err = { + let edit = TIER_DRIVER_TEST_FACTORY.scope( + factory, + apply_tier_candidate_mutation( + TierCandidateMutation::Edit( + "COLD-DEADLINE".to_string(), + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + &mut candidate, + deadline, + ), + ); + tokio::pin!(edit); + tokio::select! { + _ = get_barrier.wait_until_paused() => {} + result = &mut edit => panic!("edit completed before the probe deadline: {result:?}"), + } + + let result = tokio::time::timeout(Duration::from_secs(2), &mut edit); + tokio::pin!(result); + tokio::time::advance(Duration::from_secs(2) + Duration::from_millis(1)).await; + result + .await + .expect("the edit mutation must finish within its caller deadline") + .expect_err("a hanging in-use check must fail the edit mutation") + }; + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out validating")); + assert_eq!(backend.object_count().await, 0, "deadline-aware edit must clean up its probe object"); + assert_eq!( + candidate.tiers["COLD-DEADLINE"] + .rustfs + .as_ref() + .expect("tier payload") + .access_key, + "ak" + ); + } + // ---- edit ----------------------------------------------------------- #[tokio::test] @@ -7011,32 +7757,46 @@ mod tests { } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_rustfs() { + async fn test_edit_rejects_half_static_credentials_for_rustfs() { let mut mgr = empty_mgr(); mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); - // Empty access/secret keys => rejected before any driver rebuild. let err = mgr - .edit("COLD-R", TierCreds::default()) + .edit( + "COLD-R", + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty credentials must be rejected"); + .expect_err("a half-filled static credential pair must be rejected"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); + let rustfs = mgr.tiers["COLD-R"].rustfs.as_ref().expect("original payload should remain"); + assert_eq!(rustfs.access_key, "ak"); + assert_eq!(rustfs.secret_key, "sk"); } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_wasabi() { + async fn test_edit_rejects_half_static_credentials_for_wasabi() { let mut mgr = empty_mgr(); mgr.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI")); let err = mgr - .edit("COLD-WASABI", TierCreds::default()) + .edit( + "COLD-WASABI", + TierCreds { + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty Wasabi credentials must be rejected before backend setup"); + .expect_err("a half-filled Wasabi credential pair must be rejected before backend setup"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_minio() { + async fn test_edit_rejects_half_static_credentials_for_minio() { let mut mgr = empty_mgr(); let tier = TierConfig { version: "v1".to_string(), @@ -7056,12 +7816,290 @@ mod tests { mgr.tiers.insert("COLD-M".to_string(), tier); let err = mgr - .edit("COLD-M", TierCreds::default()) + .edit( + "COLD-M", + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty credentials must be rejected"); + .expect_err("a half-filled static credential pair must be rejected"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); } + #[tokio::test] + async fn test_edit_with_omitted_secret_fields_preserves_real_credentials() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); + let backend = RecordingWarmBackend::new(); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))), + mgr.edit("COLD-R", TierCreds::default()), + ) + .await + .expect("an edit with omitted secret fields should validate the preserved credentials"); + + let rustfs = mgr.tiers["COLD-R"].rustfs.as_ref().expect("edited payload should remain"); + assert_eq!(rustfs.access_key, "ak"); + assert_eq!(rustfs.secret_key, "sk"); + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn test_edit_rejects_redaction_placeholder() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); + + let err = mgr + .edit( + "COLD-R", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: TIER_CREDENTIAL_REDACTED.to_string(), + ..Default::default() + }, + ) + .await + .expect_err("the API redaction placeholder must never become a real credential"); + + assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); + assert_eq!( + mgr.tiers["COLD-R"] + .rustfs + .as_ref() + .expect("original payload should remain") + .secret_key, + "sk" + ); + } + + #[tokio::test] + async fn test_edit_rejects_unsupported_s3_role_before_backend_setup() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A")); + + let err = mgr + .edit( + "COLD-A", + TierCreds { + aws_role: true, + ..Default::default() + }, + ) + .await + .expect_err("unsupported role credentials must fail before backend setup"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + } + + #[tokio::test] + async fn test_edit_gcs_rotates_from_creds_json_and_runs_backend_probe() { + const OLD_CREDS: &str = r#"{"type":"service_account","project_id":"old-project"}"#; + const ROTATED_CREDS: &str = r#"{"type":"service_account","project_id":"rotated-project"}"#; + let mut mgr = empty_mgr(); + mgr.tiers + .insert("COLD-GCS".to_string(), build_gcs_tier("COLD-GCS", OLD_CREDS)); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + mgr.edit( + "COLD-GCS", + TierCreds { + creds_json: ROTATED_CREDS.as_bytes().to_vec(), + ..Default::default() + }, + ), + ) + .await + .expect("GCS service account rotation should validate through the fake backend"); + + assert_eq!( + mgr.tiers["COLD-GCS"].gcs.as_ref().expect("GCS payload should remain").creds, + ROTATED_CREDS + ); + assert_eq!( + lock_unpoisoned(&observed)[0] + .gcs + .as_ref() + .expect("observed GCS payload should exist") + .creds, + ROTATED_CREDS + ); + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn test_gcs_madmin_add_wire_normalizes_url_safe_credentials_to_raw_v2_storage() { + const INITIAL_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; + const INITIAL_MINIO_URL_BASE64: &str = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt4K-_In0="; + const ROTATED_JSON: &str = r#"{"type":"service_account","project_id":"rotated-project"}"#; + let madmin_wire_add: TierConfig = serde_json::from_value(serde_json::json!({ + "Version": "v1", + "Type": "gcs", + "Name": "COLD-GCS", + "GCS": { + "Endpoint": "https://storage.googleapis.com/", + "Creds": INITIAL_MINIO_URL_BASE64, + "Bucket": "bucket-gcs", + "Prefix": "prefix-gcs", + "Region": "", + "StorageClass": "" + } + })) + .expect("canonical madmin GCS AddTier payload should decode"); + assert_eq!(madmin_wire_add.name, "COLD-GCS"); + assert!( + madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration") + .name + .is_empty(), + "canonical madmin GCS has no nested Name field" + ); + assert_eq!( + madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration") + .creds, + INITIAL_MINIO_URL_BASE64 + ); + let gcs = madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration"); + assert_eq!(gcs.endpoint, "https://storage.googleapis.com/"); + assert_eq!(gcs.bucket, "bucket-gcs"); + assert_eq!(gcs.prefix, "prefix-gcs"); + assert!(gcs.region.is_empty()); + assert!(gcs.storage_class.is_empty()); + let rustfs_output = serde_json::to_value(&madmin_wire_add).expect("GCS config should serialize"); + assert_eq!(rustfs_output["type"], "gcs"); + assert!(rustfs_output.get("Type").is_none()); + assert!(rustfs_output.get("GCS").is_none()); + assert!(rustfs_output.get("Name").is_none()); + assert!(rustfs_output["gcs"].get("Creds").is_none()); + assert_eq!(rustfs_output["gcs"]["creds"], INITIAL_MINIO_URL_BASE64); + let mut mgr = empty_mgr(); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + mgr.add(madmin_wire_add, true), + ) + .await + .expect("GCS add should normalize madmin URL-safe credentials before validation"); + + assert_eq!( + lock_unpoisoned(&observed)[0] + .gcs + .as_ref() + .expect("the backend should receive a GCS payload") + .creds, + INITIAL_JSON + ); + assert_eq!( + mgr.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("the added GCS payload should exist") + .creds, + INITIAL_JSON + ); + assert_eq!(mgr.tiers["COLD-GCS"].name, "COLD-GCS"); + assert_eq!( + mgr.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("the added GCS payload should exist") + .name, + "COLD-GCS" + ); + + let blob = encode_external_tiering_config_blob(&mgr).expect("GCS add should persist externally"); + let external: ExternalTierConfigMgr = rmp_serde::from_slice(&blob[4..]).expect("external GCS payload should decode"); + assert_eq!( + external.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + INITIAL_JSON + ); + let mut loaded = decode_external_tiering_config_blob(&blob).expect("persisted GCS tier should load"); + assert_eq!( + loaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("loaded GCS payload should exist") + .creds, + INITIAL_JSON + ); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + loaded.edit( + "COLD-GCS", + TierCreds { + creds_json: ROTATED_JSON.as_bytes().to_vec(), + ..Default::default() + }, + ), + ) + .await + .expect("GCS edit should accept raw madmin credential bytes"); + assert_eq!( + loaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("edited GCS payload should exist") + .creds, + ROTATED_JSON + ); + let rotated_blob = encode_external_tiering_config_blob(&loaded).expect("edited GCS tier should persist externally"); + assert_eq!( + rmp_serde::from_slice::(&rotated_blob[4..]) + .expect("rotated external tier should exist") + .tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("rotated external GCS payload should exist") + .creds, + ROTATED_JSON + ); + let reloaded = decode_external_tiering_config_blob(&rotated_blob).expect("edited GCS tier should reload"); + assert_eq!( + reloaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("reloaded GCS payload should exist") + .creds, + ROTATED_JSON + ); + } + // ---- remove --------------------------------------------------------- #[tokio::test] @@ -7081,6 +8119,7 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7103,6 +8142,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7122,6 +8162,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7140,11 +8181,12 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true); mutation - .apply(&mut candidate) + .apply(&mut candidate, None) .await .expect("force remove must skip in-use probing"); assert!(!candidate.tiers.contains_key("COLD-A")); @@ -7160,10 +8202,11 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); let err = TierCandidateMutation::Remove("COLD-A".to_string(), false) - .apply(&mut candidate) + .apply(&mut candidate, None) .await .expect_err("non-force remove must reject an in-use backend"); assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code); @@ -7180,6 +8223,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); forced.clear_tier(true).await.expect("force clear must skip in-use probing"); @@ -7193,6 +8237,7 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); let err = guarded @@ -7213,6 +8258,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7243,6 +8289,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7259,6 +8306,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: false, + probe_present: AtomicBool::new(false), }, ); @@ -7272,6 +8320,7 @@ mod tests { let unhealthy: SharedWarmBackend = Arc::new(MockWarmBackend { in_use_value: Some(false), healthy: false, + probe_present: AtomicBool::new(false), }); let proxy = SharedWarmBackendProxy(unhealthy); let err = proxy.validate().await.expect_err("proxy must forward backend validation"); @@ -7280,6 +8329,7 @@ mod tests { let healthy: SharedWarmBackend = Arc::new(MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }); let proxy = SharedWarmBackendProxy(healthy); let err = proxy @@ -7329,7 +8379,7 @@ mod tests { assert_eq!(tier.tier_type.as_lowercase(), "s3"); let s3 = tier.s3.as_ref().expect("s3 payload survives"); assert_eq!(s3.bucket, "bucket-a"); - // secret_key is a serialized field (unlike Clone, marshal does not redact). + // Internal serialization preserves credentials; only explicit admin views redact them. assert_eq!(s3.secret_key, "sk"); } @@ -7374,7 +8424,8 @@ mod tests { } #[test] - fn test_external_blob_roundtrip_gcs_preserves_creds() { + fn test_external_blob_gcs_writer_stays_raw_for_v2_old_readers() { + const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; let mut mgr = empty_mgr(); mgr.tiers.insert( "COLD-G".to_string(), @@ -7385,7 +8436,7 @@ mod tests { gcs: Some(crate::services::tier::tier_config::TierGCS { name: "COLD-G".to_string(), endpoint: "https://storage.googleapis.com".to_string(), - creds: "service-account-json".to_string(), + creds: RAW_JSON.to_string(), bucket: "gbucket".to_string(), prefix: "gp".to_string(), region: "us".to_string(), @@ -7396,10 +8447,118 @@ mod tests { ); let bytes = encode_external_tiering_config_blob(&mgr).expect("encode gcs tier"); + assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes()); + assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes()); + let external: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external GCS payload should decode"); + assert_eq!( + external.tiers["COLD-G"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + RAW_JSON + ); + serde_json::from_str::( + &external.tiers["COLD-G"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + ) + .expect("the baseline RustFS v2 reader must receive raw GCS credential JSON"); let decoded = decode_external_tiering_config_blob(&bytes).expect("decode gcs tier"); let tier = decoded.tiers.get("COLD-G").expect("gcs tier survives roundtrip"); assert_eq!(tier.tier_type.as_lowercase(), "gcs"); - assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, "service-account-json"); + assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, RAW_JSON); + + let rewritten = encode_external_tiering_config_blob(&decoded).expect("decoded GCS tier should remain v2-compatible"); + let rewritten: ExternalTierConfigMgr = + rmp_serde::from_slice(&rewritten[4..]).expect("rewritten external GCS payload should decode"); + assert_eq!( + rewritten.tiers["COLD-G"] + .gcs + .as_ref() + .expect("rewritten external GCS payload should exist") + .creds, + RAW_JSON + ); + } + + fn decode_hex_fixture(hex: &str) -> Vec { + assert_eq!(hex.len() % 2, 0, "hex fixture must contain complete bytes"); + hex.as_bytes() + .chunks_exact(2) + .map(|pair| { + let pair = std::str::from_utf8(pair).expect("hex fixture should be ASCII"); + u8::from_str_radix(pair, 16).expect("hex fixture should contain only hexadecimal digits") + }) + .collect() + } + + #[test] + fn test_external_blob_decodes_fixed_minio_gcs_fixture_and_rewrites_raw_v2_credentials() { + const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; + // Produced by MinIO RELEASE.2025-10-15T17-29-55Z with madmin-go/v3 + // v3.0.109. Keeping the bytes fixed prevents this compatibility test + // from accidentally validating RustFS against its own serializer. + const MINIO_GCS_BLOB_HEX: &str = concat!( + "0100020081a5546965727381a6434f4c442d4787a756657273696f6ea27631a45479706503a44e616d65a6434f4c442d47", + "a25333c0a5417a757265c0a347435386a8456e64706f696e74bf68747470733a2f2f73746f726167652e676f6f676c656170", + "69732e636f6d2fa54372656473d94465794a306558426c496a6f6963325679646d6c6a5a56396859324e7664573530496977", + "6963484a76616d566a644639705a434936496e52705a584974344b2d5f496e303da64275636b6574a7676275636b6574a650", + "7265666978a26770a6526567696f6ea27573ac53746f72616765436c617373a84e4541524c494e45a54d696e494fc0" + ); + const MINIO_GCS_BLOB_SHA256_HEX: &str = "bee1d4822d4936bc6f764c284381596d16f2096f40671b25500aca3b710ceac8"; + let fixture = decode_hex_fixture(MINIO_GCS_BLOB_HEX); + assert_eq!(fixture.len(), 246); + assert_eq!( + Sha256::digest(&fixture).as_slice(), + decode_hex_fixture(MINIO_GCS_BLOB_SHA256_HEX).as_slice(), + "fixed foreign fixture digest changed" + ); + + let decoded = decode_external_tiering_config_blob(&fixture) + .expect("the new reader should accept MinIO URL-safe-base64 GCS credentials"); + let gcs = decoded.tiers["COLD-G"] + .gcs + .as_ref() + .expect("decoded MinIO GCS payload should exist"); + assert_eq!(gcs.endpoint, "https://storage.googleapis.com/"); + assert_eq!(gcs.bucket, "gbucket"); + assert_eq!(gcs.prefix, "gp"); + assert_eq!(gcs.region, "us"); + assert_eq!(gcs.storage_class, "NEARLINE"); + assert_eq!(gcs.creds, RAW_JSON); + + let rewritten = encode_external_tiering_config_blob(&decoded) + .expect("MinIO credentials should rewrite in the existing RustFS v2 raw format"); + let rewritten: ExternalTierConfigMgr = + rmp_serde::from_slice(&rewritten[4..]).expect("rewritten MinIO fixture should decode"); + assert_eq!( + rewritten.tiers["COLD-G"] + .gcs + .as_ref() + .expect("rewritten MinIO GCS payload should exist") + .creds, + RAW_JSON + ); + } + + #[test] + fn external_gcs_credentials_accept_all_base64_alphabets_and_padding_modes() { + let raw = r#"{"type":"service_account","project_id":"tier-🚀"}"#; + for encoder in [ + base64_simd::STANDARD, + base64_simd::STANDARD_NO_PAD, + base64_simd::URL_SAFE, + base64_simd::URL_SAFE_NO_PAD, + ] { + let encoded = encoder.encode_to_string(raw.as_bytes()); + assert_eq!( + decode_external_gcs_credentials(&encoded).expect("supported GCS encoding should decode"), + raw + ); + } } // `TierConfigMgr` intentionally does not derive `Debug` (it holds live @@ -7546,6 +8705,7 @@ mod tests { struct LeaseTestBackend { id: &'static str, calls: Arc>>, + probe_present: Arc, remove_started: Option>, remove_release: Option>, backend_in_use: bool, @@ -7558,6 +8718,7 @@ mod tests { Self { id, calls: Arc::new(std::sync::Mutex::new(Vec::new())), + probe_present: Arc::new(AtomicBool::new(false)), remove_started: None, remove_release: None, backend_in_use: false, @@ -7603,6 +8764,7 @@ mod tests { #[async_trait::async_trait] impl WarmBackend for LeaseTestBackend { async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result { + self.probe_present.store(true, Ordering::SeqCst); Ok(self.id.to_string()) } @@ -7617,7 +8779,7 @@ mod tests { } async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result { - Ok(BufReader::new(Cursor::new(Vec::new()))) + Ok(BufReader::new(Cursor::new(b"RustFS".to_vec()))) } async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> { @@ -7635,9 +8797,18 @@ mod tests { .expect("lease test release semaphore should stay open") .forget(); } + self.probe_present.store(false, Ordering::SeqCst); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> io::Result { + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(self.id.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> io::Result { if self.panic_in_use { panic!("simulated tier backend in-use panic"); @@ -7685,6 +8856,7 @@ mod tests { struct FakeTierMutationPeer { label: &'static str, calls: Arc>>, + captured_prepares: Option>>>, prepare: std::result::Result, prepare_definitely_rejected: bool, commit: std::result::Result, @@ -7709,6 +8881,7 @@ mod tests { Arc::new(Self { label, calls, + captured_prepares: None, prepare, prepare_definitely_rejected: false, commit, @@ -7716,6 +8889,22 @@ mod tests { }) } + fn boxed_capturing_prepare( + label: &'static str, + calls: Arc>>, + captured_prepares: Arc>>, + ) -> Arc { + Arc::new(Self { + label, + calls, + captured_prepares: Some(captured_prepares), + prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, + commit: Ok(PeerTierMutationState::Committed), + abort: Ok(PeerTierMutationState::Aborted), + }) + } + fn record(&self, call: String) { lock_unpoisoned(&self.calls).push(call); } @@ -7732,6 +8921,10 @@ mod tests { mutation_id: uuid::Uuid, canonical_payload: Bytes, ) -> Result { + if let Some(captured_prepares) = self.captured_prepares.as_ref() { + let intent = TierMutationIntent::decode(mutation_id, &canonical_payload).map_err(Error::other)?; + lock_unpoisoned(captured_prepares).push(intent); + } self.record(format!("{}:prepare:{}:{}", self.label, mutation_id, canonical_payload.len())); match self.prepare { Ok(state) => Ok(state), @@ -8379,6 +9572,7 @@ mod tests { let old_peer = Arc::new(FakeTierMutationPeer { label: "peer-v3", calls: calls.clone(), + captured_prepares: None, prepare: Err("unsupported tier mutation peer protocol version: 4"), prepare_definitely_rejected: true, commit: Ok(PeerTierMutationState::Committed), @@ -8449,6 +9643,7 @@ mod tests { Arc::new(FakeTierMutationPeer { label: "peer-a", calls: calls.clone(), + captured_prepares: None, prepare: Ok(PeerTierMutationState::Prepared), prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), @@ -8961,41 +10156,44 @@ mod tests { let update = TierConfigMgr::admin_update_lock(&manager).await; let calls = Arc::new(Mutex::new(Vec::new())); - TIER_MUTATION_TEST_PEERS + TIER_DRIVER_TEST_FACTORY .scope( - vec![ - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-a", - calls.clone(), - Ok(PeerTierMutationState::Committed), - Ok(PeerTierMutationState::Committed), - ), - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-b", - calls.clone(), - Ok(PeerTierMutationState::Prepared), - Ok(PeerTierMutationState::Committed), - ), - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-c", - calls.clone(), - Ok(PeerTierMutationState::Prepared), - Ok(PeerTierMutationState::Committed), - ), - ], - async { - TierConfigMgr::update_candidate_owned( - &manager, - store.clone(), - candidate, - version, - TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), - update, - None, - ) - .await - .expect("four-hot AddTier reporter replay should publish after prepared peers commit"); - }, + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + vec![ + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-a", + calls.clone(), + Ok(PeerTierMutationState::Committed), + Ok(PeerTierMutationState::Committed), + ), + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-b", + calls.clone(), + Ok(PeerTierMutationState::Prepared), + Ok(PeerTierMutationState::Committed), + ), + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-c", + calls.clone(), + Ok(PeerTierMutationState::Prepared), + Ok(PeerTierMutationState::Committed), + ), + ], + async { + TierConfigMgr::update_candidate_owned( + &manager, + store.clone(), + candidate, + version, + TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), + update, + None, + ) + .await + .expect("four-hot AddTier reporter replay should publish after prepared peers commit"); + }, + ), ) .await; @@ -10655,6 +11853,7 @@ mod tests { vec![Arc::new(FakeTierMutationPeer { label: "peer-a", calls: calls.clone(), + captured_prepares: None, prepare: Ok(PeerTierMutationState::Prepared), prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), @@ -11487,6 +12686,57 @@ mod tests { verify.await.expect("verify task should join").expect("verify should finish"); } + #[tokio::test] + async fn cancelled_admin_verify_finishes_probe_cleanup_and_releases_its_lease() { + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + { + let mut guard = manager.write().await; + guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + guard + .replace_driver("COLD-A", Box::new(backend.clone())) + .expect("verification backend generation should install"); + } + + let verify_manager = manager.clone(); + let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await }); + get_barrier.wait_until_paused().await; + verify.abort(); + assert!( + verify + .await + .expect_err("the outer verify caller should be cancelled") + .is_cancelled() + ); + assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, "COLD-A").await, 1); + + get_barrier.release(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if backend.object_count().await == 0 && TierConfigMgr::active_operation_lease_count(&manager, "COLD-A").await == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the detached verification must finish cleanup and release its lease"); + + let operations = backend.op_log().await; + assert!(matches!( + operations.as_slice(), + [ + MockWarmOp::Put { .. }, + MockWarmOp::Probe { .. }, + MockWarmOp::Get { .. }, + MockWarmOp::Remove { .. }, + MockWarmOp::Probe { .. } + ] + )); + } + #[tokio::test] async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() { let old = LeaseTestBackend::ready("old"); @@ -13021,6 +14271,480 @@ mod tests { } } + #[tokio::test] + async fn edit_and_save_with_omitted_credentials_preserves_persisted_and_runtime_secrets() { + const ACCESS_KEY: &str = "persisted-access-value"; + const SECRET_KEY: &str = "persisted-secret-value"; + + let store = Arc::new(CasConfigStore::default()); + let mut tier = build_rustfs_tier("COLD-R"); + let rustfs = tier.rustfs.as_mut().expect("RustFS payload should exist"); + rustfs.access_key = ACCESS_KEY.to_string(); + rustfs.secret_key = SECRET_KEY.to_string(); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-R".to_string(), tier); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("credential-preservation fixture should persist"); + + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-R", TierCreds::default()), + ), + ) + .await + .expect("omitted credentials should preserve and publish the current credentials"); + + let reloaded = load_tier_config_for_update(store.clone()) + .await + .expect("edited tier config should reload") + .0; + let persisted_rustfs = reloaded.tiers["COLD-R"] + .rustfs + .as_ref() + .expect("persisted RustFS payload should exist"); + assert_eq!(persisted_rustfs.access_key, ACCESS_KEY); + assert_eq!(persisted_rustfs.secret_key, SECRET_KEY); + + let config_object = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("binary tier config should remain persisted") + .0 + .clone(); + assert!( + !config_object + .windows(TIER_CREDENTIAL_REDACTED.len()) + .any(|window| window == TIER_CREDENTIAL_REDACTED.as_bytes()), + "the API placeholder must never be persisted as a credential" + ); + + let (runtime_access_key, runtime_secret_key) = { + let manager = manager.read().await; + let runtime = registered_tier_driver_runtime(&manager).expect("driver runtime should be registered"); + let runtime = lock_unpoisoned(&runtime); + let generation = runtime + .generations + .get("COLD-R") + .expect("edited generation should be installed"); + let rustfs = generation + .tier_config + .rustfs + .as_ref() + .expect("runtime RustFS payload should exist"); + (rustfs.access_key.clone(), rustfs.secret_key.clone()) + }; + assert_eq!(runtime_access_key, ACCESS_KEY); + assert_eq!(runtime_secret_key, SECRET_KEY); + + let api_view = manager + .read() + .await + .get("COLD-R") + .expect("edited tier should remain visible through the admin view"); + assert_eq!( + api_view.rustfs.expect("admin RustFS payload should exist").secret_key, + TIER_CREDENTIAL_REDACTED + ); + let observed = lock_unpoisoned(&observed); + assert_eq!(observed.len(), 1); + assert_eq!( + observed[0] + .rustfs + .as_ref() + .expect("backend factory should observe the RustFS payload") + .secret_key, + SECRET_KEY + ); + drop(observed); + + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn edit_and_save_rotates_credentials_while_an_active_lifecycle_rule_references_the_tier() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("credential rotation fixture should persist"); + store.add_listed_version(ObjectInfo { + bucket: "photos".to_string(), + name: "safe.txt".to_string(), + ..Default::default() + }); + store.add_lifecycle_config( + "photos", + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("move-current".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: Some(vec![Transition { + days: Some(1), + date: None, + storage_class: Some(TransitionStorageClass::from_static("COLD-A")), + }]), + }], + }, + ); + + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::edit_and_save_with( + &manager, + store.clone(), + "COLD-A", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + ), + ) + .await + .expect("same-destination credential rotation must not be treated as a lifecycle rebind"); + + let reloaded = load_tier_config_for_update(store) + .await + .expect("rotated tier config should reload") + .0; + let rustfs = reloaded.tiers["COLD-A"] + .rustfs + .as_ref() + .expect("RustFS payload should remain"); + assert_eq!(rustfs.access_key, "rotated-access"); + assert_eq!(rustfs.secret_key, "rotated-secret"); + } + + async fn assert_edit_rejected_before_distributed_commit( + backend: RecordingWarmBackend, + credentials: TierCreds, + expected_code: &str, + expected_backend_builds: usize, + ) { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("rejected-edit fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("old")); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should be available") + .generation(); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::new(Mutex::new(Vec::new())); + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-A", credentials), + ), + ) + .await + .expect_err("credential validation must fail before distributed prepare"); + + let TierConfigUpdateError::Mutation(err) = err else { + panic!("credential validation should be reported as a mutation error: {err:?}"); + }; + assert_eq!(err.code, expected_code); + assert!(lock_unpoisoned(&peer_calls).is_empty()); + assert_eq!(lock_unpoisoned(&observed).len(), expected_backend_builds); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + let current = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("the old generation should be restored after validation failure"); + assert_eq!(current.generation(), old_generation); + assert_eq!( + current + .inner + .tier_config + .rustfs + .as_ref() + .expect("runtime RustFS payload should remain") + .secret_key, + "sk" + ); + } + + #[tokio::test] + async fn edit_and_save_rejects_half_credentials_before_peer_prepare_or_config_cas() { + assert_edit_rejected_before_distributed_commit( + RecordingWarmBackend::new(), + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + &ERR_TIER_MISSING_CREDENTIALS.code, + 0, + ) + .await; + } + + #[tokio::test] + async fn edit_and_save_rejects_canonical_azure_service_principal_before_peer_prepare() { + let credentials: TierCreds = serde_json::from_value(serde_json::json!({ + "azSP": { + "TenantID": "tenant", + "ClientID": "client", + "ClientSecret": "service-principal-secret" + } + })) + .expect("canonical madmin credentials should decode"); + assert_edit_rejected_before_distributed_commit( + RecordingWarmBackend::new(), + credentials, + &ERR_TIER_INVALID_CONFIG.code, + 0, + ) + .await; + } + + #[tokio::test] + async fn edit_and_save_rejects_legacy_azure_options_before_probe_or_peer_prepare() { + let store = Arc::new(CasConfigStore::default()); + let mut legacy = build_azure_tier("account-a"); + legacy.azure.as_mut().expect("Azure payload should exist").storage_class = "HOT".to_string(); + let mut persisted = empty_mgr(); + persisted + .tiers + .insert("COLD-AZURE".to_string(), legacy.clone_with_credentials()); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("legacy Azure fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + manager.tiers.insert("COLD-AZURE".to_string(), legacy); + manager + .replace_driver("COLD-AZURE", Box::new(LeaseTestBackend::ready("old"))) + .expect("legacy generation should install"); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-AZURE") + .await + .expect("legacy generation should be available") + .generation(); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(RecordingWarmBackend::new(), observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-AZURE", TierCreds::default()), + ), + ) + .await + .expect_err("legacy unsupported Azure options must fail closed on edit"); + + let TierConfigUpdateError::Mutation(err) = err else { + panic!("legacy Azure validation should be a mutation error: {err:?}"); + }; + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("storageClass"), "{}", err.message); + assert!(lock_unpoisoned(&observed).is_empty(), "validation must precede backend preparation"); + assert!(lock_unpoisoned(&peer_calls).is_empty(), "validation must precede peer prepare"); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + let current = TierConfigMgr::acquire_operation_lease(&manager, "COLD-AZURE") + .await + .expect("legacy generation should remain available"); + assert_eq!(current.generation(), old_generation); + assert_eq!( + current + .inner + .tier_config + .azure + .as_ref() + .expect("runtime Azure payload should remain") + .storage_class, + "HOT" + ); + } + + #[tokio::test] + async fn edit_and_save_probe_failure_keeps_old_generation_and_skips_peer_prepare() { + let backend = RecordingWarmBackend::new(); + backend.set_reject_credentials(true).await; + assert_edit_rejected_before_distributed_commit( + backend, + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + &ERR_TIER_PERM_ERR.code, + 1, + ) + .await; + } + + #[tokio::test(start_paused = true)] + async fn edit_and_save_get_timeout_cleans_probe_and_keeps_config_and_generation() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("timeout fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("old")); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should be available") + .generation(); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let edit = TIER_DRIVER_TEST_FACTORY.scope( + recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with( + &manager, + store.clone(), + "COLD-A", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + ), + ); + tokio::pin!(edit); + tokio::select! { + _ = get_barrier.wait_until_paused() => {} + result = &mut edit => panic!("edit completed before the probe GET timeout: {result:?}"), + } + tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await; + let err = edit.await.expect_err("a hanging probe GET must time out"); + let TierConfigUpdateError::Mutation(err) = err else { + panic!("probe timeout should be reported as a mutation error: {err:?}"); + }; + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(lock_unpoisoned(&peer_calls).is_empty()); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + assert_eq!( + TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should remain available") + .generation(), + old_generation + ); + let operations = backend.op_log().await; + assert!(matches!(operations.first(), Some(MockWarmOp::Put { .. }))); + assert!( + operations + .iter() + .any(|operation| matches!(operation, MockWarmOp::Remove { .. })) + ); + assert!(matches!(operations.last(), Some(MockWarmOp::Probe { .. }))); + } + #[tokio::test] async fn remove_and_clear_full_update_paths_preserve_force() { let remove_store = Arc::new(CasConfigStore::default()); @@ -13690,13 +15414,17 @@ mod tests { .await .expect("add fixture should persist"); - TierConfigMgr::update_candidate_with_config_lock( - &manager, - store.clone(), - TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true), - ) - .await - .expect("add proof must ignore unchanged durable tiers missing from the stale manager"); + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TierConfigMgr::update_candidate_with_config_lock( + &manager, + store.clone(), + TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true), + ), + ) + .await + .expect("add proof must ignore unchanged durable tiers missing from the stale manager"); let reloaded = load_tier_config_for_update(store) .await @@ -13706,6 +15434,84 @@ mod tests { assert!(reloaded.tiers.contains_key("COLD-B")); } + #[tokio::test] + async fn nested_only_legacy_add_builds_nonempty_coordinator_target_and_fans_out() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("empty tier config fixture should persist"); + let legacy_wire: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "s3", + "s3": { + "name": "COLD-LEGACY", + "endpoint": "https://s3.example.invalid", + "accessKey": "access", + "secretKey": "secret", + "bucket": "archive", + "prefix": "objects", + "region": "us-east-1", + "storageClass": "STANDARD" + } + })) + .expect("legacy RustFS nested-name AddTier payload should decode"); + assert!(legacy_wire.name.is_empty()); + let mutation = TierCandidateMutation::add(legacy_wire, true) + .expect("legacy nested name must normalize before constructing the Add mutation"); + assert_eq!(mutation.explicit_tier_name(), Some("COLD-LEGACY")); + + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let prepared_intents = Arc::new(Mutex::new(Vec::new())); + let backend = RecordingWarmBackend::new(); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed_capturing_prepare( + "peer-a", + peer_calls.clone(), + prepared_intents.clone(), + )], + TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation), + ), + ) + .await + .expect("legacy nested-name Add must run the full coordinator fanout"); + + let prepared_intents = lock_unpoisoned(&prepared_intents); + assert_eq!(prepared_intents.len(), 1); + assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add); + assert_eq!(prepared_intents[0].affected_targets.len(), 1); + assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY"); + assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none()); + assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some()); + drop(prepared_intents); + + let peer_calls = lock_unpoisoned(&peer_calls).clone(); + let prepare_index = peer_calls + .iter() + .position(|call| call.starts_with("peer-a:prepare:")) + .expect("legacy Add should prepare the peer"); + let commit_index = peer_calls + .iter() + .position(|call| call.starts_with("peer-a:commit:")) + .expect("legacy Add should commit the peer"); + assert!(prepare_index < commit_index, "{peer_calls:?}"); + + let reloaded = load_tier_config_for_update(store) + .await + .expect("legacy Add result should reload") + .0; + let persisted = reloaded + .tiers + .get("COLD-LEGACY") + .expect("normalized tier name should be persisted"); + assert_eq!(persisted.name, "COLD-LEGACY"); + assert_eq!(persisted.s3.as_ref().expect("persisted S3 payload should exist").name, "COLD-LEGACY"); + assert!(manager.read().await.tiers.contains_key("COLD-LEGACY")); + } + #[tokio::test] async fn config_update_refuses_pending_prepared_recovery_before_advancing_etag() { let manager = TierConfigMgr::new(); diff --git a/crates/ecstore/src/services/tier/tier_admin.rs b/crates/ecstore/src/services/tier/tier_admin.rs index 1b3519c02..32cd844a0 100644 --- a/crates/ecstore/src/services/tier/tier_admin.rs +++ b/crates/ecstore/src/services/tier/tier_admin.rs @@ -18,25 +18,203 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; -#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Clone)] +#[serde(default)] +pub struct TierServicePrincipalAuth { + #[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")] + pub tenant_id: String, + #[serde(rename = "ClientID", alias = "clientID", alias = "client_id")] + pub client_id: String, + #[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")] + pub client_secret: String, +} + +impl TierServicePrincipalAuth { + pub(crate) fn is_empty(&self) -> bool { + self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty() + } +} + +impl std::fmt::Debug for TierServicePrincipalAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TierServicePrincipalAuth") + .field("tenant_id", &self.tenant_id) + .field("client_id", &self.client_id) + .field("client_secret", &"REDACTED") + .finish() + } +} + +#[derive(Serialize, Deserialize, Default, Clone)] #[serde(default)] pub struct TierCreds { - #[serde(rename = "accessKey")] + #[serde(rename = "access", alias = "accessKey")] pub access_key: String, - #[serde(rename = "secretKey")] + #[serde(rename = "secret", alias = "secretKey")] pub secret_key: String, - #[serde(rename = "awsRole")] + #[serde(rename = "awsrole", alias = "awsRole")] pub aws_role: bool, - #[serde(rename = "awsRoleWebIdentityTokenFile")] + #[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")] pub aws_role_web_identity_token_file: String, - #[serde(rename = "awsRoleArn")] + #[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")] pub aws_role_arn: String, - //azsp: ServicePrincipalAuth, + #[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")] + pub azure_service_principal: TierServicePrincipalAuth, - //#[serde(rename = "credsJson")] + #[serde( + rename = "creds", + alias = "credsJson", + alias = "credsJSON", + alias = "creds_json", + default, + skip_serializing_if = "Vec::is_empty", + with = "base64_bytes" + )] pub creds_json: Vec, } + +impl std::fmt::Debug for TierCreds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TierCreds") + .field("access_key", &self.access_key) + .field("secret_key", &"REDACTED") + .field("aws_role", &self.aws_role) + .field( + "aws_role_web_identity_token_file", + &(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"), + ) + .field("aws_role_arn", &self.aws_role_arn) + .field("azure_service_principal", &self.azure_service_principal) + .field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED")) + .finish() + } +} + +mod base64_bytes { + use super::*; + + pub(super) fn serialize(value: &[u8], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value)) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum EncodedBytes { + Base64(String), + Legacy(Vec), + } + + match EncodedBytes::deserialize(deserializer)? { + EncodedBytes::Base64(value) => base64_simd::STANDARD + .decode_to_vec(value.as_bytes()) + .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes())) + .map_err(de::Error::custom), + EncodedBytes::Legacy(value) => Ok(value), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() { + let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes(); + let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ=="; + let creds: TierCreds = serde_json::from_value(serde_json::json!({ + "access": "access", + "secret": "secret", + "awsrole": false, + "creds": encoded, + })) + .expect("madmin tier credentials should decode"); + + assert_eq!(creds.access_key, "access"); + assert_eq!(creds.secret_key, "secret"); + assert_eq!(creds.creds_json.as_slice(), &service_account[..]); + + let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode"); + assert_eq!(wire["access"], "access"); + assert_eq!(wire["secret"], "secret"); + assert_eq!(wire["creds"], encoded); + assert!(wire.get("accessKey").is_none()); + assert!(wire.get("secretKey").is_none()); + + let legacy: TierCreds = serde_json::from_value(serde_json::json!({ + "accessKey": "legacy-access", + "secretKey": "legacy-secret", + "credsJson": service_account, + })) + .expect("the former RustFS field names and byte-array encoding should remain readable"); + assert_eq!(legacy.access_key, "legacy-access"); + assert_eq!(legacy.secret_key, "legacy-secret"); + assert_eq!(legacy.creds_json.as_slice(), &service_account[..]); + } + + #[test] + fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() { + let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes(); + for encoder in [ + base64_simd::STANDARD, + base64_simd::STANDARD_NO_PAD, + base64_simd::URL_SAFE, + base64_simd::URL_SAFE_NO_PAD, + ] { + let encoded = encoder.encode_to_string(service_account); + let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded })) + .expect("all supported madmin base64 forms should decode"); + assert_eq!(creds.creds_json, service_account); + } + } + + #[test] + fn tier_creds_debug_redacts_secret_payloads() { + let creds = TierCreds { + access_key: "access".to_string(), + secret_key: "tier-secret-value".to_string(), + aws_role_web_identity_token_file: "/var/run/private-token".to_string(), + creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(), + ..Default::default() + }; + + let rendered = format!("{creds:?}"); + assert!(!rendered.contains("tier-secret-value")); + assert!(!rendered.contains("/var/run/private-token")); + assert!(!rendered.contains("gcs-private-key-value")); + } + + #[test] + fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() { + let creds: TierCreds = serde_json::from_value(serde_json::json!({ + "azSP": { + "TenantID": "tenant", + "ClientID": "client", + "ClientSecret": "service-principal-secret" + } + })) + .expect("canonical madmin azure service principal credentials should decode"); + + assert_eq!(creds.azure_service_principal.tenant_id, "tenant"); + assert_eq!(creds.azure_service_principal.client_id, "client"); + assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret"); + let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode"); + assert_eq!(wire["azSP"]["TenantID"], "tenant"); + assert_eq!(wire["azSP"]["ClientID"], "client"); + assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret"); + assert!(!format!("{creds:?}").contains("service-principal-secret")); + } +} diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index 94fe4e408..3d4833727 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[ pub enum TierType { #[default] Unsupported, - #[serde(rename = "s3")] + #[serde(rename = "s3", alias = "S3")] S3, #[serde(rename = "wasabi")] Wasabi, @@ -58,7 +58,7 @@ pub enum TierType { Huaweicloud, #[serde(rename = "azure")] Azure, - #[serde(rename = "gcs")] + #[serde(rename = "gcs", alias = "GCS")] GCS, #[serde(rename = "r2")] R2, @@ -138,16 +138,18 @@ impl TierType { } } -#[derive(Default, Debug, Serialize, Deserialize)] +pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED"; + +#[derive(Default, Serialize, Deserialize)] #[serde(default)] pub struct TierConfig { #[serde(skip)] pub version: String, - #[serde(rename = "type")] + #[serde(rename = "type", alias = "Type")] pub tier_type: TierType, - #[serde(skip)] + #[serde(rename = "Name", alias = "name", skip_serializing)] pub name: String, - #[serde(rename = "s3", skip_serializing_if = "Option::is_none")] + #[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")] pub s3: Option, #[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")] pub wasabi: Option, @@ -159,7 +161,7 @@ pub struct TierConfig { pub huaweicloud: Option, #[serde(rename = "azure", skip_serializing_if = "Option::is_none")] pub azure: Option, - #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")] + #[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")] pub gcs: Option, #[serde(rename = "r2", skip_serializing_if = "Option::is_none")] pub r2: Option, @@ -170,109 +172,91 @@ pub struct TierConfig { } impl Clone for TierConfig { - fn clone(&self) -> TierConfig { - let mut s3 = None; - let mut wasabi = None; - let mut r = None; - let mut compatible_backend = None; - let mut aliyun = None; - let mut tencent = None; - let mut huaweicloud = None; - let mut azure = None; - let mut gcs = None; - let mut r2 = None; - match self.tier_type { - TierType::S3 => { - if let Some(s3_) = self.s3.as_ref() { - let mut s3_clone = s3_.clone(); - s3_clone.secret_key = "REDACTED".to_string(); - s3 = Some(s3_clone); - } - } - TierType::Wasabi => { - if let Some(wasabi_) = self.wasabi.as_ref() { - let mut wasabi_clone = wasabi_.clone(); - wasabi_clone.secret_key = "REDACTED".to_string(); - wasabi = Some(wasabi_clone); - } - } - TierType::RustFS => { - if let Some(r_) = self.rustfs.as_ref() { - let mut r_clone = r_.clone(); - r_clone.secret_key = "REDACTED".to_string(); - r = Some(r_clone); - } - } - TierType::MinIO => { - if let Some(compatible_backend_) = self.minio.as_ref() { - let mut compatible_backend_clone = compatible_backend_.clone(); - compatible_backend_clone.secret_key = "REDACTED".to_string(); - compatible_backend = Some(compatible_backend_clone); - } - } - TierType::Aliyun => { - if let Some(aliyun_) = self.aliyun.as_ref() { - let mut aliyun_clone = aliyun_.clone(); - aliyun_clone.secret_key = "REDACTED".to_string(); - aliyun = Some(aliyun_clone); - } - } - TierType::Tencent => { - if let Some(tencent_) = self.tencent.as_ref() { - let mut tencent_clone = tencent_.clone(); - tencent_clone.secret_key = "REDACTED".to_string(); - tencent = Some(tencent_clone); - } - } - TierType::Huaweicloud => { - if let Some(huaweicloud_) = self.huaweicloud.as_ref() { - let mut huaweicloud_clone = huaweicloud_.clone(); - huaweicloud_clone.secret_key = "REDACTED".to_string(); - huaweicloud = Some(huaweicloud_clone); - } - } - TierType::Azure => { - if let Some(azure_) = self.azure.as_ref() { - let mut azure_clone = azure_.clone(); - azure_clone.secret_key = "REDACTED".to_string(); - azure = Some(azure_clone); - } - } - TierType::GCS => { - if let Some(gcs_) = self.gcs.as_ref() { - let mut gcs_clone = gcs_.clone(); - gcs_clone.creds = "REDACTED".to_string(); - gcs = Some(gcs_clone); - } - } - TierType::R2 => { - if let Some(r2_) = self.r2.as_ref() { - let mut r2_clone = r2_.clone(); - r2_clone.secret_key = "REDACTED".to_string(); - r2 = Some(r2_clone); - } - } - _ => (), - } - TierConfig { - version: self.version.clone(), - tier_type: self.tier_type.clone(), - name: self.name.clone(), - s3, - wasabi, - rustfs: r, - minio: compatible_backend, - aliyun, - tencent, - huaweicloud, - azure, - gcs, - r2, - } + fn clone(&self) -> Self { + self.redacted() } } impl TierConfig { + pub(crate) fn redacted(&self) -> Self { + let mut redacted = Self { + version: self.version.clone(), + tier_type: self.tier_type.clone(), + name: self.name.clone(), + ..Default::default() + }; + match self.tier_type { + TierType::S3 => { + redacted.s3 = self.s3.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + if !backend.aws_role_web_identity_token_file.is_empty() { + backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string(); + } + backend + }); + } + TierType::Wasabi => { + redacted.wasabi = self.wasabi.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::RustFS => { + redacted.rustfs = self.rustfs.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::MinIO => { + redacted.minio = self.minio.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Aliyun => { + redacted.aliyun = self.aliyun.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Tencent => { + redacted.tencent = self.tencent.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Huaweicloud => { + redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Azure => { + redacted.azure = self.azure.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + if !backend.sp_auth.client_secret.is_empty() { + backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string(); + } + backend + }); + } + TierType::GCS => { + redacted.gcs = self.gcs.clone().map(|mut backend| { + backend.creds = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::R2 => { + redacted.r2 = self.r2.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Unsupported => {} + } + redacted + } + pub(crate) fn clone_with_credentials(&self) -> Self { Self { version: self.version.clone(), @@ -372,31 +356,61 @@ impl TierConfig { } } +impl std::fmt::Debug for TierConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let redacted = self.redacted(); + f.debug_struct("TierConfig") + .field("version", &redacted.version) + .field("tier_type", &redacted.tier_type) + .field("name", &redacted.name) + .field("s3", &redacted.s3) + .field("wasabi", &redacted.wasabi) + .field("aliyun", &redacted.aliyun) + .field("tencent", &redacted.tencent) + .field("huaweicloud", &redacted.huaweicloud) + .field("azure", &redacted.azure) + .field("gcs", &redacted.gcs) + .field("r2", &redacted.r2) + .field("rustfs", &redacted.rustfs) + .field("minio", &redacted.minio) + .finish() + } +} + //type S3Options = impl Fn(TierS3) -> Pin>> + Send + Sync + 'static; #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierS3 { + #[serde(alias = "Name")] pub name: String, + #[serde(alias = "Endpoint")] pub endpoint: String, - #[serde(rename = "accessKey")] + #[serde(rename = "accessKey", alias = "AccessKey")] pub access_key: String, - #[serde(rename = "secretKey")] + #[serde(rename = "secretKey", alias = "SecretKey")] pub secret_key: String, + #[serde(alias = "Bucket")] pub bucket: String, + #[serde(alias = "Prefix")] pub prefix: String, + #[serde(alias = "Region")] pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, - #[serde(skip)] + #[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)] pub aws_role: bool, - #[serde(skip)] + #[serde( + rename = "AWSRoleWebIdentityTokenFile", + alias = "awsRoleWebIdentityTokenFile", + skip_serializing + )] pub aws_role_web_identity_token_file: String, - #[serde(skip)] + #[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)] pub aws_role_arn: String, - #[serde(skip)] + #[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)] pub aws_role_session_name: String, - #[serde(skip)] + #[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)] pub aws_role_duration_seconds: i32, } @@ -623,8 +637,11 @@ pub struct TierHuaweicloud { #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct ServicePrincipalAuth { + #[serde(alias = "TenantID")] pub tenant_id: String, + #[serde(alias = "ClientID")] pub client_id: String, + #[serde(alias = "ClientSecret")] pub client_secret: String, } @@ -640,9 +657,9 @@ pub struct TierAzure { pub bucket: String, pub prefix: String, pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, - #[serde(rename = "spAuth")] + #[serde(rename = "spAuth", alias = "SPAuth")] pub sp_auth: ServicePrincipalAuth, } @@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error { #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierGCS { + #[serde(alias = "Name")] pub name: String, + #[serde(alias = "Endpoint")] pub endpoint: String, - #[serde(rename = "creds")] + #[serde(rename = "creds", alias = "Creds")] pub creds: String, + #[serde(alias = "Bucket")] pub bucket: String, + #[serde(alias = "Prefix")] pub prefix: String, + #[serde(alias = "Region")] pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, } @@ -725,6 +747,43 @@ pub struct TierR2 { mod tests { use super::*; + #[test] + fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() { + let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode"); + let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode"); + assert!(matches!(s3, TierType::S3)); + assert!(matches!(gcs, TierType::GCS)); + assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#); + assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#); + } + + #[test] + fn azure_service_principal_accepts_canonical_madmin_field_names() { + for field in ["TenantID", "ClientID", "ClientSecret"] { + let mut sp_auth = serde_json::Map::new(); + sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string())); + let config: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "azure", + "Name": "COLD-AZURE", + "azure": { + "name": "COLD-AZURE", + "endpoint": "https://azure.example.invalid", + "accessKey": "account", + "secretKey": "key", + "bucket": "archive", + "SPAuth": sp_auth + } + })) + .expect("mixed RustFS/madmin Azure payload should decode"); + let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth; + + assert!( + !sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(), + "canonical {field} must not be silently discarded" + ); + } + } + fn wasabi_config() -> TierWasabi { TierWasabi { name: "COLD-WASABI".to_string(), @@ -838,9 +897,14 @@ mod tests { let config = TierConfig { tier_type: TierType::Wasabi, wasabi: Some(wasabi_config()), + rustfs: Some(TierRustFS { + access_key: "inactive-access".to_string(), + secret_key: "inactive-secret".to_string(), + ..Default::default() + }), ..Default::default() }; - let redacted = config.clone(); + let redacted = config.redacted(); assert_eq!( redacted .wasabi @@ -849,21 +913,46 @@ mod tests { .secret_key, "REDACTED" ); + assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider"); + let cloned = config.clone(); + assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED"); + assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics"); + let preserved = config.clone_with_credentials(); assert_eq!( - config - .clone_with_credentials() + preserved .wasabi .as_ref() - .expect("credential-bearing Wasabi payload should remain") + .expect("credential-bearing Wasabi snapshot should remain") .secret_key, "secret" ); + assert_eq!( + preserved + .rustfs + .expect("credential-bearing snapshots should preserve inactive provider data") + .secret_key, + "inactive-secret" + ); let mut debug_config = wasabi_config(); debug_config.secret_key = "wasabi-debug-secret-value".to_string(); let debug = format!("{debug_config:?}"); assert!(debug.contains("REDACTED")); assert!(!debug.contains("wasabi-debug-secret-value")); + + let debug = format!( + "{:?}", + TierConfig { + tier_type: TierType::RustFS, + rustfs: Some(TierRustFS { + secret_key: "rustfs-debug-secret-value".to_string(), + ..Default::default() + }), + ..Default::default() + } + ); + assert!(debug.contains("REDACTED")); + assert!(!debug.contains("rustfs-debug-secret-value")); } #[test] @@ -894,7 +983,7 @@ mod tests { assert_eq!(encoded, expected); let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode"); assert!(matches!(decoded.tier_type, TierType::Wasabi)); - let redacted = config.clone(); + let redacted = config.redacted(); assert_eq!( config .wasabi @@ -916,4 +1005,87 @@ mod tests { "REDACTED" ); } + + #[test] + fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() { + let cases = [ + ( + "s3", + TierConfig { + tier_type: TierType::S3, + s3: Some(TierS3 { + secret_key: "s3-secret-bytes".to_string(), + aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(), + ..Default::default() + }), + ..Default::default() + }, + vec!["s3-secret-bytes", "/var/run/s3-private-token"], + ), + ( + "gcs", + TierConfig { + tier_type: TierType::GCS, + gcs: Some(TierGCS { + creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(), + ..Default::default() + }), + ..Default::default() + }, + vec!["gcs-private-key-bytes"], + ), + ( + "azure", + TierConfig { + tier_type: TierType::Azure, + azure: Some(TierAzure { + secret_key: "azure-account-secret-bytes".to_string(), + sp_auth: ServicePrincipalAuth { + client_secret: "azure-client-secret-bytes".to_string(), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }, + vec!["azure-account-secret-bytes", "azure-client-secret-bytes"], + ), + ]; + + for (provider, config, secrets) in cases { + let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize"); + let debug = format!("{config:?}"); + assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted"); + assert!( + debug.contains(TIER_CREDENTIAL_REDACTED), + "{provider} Debug output should be visibly redacted" + ); + for secret in secrets { + assert!(!api.contains(secret), "{provider} API output exposed credential bytes"); + assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes"); + } + } + } + + #[test] + fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() { + let config = TierConfig { + tier_type: TierType::Azure, + azure: Some(TierAzure { + secret_key: "azure-account-secret-bytes".to_string(), + sp_auth: ServicePrincipalAuth::default(), + ..Default::default() + }), + ..Default::default() + }; + + let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize"); + let debug = format!("{config:?}"); + + assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED); + assert_eq!(api["azure"]["spAuth"]["client_secret"], ""); + assert!(debug.contains("client_secret: \"\"")); + assert!(!debug.contains("client_secret: \"REDACTED\"")); + assert!(!debug.contains("azure-account-secret-bytes")); + } } diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 8eb6e4058..ee48c116c 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -20,7 +20,7 @@ use crate::error::is_err_bucket_not_found; use crate::services::tier::{ - tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, + tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR}, warm_backend_aliyun::WarmBackendAliyun, @@ -55,18 +55,21 @@ use s3s::header::{ }; use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use time::OffsetDateTime; use time::format_description::well_known::{Rfc2822, Rfc3339}; +use tokio::io::AsyncReadExt; use tracing::{info, warn}; pub type WarmBackendImpl = Box; -const PROBE_OBJECT: &str = "probeobject"; - /// Largest object the S3-compatible warm backends accept for a multipart put. pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; /// Part-count ceiling S3-compatible services impose on a multipart upload. pub(crate) const MAX_PARTS_COUNT: i64 = 10000; +pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30); +const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1); +const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1); #[derive(Default)] pub struct WarmBackendGetOpts { @@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> { pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>, } +/// Return the authority format accepted by `TransitionClient::new` while +/// retaining an explicitly configured port. `url::Url::host_str()` omits the +/// brackets needed when an IPv6 literal is combined with a port. +pub(crate) fn endpoint_authority(url: &url::Url) -> Result { + let host = url + .host_str() + .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; + let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 }); + if host.starts_with('[') && host.ends_with(']') { + Ok(format!("{host}:{port}")) + } else if host.contains(':') { + Ok(format!("[{host}]:{port}")) + } else { + Ok(format!("{host}:{port}")) + } +} + /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers. /// /// Credential, bucket, and endpoint validation run in this order because the @@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend( bucket_lookup: params.bucket_lookup, ..Default::default() }; - let scheme = u.scheme(); - let default_port = if scheme == "https" { 443 } else { 80 }; - let host = u - .host_str() - .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; - // Runs after the host-presence check above (not immediately after Url::parse) so a - // host-less endpoint still reports this constructor's own "missing host" text instead of - // validate_endpoint's differently-worded rejection for the same input. + let endpoint = endpoint_authority(&u)?; + // Run the SSRF guard after the host-presence check so a host-less endpoint + // keeps this constructor's stable error text. (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; - let client = - TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?; + let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); @@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler { } } -pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { - let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; - w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?; - let remote_version_id = w - .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5) - .await - .map_err(|_| ERR_TIER_PERM_ERR.clone())?; +async fn remove_discovered_probe_candidate( + w: &WarmBackendImpl, + probe_object: &str, + candidate: TransitionCandidateProbe, +) -> Result { + match candidate { + TransitionCandidateProbe::Missing => Ok(false), + TransitionCandidateProbe::VersionedPresent(remote_version_id) => { + w.remove_exact(probe_object, &remote_version_id).await?; + Ok(true) + } + TransitionCandidateProbe::UnversionedPresent => { + w.remove(probe_object, "").await?; + Ok(true) + } + TransitionCandidateProbe::Ambiguous => { + Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions")) + } + TransitionCandidateProbe::Unsupported => Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot discover the outcome of a probe PUT", + )), + } +} - if w.validate_remote_version_id(&remote_version_id).is_err() { - w.remove_exact(PROBE_OBJECT, &remote_version_id) +async fn compensate_uncertain_probe_put( + w: &WarmBackendImpl, + probe_object: &str, + settle_deadline: tokio::time::Instant, +) -> Result<(), std::io::Error> { + let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT; + let mut removed_any = false; + while tokio::time::Instant::now() < settle_deadline { + let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await { + Ok(candidate) => candidate?, + Err(_) => break, + }; + if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any { + break; + } + removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate)) .await - .map_err(|_| ERR_TIER_PERM_ERR.clone())?; - return Err(ERR_TIER_INVALID_CONFIG.clone()); + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??; + + let now = tokio::time::Instant::now(); + if now >= settle_deadline { + break; + } + tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await; } - let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await; - let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await; - //xhttp.DrainBody(r); - if read_result.is_err() || remove_result.is_err() { + let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object)) + .await + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state") + })??; + if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate)) + .await + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate") + })?? + { + return Ok(()); + } + + let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??; + match final_candidate { + TransitionCandidateProbe::Missing => Ok(()), + _ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")), + } +} + +fn probe_cleanup_incomplete_error() -> AdminError { + let mut err = ERR_TIER_PERM_ERR.clone(); + err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string(); + err +} + +async fn check_warm_backend_with_deadlines( + w: Option<&WarmBackendImpl>, + deadline: tokio::time::Instant, + cleanup_deadline: tokio::time::Instant, +) -> Result<(), AdminError> { + let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; + let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4()); + let timeout_error = || { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out validating the remote tier mutation".to_string(); + err + }; + tokio::time::timeout_at(deadline, w.validate()) + .await + .map_err(|_| timeout_error())? + .map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?; + let put_result = + tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await; + let remote_version_id = match put_result { + Ok(Ok(remote_version_id)) => remote_version_id, + Ok(Err(_)) => { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_PERM_ERR.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + Err(_) => { + let err = timeout_error(); + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => err, + Err(_) => probe_cleanup_incomplete_error(), + }); + } + }; + + // S3-family backends do not replay a failed request before returning `Ok`, + // while GCS discovers every matching generation. The authoritative probe + // below therefore closes the acknowledged-PUT path; only an error or + // timeout needs the longer visibility reconciliation above. + let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await { + Ok(Ok(candidate)) => candidate, + Ok(Err(_)) | Err(_) => { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_INVALID_CONFIG.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + }; + let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok(); + let response_matches_candidate = match &authoritative_candidate { + TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(), + TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id, + TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false, + }; + if !response_version_is_valid || !response_matches_candidate { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_INVALID_CONFIG.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + + let read_result = tokio::time::timeout_at(deadline, async { + let mut reader = w + .get( + &probe_object, + &remote_version_id, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + .map_err(|_| ERR_TIER_PERM_ERR.clone())?; + let mut body = Vec::new(); + reader + .take(7) + .read_to_end(&mut body) + .await + .map_err(|_| ERR_TIER_PERM_ERR.clone())?; + if body != b"RustFS" { + return Err(ERR_TIER_PERM_ERR.clone()); + } + Ok(()) + }) + .await + .map_err(|_| timeout_error()) + .and_then(|result| result); + let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async { + if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? { + return Err(std::io::Error::other("remote tier probe disappeared before cleanup")); + } + match w.probe_transition_candidate(&probe_object).await? { + TransitionCandidateProbe::Missing => Ok(()), + _ => Err(std::io::Error::other("remote tier probe remained after cleanup")), + } + }) + .await; + if !matches!(cleanup_result, Ok(Ok(()))) { + return Err(probe_cleanup_incomplete_error()); + } + if let Err(err) = read_result { //if is_err_bucket_not_found(&err) { // return Err(ERR_TIER_BUCKET_NOT_FOUND); //} @@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin return Err(ERR_TIER_MISSING_CREDENTIALS); }*/ //else { - return Err(ERR_TIER_PERM_ERR.clone()); + return Err(err); //} } Ok(()) } +/// Validate a backend using a caller-owned deadline while retaining a bounded +/// reconciliation window for an uncertain probe PUT. The validation future is +/// kept alive through cleanup so an outer timeout cannot abandon the remote +/// probe object. +pub(crate) async fn check_warm_backend_until( + w: Option<&WarmBackendImpl>, + deadline: tokio::time::Instant, +) -> Result<(), AdminError> { + check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await +} + +pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { + let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT; + check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await +} + pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { let mut d: Option = None; match tier.tier_type { @@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result, + present: Arc, removed_versions: Arc>>, + returned_version: String, } struct RecordingProbeBackend { get_versions: Arc>>, + present: Arc, removed_versions: Arc>>, + remove_clears_candidate: bool, fail_get: bool, + body: ProbeBody, + } + + struct HangingProbePutBackend { + put_started: Arc, + present: Arc, + probes: Arc, + removed_versions: Arc>>, + } + + struct LateVisibleProbeBackend { + visible_at: tokio::time::Instant, + removed: Arc, + probes: Arc, + removed_versions: Arc>>, + } + + #[derive(Clone, Copy)] + enum ProbeBody { + Exact, + Mismatch, } #[async_trait::async_trait] @@ -976,7 +1194,7 @@ mod tests { } async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { - Ok(uuid::Uuid::nil().to_string()) + Ok(self.returned_version.clone()) } async fn put_with_meta( @@ -999,10 +1217,19 @@ mod tests { } async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.present.store(false, Ordering::SeqCst); self.removed_versions.lock().await.push(rv.to_string()); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> Result { + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> Result { Ok(false) } @@ -1029,15 +1256,121 @@ mod tests { if self.fail_get { Err(std::io::Error::other("probe GET failed")) } else { - Ok(ReadCloser::new(std::io::Cursor::new(Vec::new()))) + match self.body { + ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))), + ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))), + } } } async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + if self.remove_clears_candidate { + self.present.store(false, Ordering::SeqCst); + } self.removed_versions.lock().await.push(rv.to_string()); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> Result { + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> Result { + Ok(false) + } + } + + #[async_trait::async_trait] + impl WarmBackend for HangingProbePutBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { + self.put_started.notify_one(); + std::future::pending().await + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result { + Err(std::io::Error::other("GET must not run after a timed out probe PUT")) + } + + async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> { + Err(std::io::Error::other("generic remove must not replace exact probe cleanup")) + } + + async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.present.store(false, Ordering::SeqCst); + self.removed_versions.lock().await.push(rv.to_string()); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> Result { + self.probes.fetch_add(1, Ordering::SeqCst); + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> Result { + Ok(false) + } + } + + #[async_trait::async_trait] + impl WarmBackend for LateVisibleProbeBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "probe PUT response was lost before the object became visible", + )) + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result { + Err(std::io::Error::other("GET must not run after a lost probe PUT response")) + } + + async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> { + Err(std::io::Error::other("generic remove must not replace exact probe cleanup")) + } + + async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.removed.store(true, Ordering::SeqCst); + self.removed_versions.lock().await.push(rv.to_string()); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> Result { + self.probes.fetch_add(1, Ordering::SeqCst); + if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> Result { Ok(false) } @@ -1098,13 +1431,15 @@ mod tests { assert_eq!(probe, TransitionCandidateProbe::Unsupported); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() { let gets = Arc::new(AtomicUsize::new(0)); let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend { gets: gets.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + returned_version: uuid::Uuid::nil().to_string(), }); let err = check_warm_backend(Some(&backend)) @@ -1113,7 +1448,27 @@ mod tests { assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); assert_eq!(gets.load(Ordering::SeqCst), 0); - assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() { + let gets = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend { + gets: gets.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + returned_version: String::new(), + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("an empty PUT version must not read or generically delete a versioned object"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert_eq!(gets.load(Ordering::SeqCst), 0); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); } #[tokio::test] @@ -1122,8 +1477,11 @@ mod tests { let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { get_versions: get_versions.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + remove_clears_candidate: true, fail_get: false, + body: ProbeBody::Exact, }); check_warm_backend(Some(&backend)) @@ -1140,8 +1498,11 @@ mod tests { let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { get_versions: get_versions.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + remove_clears_candidate: true, fail_get: true, + body: ProbeBody::Exact, }); let err = check_warm_backend(Some(&backend)) @@ -1153,6 +1514,169 @@ mod tests { assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); } + #[tokio::test] + async fn check_warm_backend_removes_probe_after_body_mismatch() { + let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { + get_versions, + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + remove_clears_candidate: true, + fail_get: false, + body: ProbeBody::Mismatch, + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("a mismatched body should fail after cleanup"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test] + async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() { + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { + get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + remove_clears_candidate: false, + fail_get: false, + body: ProbeBody::Exact, + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("cleanup must not succeed while the deleted candidate remains visible"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_reconciles_a_lost_put_response() { + let backend = MockWarmBackend::new(); + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("a lost probe PUT response must fail after compensation"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.exact_remove_count(), 1); + let operations = backend.op_log().await; + let put = operations.iter().find_map(|operation| match operation { + MockWarmOp::Put { object } => Some(object), + _ => None, + }); + let probe = operations.iter().find_map(|operation| match operation { + MockWarmOp::Probe { object } => Some(object), + _ => None, + }); + let remove = operations.iter().find_map(|operation| match operation { + MockWarmOp::Remove { object } => Some(object), + _ => None, + }); + let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else { + panic!("lost-response compensation should PUT, probe, and remove"); + }; + assert_eq!(put, probe); + assert_eq!(probe, remove); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_retries_until_a_late_put_becomes_visible() { + let probes = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend { + visible_at: tokio::time::Instant::now() + Duration::from_secs(5), + removed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + probes: probes.clone(), + removed_versions: removed_versions.clone(), + }); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("a late-visible probe PUT must still report the lost response"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!( + probes.load(Ordering::SeqCst) > 5, + "reconciliation must not stop at the first Missing result" + ); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test] + async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() { + for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] { + let backend = MockWarmBackend::new(); + backend.set_transition_candidate_probe_override(Some(candidate)).await; + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("an uncertain candidate must fail without a guessed delete"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(backend.remove_count().await, 0); + assert_eq!(backend.object_count().await, 1); + } + } + + #[tokio::test] + async fn check_warm_backend_reports_an_exact_cleanup_failure() { + let backend = MockWarmBackend::new(); + backend.set_remove_failure(true); + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("an exact cleanup failure must replace the ambiguous PUT error"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(backend.exact_remove_count(), 1); + assert_eq!(backend.object_count().await, 1); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_reconciles_a_timed_out_put() { + let put_started = Arc::new(tokio::sync::Notify::new()); + let probes = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let driver: WarmBackendImpl = Box::new(HangingProbePutBackend { + put_started: put_started.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + probes: probes.clone(), + removed_versions: removed_versions.clone(), + }); + let check = check_warm_backend(Some(&driver)); + tokio::pin!(check); + tokio::select! { + _ = put_started.notified() => {} + result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"), + } + + tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await; + let err = check.await.expect_err("a timed out probe PUT must fail after compensation"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!( + probes.load(Ordering::SeqCst) > 1, + "timed-out PUT reconciliation must keep checking through the visibility window" + ); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + #[tokio::test] async fn new_wasabi_backend_honors_probe_flag() { let tier = TierConfig { @@ -1298,6 +1822,15 @@ mod tests { assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80)); } + #[test] + fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() { + let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse"); + assert_eq!( + endpoint_authority(&url).expect("the endpoint should have an authority"), + "[2001:db8::1]:9443" + ); + } + #[tokio::test] async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() { let mut params = s3_compatible_params("http://tier.example.com:9000"); diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index e848f4f29..27fb7decd 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierAliyun, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index 7b3723eac..ed9ef0cf8 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierAzure, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + // Azure currently uses the shared S3/SigV4 transport, but its normal + // object path cannot persist exact remote versions across mixed + // RustFS releases. The mutation probe may still detect and precisely + // remove a versioned test object before rejecting that configuration. + self.0 + .probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id") + .await + } + async fn in_use(&self) -> Result { self.0.in_use().await } @@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure { mod tests { use super::*; use crate::services::tier::tier_config::TierAzure; + use rustfs_s3_client::{ + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, TransitionClient, TransitionCore}, + }; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String { + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut buffer).await.expect("fixture request should be readable"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + } + String::from_utf8_lossy(&request).into_owned() + } /// The SSRF guard itself is exercised once, generically, in /// `warm_backend::tests` (see backlog#2040/backlog#2041 and @@ -119,4 +146,85 @@ mod tests { Err(err) => assert!(err.to_string().contains("not allowed")), } } + + #[tokio::test] + async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET"); + let get_request = read_request_head(&mut get_stream).await; + get_stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx", + ) + .await + .expect("fixture should write candidate GET response"); + + let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE"); + let delete_request = read_request_head(&mut delete_stream).await; + delete_stream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("fixture should write exact DELETE response"); + (get_request, delete_request) + }); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "azure", + ) + .await + .expect("fixture client should build"), + ); + let backend = WarmBackendAzure(WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }); + assert!( + !backend.0.client.provider_version_capabilities().exact_get_delete, + "probe-only version discovery must not change Azure's persisted data-path contract" + ); + + let candidate = backend + .probe_transition_candidate("probe") + .await + .expect("Azure candidate should be discovered"); + assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string())); + backend + .remove_exact("probe", "azure-version") + .await + .expect("Azure candidate should be deleted by exact version"); + + let (get_request, delete_request) = fixture.await.expect("fixture should join"); + assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n")); + assert!( + delete_request + .lines() + .next() + .is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version ")) + ); + } } diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 63c9a62c1..03c2df988 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -18,13 +18,13 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::io::{Error, ErrorKind}; use std::sync::Arc; use bytes::Bytes; -use google_cloud_auth::credentials::Credentials; -use google_cloud_auth::credentials::user_account::Builder; +use google_cloud_auth::credentials::service_account::Builder; use google_cloud_storage as gcs; use google_cloud_storage::client::Storage; use google_cloud_storage::client::StorageControl; @@ -32,7 +32,7 @@ use std::convert::TryFrom; use crate::services::tier::{ tier_config::TierGCS, - warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}, }; use rustfs_s3_client::{ admin_handler_utils::AdminError, @@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MAX_GCS_CANDIDATE_PAGES: usize = 64; fn parse_generation(remote_version: &str) -> Result, Error> { if remote_version.is_empty() { @@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result, Error> { Ok(Some(generation)) } +fn append_gcs_chunk( + contents: &mut Vec, + chunk: Result, + max_response_bytes: Option, +) -> std::io::Result<()> { + let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?; + if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS object response exceeded the configured byte limit", + )); + } + contents.extend_from_slice(&chunk); + Ok(()) +} + +fn gcs_bucket_resource_name(bucket: &str) -> String { + format!("projects/_/buckets/{bucket}") +} + +struct GcsCandidateObject { + name: String, + generation: i64, +} + +struct GcsCandidatePage { + objects: Vec, + next_page_token: String, +} + +async fn probe_exact_gcs_candidate( + remote_object: &str, + mut fetch_page: F, +) -> Result +where + F: FnMut(String) -> Fut, + Fut: Future>, +{ + let mut page_token = String::new(); + let mut seen_page_tokens = HashSet::new(); + let mut generation = None; + let mut pages_seen = 0_usize; + + loop { + pages_seen += 1; + if pages_seen > MAX_GCS_CANDIDATE_PAGES { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS candidate listing exceeded the page limit", + )); + } + let response = fetch_page(page_token.clone()).await?; + for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) { + if candidate.generation <= 0 { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS candidate listing returned a non-positive generation", + )); + } + if generation.replace(candidate.generation).is_some() { + return Ok(TransitionCandidateProbe::Ambiguous); + } + } + + if response.next_page_token.is_empty() { + break; + } + if !seen_page_tokens.insert(response.next_page_token.clone()) { + return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token")); + } + page_token = response.next_page_token; + } + + Ok(match generation { + Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()), + None => TransitionCandidateProbe::Missing, + }) +} + pub struct WarmBackendGCS { pub client: Arc, pub control: Arc, @@ -80,8 +160,8 @@ impl WarmBackendGCS { .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; } - let authorized_user = serde_json::from_str(&conf.creds)?; - let credentials = Builder::new(authorized_user) + let service_account = serde_json::from_str(&conf.creds)?; + let credentials = Builder::new(service_account) //.with_retry_policy(AlwaysRetry.with_attempt_limit(3)) //.with_backoff_policy(backoff) .build() @@ -98,7 +178,11 @@ impl WarmBackendGCS { let client = Arc::new(client); // Control-plane client: the data-plane `Storage` client cannot delete or list objects; // delete_object/list_objects live on StorageControl. - let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else { + let mut control_builder = StorageControl::builder().with_credentials(credentials); + if !conf.endpoint.is_empty() { + control_builder = control_builder.with_endpoint(conf.endpoint.clone()); + } + let Ok(control) = control_builder.build().await else { return Err(std::io::Error::other("StorageControl::builder error")); }; let control = Arc::new(control); @@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS { ReaderImpl::Body(content_body) => content_body.to_vec(), ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?, }; + let bucket = gcs_bucket_resource_name(&self.bucket); let Ok(res) = Box::pin( self.client - .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d)) + .write_object(&bucket, &self.get_dest(object), Bytes::from(d)) .send_buffered(), ) .await @@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS { } async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { - let mut req = self.client.read_object(&self.bucket, &self.get_dest(object)); + let bucket = gcs_bucket_resource_name(&self.bucket); + let mut req = self.client.read_object(&bucket, &self.get_dest(object)); + let mut max_response_bytes = None; if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); } @@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS { .length .try_into() .map_err(|_| std::io::Error::other("invalid range: negative length"))?; + max_response_bytes = Some( + opts.length + .try_into() + .map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?, + ); req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count)); } @@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS { return Err(std::io::Error::other("read_object error")); }; let mut contents = Vec::new(); - while let Ok(Some(chunk)) = reader.next().await.transpose() { - contents.extend_from_slice(&chunk); + while let Some(chunk) = reader.next().await { + append_gcs_chunk(&mut contents, chunk, max_response_bytes)?; } Ok(ReadCloser::new(std::io::Cursor::new(contents))) } @@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS { let mut req = self .control .delete_object() - .set_bucket(format!("projects/_/buckets/{}", self.bucket)) + .set_bucket(gcs_bucket_resource_name(&self.bucket)) .set_object(self.get_dest(object)); if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); @@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS { Ok(()) } + async fn probe_transition_candidate(&self, object: &str) -> Result { + let remote_object = self.get_dest(object); + let parent = gcs_bucket_resource_name(&self.bucket); + probe_exact_gcs_candidate(&remote_object, |page_token| { + let control = self.control.clone(); + let parent = parent.clone(); + let prefix = remote_object.clone(); + async move { + let response = control + .list_objects() + .set_parent(parent) + .set_prefix(prefix) + .set_versions(true) + .set_page_size(2) + .set_page_token(page_token) + .send() + .await + .map_err(|err| std::io::Error::other(err.to_string()))?; + Ok(GcsCandidatePage { + objects: response + .objects + .into_iter() + .map(|candidate| GcsCandidateObject { + name: candidate.name, + generation: candidate.generation, + }) + .collect(), + next_page_token: response.next_page_token, + }) + } + }) + .await + } + async fn in_use(&self) -> Result { // Scope the listing to this tier's prefix (matching the other warm backends) and only // need to know whether a single object exists. let resp = self .control .list_objects() - .set_parent(format!("projects/_/buckets/{}", self.bucket)) + .set_parent(gcs_bucket_resource_name(&self.bucket)) .set_prefix(self.prefix.clone()) .set_page_size(1) .send() @@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS { #[cfg(test)] mod tests { + use super::GcsCandidateObject; + use super::GcsCandidatePage; + use super::MAX_GCS_CANDIDATE_PAGES; use super::WarmBackendGCS; + use super::append_gcs_chunk; + use super::gcs_bucket_resource_name; use super::parse_generation; + use super::probe_exact_gcs_candidate; use crate::services::tier::tier_config::TierGCS; + use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}; + use google_cloud_auth::credentials::anonymous::Builder as Anonymous; + use google_cloud_storage::client::{Storage, StorageControl}; use std::io::ErrorKind; + use std::sync::Arc; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + async fn serve_data_plane_fixture(listener: TcpListener) -> Vec { + let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#; + let responses = [ + format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}", + upload_body.len() + ), + "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!" + .to_string(), + "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!" + .to_string(), + ]; + let mut requests = Vec::new(); + + for response in responses { + let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request"); + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 1024]; + let count = stream + .read(&mut chunk) + .await + .expect("the GCS fixture should read request headers"); + if count == 0 { + break; + } + request.extend_from_slice(&chunk[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let header_end = request + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("the GCS fixture should receive complete request headers"); + let headers = String::from_utf8_lossy(&request[..header_end]); + if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) { + stream + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .await + .expect("the GCS fixture should acknowledge 100-continue"); + } + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("content-length should be numeric")) + }) + .unwrap_or_default(); + while request.len() < header_end.saturating_add(content_length) { + let mut chunk = [0_u8; 1024]; + let count = stream + .read(&mut chunk) + .await + .expect("the GCS fixture should read the request body"); + if count == 0 { + break; + } + request.extend_from_slice(&chunk[..count]); + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + stream + .write_all(response.as_bytes()) + .await + .expect("the GCS fixture should write its response"); + } + + requests + } + + fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage { + GcsCandidatePage { + objects: objects + .iter() + .map(|(name, generation)| GcsCandidateObject { + name: (*name).to_string(), + generation: *generation, + }) + .collect(), + next_page_token: next_page_token.to_string(), + } + } + + async fn probe_candidate_pages( + remote_object: &str, + pages: Vec, + ) -> (Result, Vec) { + let mut pages = pages.into_iter(); + let mut requested_tokens = Vec::new(); + let result = probe_exact_gcs_candidate(remote_object, |page_token| { + requested_tokens.push(page_token); + std::future::ready( + pages + .next() + .ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")), + ) + }) + .await; + (result, requested_tokens) + } #[test] fn generation_parser_preserves_exact_numeric_versions() { @@ -241,6 +483,243 @@ mod tests { } } + #[test] + fn body_collection_propagates_an_error_after_a_complete_prefix() { + let mut contents = Vec::new(); + append_gcs_chunk::(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7)) + .expect("the prefix chunk should be collected"); + let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7)) + .expect_err("a trailing stream error must not be mistaken for EOF"); + + assert_eq!(contents, b"RustFS"); + assert!(err.to_string().contains("trailing stream failure")); + } + + #[test] + fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() { + let mut contents = Vec::new(); + let err = append_gcs_chunk::(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7)) + .expect_err("the GCS collection layer must reject an oversized probe response"); + + assert!(contents.is_empty()); + assert_eq!(err.kind(), ErrorKind::InvalidData); + } + + #[tokio::test] + async fn candidate_probe_finds_exact_name_on_first_or_later_page() { + let (first, first_tokens) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await; + assert_eq!( + first.expect("an exact first-page object should be discovered"), + TransitionCandidateProbe::VersionedPresent("7".to_string()) + ); + assert_eq!(first_tokens, [""]); + + let (later, later_tokens) = probe_candidate_pages( + "prefix/object", + vec![ + candidate_page(&[("prefix/object-shadow", 8)], "next"), + candidate_page(&[("prefix/object", 9)], ""), + ], + ) + .await; + assert_eq!( + later.expect("an exact later-page object should be discovered"), + TransitionCandidateProbe::VersionedPresent("9".to_string()) + ); + assert_eq!(later_tokens, ["", "next"]); + } + + #[tokio::test] + async fn candidate_probe_ignores_non_exact_prefix_matches() { + let (probe, _) = probe_candidate_pages( + "prefix/object", + vec![candidate_page( + &[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)], + "", + )], + ) + .await; + + assert_eq!( + probe.expect("prefix-only matches should not hide the exact object"), + TransitionCandidateProbe::VersionedPresent("7".to_string()) + ); + } + + #[tokio::test] + async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() { + let (probe, _) = probe_candidate_pages( + "prefix/object", + vec![ + candidate_page(&[("prefix/object", 7)], "next"), + candidate_page(&[("prefix/object", 8)], ""), + ], + ) + .await; + + assert_eq!( + probe.expect("multiple exact generations should produce a conservative result"), + TransitionCandidateProbe::Ambiguous + ); + } + + #[tokio::test] + async fn candidate_probe_reports_missing_without_an_exact_name() { + let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await; + + assert_eq!( + probe.expect("a complete listing without an exact name should be definitive"), + TransitionCandidateProbe::Missing + ); + } + + #[tokio::test] + async fn candidate_probe_rejects_non_positive_generations() { + for generation in [0, -1] { + let (probe, _) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await; + let err = probe.expect_err("a non-positive GCS generation must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}"); + } + } + + #[tokio::test] + async fn candidate_probe_rejects_a_page_token_that_does_not_advance() { + let (probe, requested_tokens) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await; + + let err = probe.expect_err("a repeated GCS page token must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens, ["", "next"]); + } + + #[tokio::test] + async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() { + let (probe, requested_tokens) = probe_candidate_pages( + "prefix/object", + vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")], + ) + .await; + + let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens, ["", "a", "b"]); + } + + #[tokio::test] + async fn candidate_probe_rejects_an_unbounded_unique_token_chain() { + let pages = (0..MAX_GCS_CANDIDATE_PAGES) + .map(|index| candidate_page(&[], &format!("token-{index}"))) + .collect(); + let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await; + + let err = probe.expect_err("an unbounded unique page-token chain must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES); + } + + #[tokio::test] + async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("the GCS fixture should bind a loopback port"); + let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address")); + let fixture = tokio::spawn(serve_data_plane_fixture(listener)); + let credentials = Anonymous::new().build(); + let client = Storage::builder() + .with_endpoint(endpoint.clone()) + .with_credentials(credentials.clone()) + .build() + .await + .expect("the GCS data client should build"); + let control = StorageControl::builder() + .with_endpoint(endpoint) + .with_credentials(credentials) + .build() + .await + .expect("the GCS control client should build"); + let backend = WarmBackendGCS { + client: Arc::new(client), + control: Arc::new(control), + bucket: "tier-bucket".to_string(), + prefix: String::new(), + }; + + let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async { + let version = backend + .put( + "probe", + rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")), + 6, + ) + .await + .expect("a plain configured bucket should reach the GCS upload transport"); + let mut reader = backend + .get( + "probe", + &version, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + .expect("a plain configured bucket should reach the GCS read transport"); + let mut body = Vec::new(); + reader + .read_to_end(&mut body) + .await + .expect("the fixture body should be readable"); + let oversized_error = match backend + .get( + "probe", + &version, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + { + Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"), + Err(err) => err, + }; + let requests = fixture.await.expect("the GCS fixture task should finish"); + (version, body, oversized_error.kind(), requests) + }) + .await + .expect("the GCS data-plane requests should not be rejected before transport"); + + assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket"); + assert_eq!(version, "123"); + assert_eq!(body, b"RustFS!"); + assert_eq!(oversized_error_kind, ErrorKind::InvalidData); + assert!( + requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"), + "unexpected upload request line: {}", + requests[0].lines().next().unwrap_or_default() + ); + assert!( + requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"), + "unexpected read request line: {}", + requests[1].lines().next().unwrap_or_default() + ); + assert!( + requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"), + "the GCS probe read must preserve its seven-byte range" + ); + assert!( + requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"), + "unexpected oversized read request line: {}", + requests[2].lines().next().unwrap_or_default() + ); + assert!( + requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"), + "the oversized response must be fetched under the same seven-byte request boundary" + ); + } + #[tokio::test] async fn new_rejects_loopback_endpoint_before_credential_setup() { let conf = TierGCS { diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index f4f489e9c..73ced2cef 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierHuaweicloud, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 55a29418c..5462fc52c 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -26,11 +26,12 @@ use crate::services::tier::{ tier_config::TierS3, warm_backend::{ TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, - build_transition_put_options, + build_transition_put_options, endpoint_authority, }, }; use http::HeaderMap; use rustfs_s3_client::{ + api_error_response::to_error_response, api_get_options::GetObjectOptions, api_list::ListObjectsOptions, api_put_object::PutObjectOptions, @@ -43,7 +44,7 @@ use rustfs_s3_client::{ }; use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::path::SLASH_SEPARATOR; -use s3s::dto::BucketVersioningStatus; +use s3s::{S3ErrorCode, dto::BucketVersioningStatus}; pub struct WarmBackendS3 { pub client: Arc, @@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result Result, std::io::Error> { + if opts.start_offset < 0 || opts.length <= 0 { + return Ok(None); + } + usize::try_from(opts.length) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?; + let end_offset = opts + .start_offset + .checked_add(opts.length - 1) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?; + Ok(Some((opts.start_offset, end_offset))) +} + impl WarmBackendS3 { pub async fn new(conf: &TierS3, _tier: &str) -> Result { Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await @@ -132,10 +146,8 @@ impl WarmBackendS3 { bucket_lookup, ..Default::default() }; - let host = u - .host() - .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; - let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?; + let endpoint = endpoint_authority(&u)?; + let client = TransitionClient::new(&endpoint, opts, tier_type).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); @@ -177,10 +189,8 @@ impl WarmBackendS3 { if !rv.is_empty() { gopts.version_id = rv.to_string(); } - if opts.start_offset >= 0 && opts.length > 0 { - gopts - .set_range(opts.start_offset, opts.start_offset + opts.length - 1) - .map_err(std::io::Error::other)?; + if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? { + gopts.set_range(start_offset, end_offset)?; } let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?; Ok((headers, reader)) @@ -191,36 +201,64 @@ impl WarmBackendS3 { remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str())) } - async fn probe_transition_candidate_versions( + async fn probe_current_transition_candidate_with_header( &self, object: &str, - bucket_versioning: RemoteBucketVersioning, + raw_version_header: Option<&'static str>, ) -> Result { - let remote_object = self.get_dest(object); - let mut opts = ListObjectsOptions::default(); - opts.set("prefix", &remote_object); - opts.set("max-keys", "1000"); - - let mut key_marker = String::new(); - let mut version_id_marker = String::new(); - let mut candidates = TransitionCandidateVersions::default(); - loop { - let versions = self - .client - .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "") - .await?; - candidates.extend(&remote_object, &versions); - if candidates.is_ambiguous() { - return Ok(TransitionCandidateProbe::Ambiguous); + match self + .get_with_headers( + object, + "", + WarmBackendGetOpts { + start_offset: 0, + length: 1, + }, + ) + .await + { + Ok((headers, _)) => { + let version_id = match raw_version_header { + Some(header_name) => match headers.get(header_name) { + Some(value) => { + let version_id = value.to_str().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote object version id is not valid ASCII", + ) + })?; + validate_remote_version_id(version_id)?; + Some(version_id) + } + None => None, + }, + None => self.client.raw_version_id(&headers)?, + }; + Ok(match version_id { + Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()), + None => TransitionCandidateProbe::UnversionedPresent, + }) } - if !versions.is_truncated { - return classify_transition_candidates(candidates, bucket_versioning); + Err(err) => { + let response = to_error_response(&err); + if response.code == S3ErrorCode::NoSuchKey { + Ok(TransitionCandidateProbe::Missing) + } else { + Err(err) + } } - - advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?; } } + pub(crate) async fn probe_transition_candidate_with_raw_version_header( + &self, + object: &str, + raw_version_header: &'static str, + ) -> Result { + self.probe_current_transition_candidate_with_header(object, Some(raw_version_header)) + .await + } + async fn probe_transition_candidate_identity( &self, object: &str, @@ -343,6 +381,7 @@ struct TransitionCandidateVersions { } impl TransitionCandidateVersions { + #[cfg(test)] fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) { for version in versions.versions.iter().filter(|version| version.key == remote_object) { if self.version_id.is_some() { @@ -353,10 +392,6 @@ impl TransitionCandidateVersions { } } - fn is_ambiguous(&self) -> bool { - self.ambiguous - } - fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe { if self.ambiguous { return TransitionCandidateProbe::Ambiguous; @@ -380,6 +415,8 @@ impl TransitionCandidateVersions { mod tests { use super::*; use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version}; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[tokio::test] async fn new_rejects_loopback_endpoint_before_network_setup() { @@ -398,6 +435,204 @@ mod tests { } } + #[tokio::test] + async fn new_preserves_an_explicit_endpoint_port() { + let conf = TierS3 { + endpoint: "https://tier.example.com:9443".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + let backend = WarmBackendS3::new(&conf, "tier") + .await + .expect("a well-formed S3 endpoint should initialize without network I/O"); + assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com")); + assert_eq!(backend.client.endpoint_url.port(), Some(9443)); + } + + #[tokio::test] + async fn overflowing_get_range_is_rejected_before_network_io() { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "s3", + ) + .await + .expect("fixture client should build"), + ); + let backend = WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }; + + let err = backend + .get_with_headers( + "probe", + "", + WarmBackendGetOpts { + start_offset: i64::MAX, + length: 2, + }, + ) + .await + .expect_err("an overflowing range must fail before issuing a GET"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err() + ); + } + + async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle>)> { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let responses = [ + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", + "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", + ]; + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + stream + .write_all(response.as_bytes()) + .await + .expect("fixture should write candidate response"); + } + requests + }); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "s3", + ) + .await + .expect("fixture client should build"), + ); + Some(( + WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }, + fixture, + )) + } + + #[tokio::test] + async fn candidate_probe_uses_only_exact_bounded_get_permissions() { + let Some((backend, fixture)) = candidate_probe_fixture().await else { + return; + }; + + assert_eq!( + backend + .probe_transition_candidate("versioned-probe") + .await + .expect("versioned candidate should be discovered"), + TransitionCandidateProbe::VersionedPresent("opaque-version".to_string()) + ); + assert_eq!( + backend + .probe_transition_candidate("unversioned-probe") + .await + .expect("unversioned candidate should be discovered"), + TransitionCandidateProbe::UnversionedPresent + ); + assert_eq!( + backend + .probe_transition_candidate("missing-probe") + .await + .expect("a missing key should be classified"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_candidate("provider-missing-probe") + .await + .expect("a provider-specific missing code should be classified"), + TransitionCandidateProbe::Missing + ); + let err = backend + .probe_transition_candidate("forbidden-probe") + .await + .expect_err("an authorization failure must not be mistaken for a missing key"); + assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied); + + let requests = fixture.await.expect("candidate fixture should join"); + for request in requests { + let request = request.to_ascii_lowercase(); + assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET"); + assert!(request.contains("\r\nrange: bytes=0-0\r\n")); + assert!(!request.contains("?versioning")); + assert!(!request.contains("?versions")); + } + } + fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { ListVersionsResult { versions: versions @@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 { } async fn probe_transition_candidate(&self, object: &str) -> Result { - let bucket_versioning = self.remote_bucket_versioning().await?; - self.probe_transition_candidate_versions(object, bucket_versioning).await + self.probe_current_transition_candidate_with_header(object, None).await } async fn in_use(&self) -> Result { diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index e2810e316..20eb7ee81 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierTencent, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs index 2311e6576..8398aec52 100644 --- a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs +++ b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs @@ -23,7 +23,7 @@ use uuid::Uuid; use crate::services::tier::{ tier_config::{TierS3, TierWasabi}, - warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}; @@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi { self.s3.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> io::Result { + self.s3.probe_transition_candidate(object).await + } + async fn in_use(&self) -> io::Result { self.check_remote_bucket_unversioned().await?; let in_use = self.s3.in_use().await?; diff --git a/crates/s3-client/src/api_error_response.rs b/crates/s3-client/src/api_error_response.rs index 8373517e7..7c653d6a8 100644 --- a/crates/s3-client/src/api_error_response.rs +++ b/crates/s3-client/src/api_error_response.rs @@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result where D: Deserializer<'de>, { - Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) + let code = String::deserialize(d)?; + if code == "NoSuchObject" { + return Ok(S3ErrorCode::NoSuchKey); + } + Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) } impl Default for ErrorResponse { @@ -325,4 +329,21 @@ mod tests { assert_eq!(response.code, S3ErrorCode::NoSuchVersion); assert_eq!(response.status_code, StatusCode::NOT_FOUND); } + + #[test] + fn normalizes_provider_specific_missing_object_code() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse")); + + let response = http_resp_to_error_response( + StatusCode::NOT_FOUND, + &headers, + b"NoSuchObjectremote detail".to_vec(), + "bucket", + "object", + ); + + assert_eq!(response.code, S3ErrorCode::NoSuchKey); + assert_eq!(response.status_code, StatusCode::NOT_FOUND); + } } diff --git a/crates/s3-client/src/api_get_object.rs b/crates/s3-client/src/api_get_object.rs index cc06fe388..872c1a90d 100644 --- a/crates/s3-client/src/api_get_object.rs +++ b/crates/s3-client/src/api_get_object.rs @@ -30,7 +30,9 @@ use tokio_util::io::StreamReader; use crate::{ api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, - transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider}, + transition_api::{ + ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider, + }, }; use futures_util::StreamExt; use http_body_util::BodyExt; @@ -39,6 +41,42 @@ use hyper::body::Bytes; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use tokio_util::io::ReaderStream; +fn response_limit_from_range(opts: &GetObjectOptions) -> Result, std::io::Error> { + let Some(range) = opts + .headers + .iter() + .find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str())) + else { + return Ok(None); + }; + let Some((unit, bounds)) = range.split_once('=') else { + return Ok(None); + }; + if !unit.eq_ignore_ascii_case("bytes") { + return Ok(None); + } + let Some((start, end)) = bounds.split_once('-') else { + return Ok(None); + }; + if start.is_empty() || end.is_empty() || end.contains(',') { + return Ok(None); + } + let start = start + .parse::() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?; + let end = end + .parse::() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?; + let length = end + .checked_sub(start) + .and_then(|length| length.checked_add(1)) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?; + let limit = usize::try_from(length).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory") + })?; + Ok(Some(limit)) +} + impl TransitionClient { pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result { let _ = opts; @@ -54,6 +92,7 @@ impl TransitionClient { object_name: &str, opts: &GetObjectOptions, ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let max_response_bytes = response_limit_from_range(opts)?; let resp = self .execute_method( http::Method::GET, @@ -81,18 +120,214 @@ impl TransitionClient { let h = resp.headers().clone(); - let mut body_vec = Vec::new(); let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); + let body_vec = if let Some(limit) = max_response_bytes { + collect_response_body(body, limit).await? + } else { + let mut body_vec = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; + if let Some(data) = frame.data_ref() { + body_vec.extend_from_slice(data); + } } - } + body_vec + }; Ok((object_stat, h, BufReader::new(Cursor::new(body_vec)))) } } +#[cfg(test)] +mod bounded_response_tests { + use super::response_limit_from_range; + use crate::{ + api_get_options::GetObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body}, + }; + use http_body_util::Full; + use hyper::body::Bytes; + use std::time::Duration; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + #[test] + fn closed_range_derives_a_collection_limit_without_new_public_options() { + let mut opts = GetObjectOptions::default(); + opts.set_range(5, 11).expect("the closed range should be valid"); + + assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7)); + } + + #[tokio::test] + async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() { + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + let max_response_bytes = response_limit_from_range(&opts) + .expect("the range should parse") + .expect("the closed range should have a limit"); + let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes) + .await + .expect_err("the collection layer must reject a response larger than its limit"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle)> { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let request = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request).into_owned(); + let response = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream + .write_all(response.as_bytes()) + .await + .expect("fixture should write response headers"); + stream.write_all(body).await.expect("fixture should write response body"); + request + }); + let client = TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "", + ) + .await + .expect("fixture client should build"); + Some((client, request)) + } + + #[tokio::test] + async fn real_transport_accepts_the_exact_closed_range_length() { + let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else { + return; + }; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let (_, _, mut reader) = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect("a seven-byte response should fit the requested range"); + let mut body = Vec::new(); + reader + .read_to_end(&mut body) + .await + .expect("bounded response should be readable"); + + assert_eq!(body, b"RustFS!"); + assert!( + request + .await + .expect("fixture should join") + .to_ascii_lowercase() + .contains("\r\nrange: bytes=0-6\r\n") + ); + } + + #[tokio::test] + async fn real_transport_rejects_a_body_larger_than_the_closed_range() { + let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else { + return; + }; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("an eight-byte response must exceed the seven-byte range limit"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + request + .await + .expect("fixture should join") + .to_ascii_lowercase() + .contains("\r\nrange: bytes=0-6\r\n") + ); + } + + #[tokio::test] + async fn overflowing_closed_range_is_rejected_before_network_io() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let client = TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "", + ) + .await + .expect("fixture client should build"); + let mut opts = GetObjectOptions::default(); + opts.headers + .insert("range".to_string(), "bytes=0-18446744073709551615".to_string()); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("an overflowing closed range must be rejected locally"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err() + ); + } +} + #[derive(Default)] pub struct GetRequest { pub buffer: Vec, diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 535554a46..310212cc4 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -80,6 +80,82 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result { .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration")) } +fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> { + match config.tier_type { + TierType::S3 => { + let _ = config + .s3 + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?; + } + TierType::Wasabi => config.name = wasabi_payload_name(config)?, + TierType::RustFS => { + config.name = config + .rustfs + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))? + .name + .clone(); + } + TierType::MinIO => { + config.name = config + .minio + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))? + .name + .clone(); + } + TierType::Aliyun => { + config.name = config + .aliyun + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))? + .name + .clone(); + } + TierType::Tencent => { + config.name = config + .tencent + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))? + .name + .clone(); + } + TierType::Huaweicloud => { + config.name = config + .huaweicloud + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))? + .name + .clone(); + } + TierType::Azure => { + config.name = config + .azure + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))? + .name + .clone(); + } + TierType::GCS => { + let _ = config + .gcs + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?; + } + TierType::R2 => { + config.name = config + .r2 + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))? + .name + .clone(); + } + TierType::Unsupported => {} + } + Ok(()) +} + fn spawn_transition_tier_config_propagation(action: &'static str) { if let Some(notification_sys) = current_notification_system() { debug!( @@ -263,75 +339,7 @@ impl Operation for AddTier { let mut args: TierConfig = serde_json::from_slice(&body) .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; - match args.tier_type { - TierType::S3 => { - args.name = args - .s3 - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))? - .name; - } - TierType::Wasabi => { - args.name = wasabi_payload_name(&args)?; - } - TierType::RustFS => { - args.name = args - .rustfs - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))? - .name; - } - TierType::MinIO => { - args.name = args - .minio - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))? - .name; - } - TierType::Aliyun => { - args.name = args - .aliyun - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))? - .name; - } - TierType::Tencent => { - args.name = args - .tencent - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))? - .name; - } - TierType::Huaweicloud => { - args.name = args - .huaweicloud - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))? - .name; - } - TierType::Azure => { - args.name = args - .azure - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))? - .name; - } - TierType::GCS => { - args.name = args - .gcs - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))? - .name; - } - TierType::R2 => { - args.name = args - .r2 - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))? - .name; - } - _ => (), - } + normalize_add_tier_payload_name(&mut args)?; debug!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, @@ -1150,6 +1158,44 @@ mod tests { assert_eq!(err.message(), Some("missing Wasabi configuration")); } + #[test] + fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() { + for (provider, wire) in [ + ( + "S3", + serde_json::json!({ + "Type": "s3", + "Name": "COLD-S3", + "S3": { + "Endpoint": "https://s3.example.invalid", + "AccessKey": "access", + "SecretKey": "secret", + "Bucket": "archive" + } + }), + ), + ( + "GCS", + serde_json::json!({ + "Type": "gcs", + "Name": "COLD-GCS", + "GCS": { + "Endpoint": "https://storage.googleapis.com", + "Creds": "e30=", + "Bucket": "archive" + } + }), + ), + ] { + let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode"); + let expected = config.name.clone(); + + normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary"); + + assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared"); + } + } + #[test] fn resolve_tier_name_prefers_path_parameter() { let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse"); @@ -1756,5 +1802,11 @@ mod tests { } assert!(!production.contains("check_key_valid(get_session_token")); + + let add_tier = source_block(production, "impl Operation for AddTier"); + assert!( + add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"), + "AddTier must preserve canonical top-level provider names through the tested boundary helper" + ); } } From 65ed86f76e4536fcb76db937dd98c0993a06c370 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 4 Sep 2026 20:39:18 +0800 Subject: [PATCH 17/19] test(ecstore): run checkpoint publication test on large stack (#7134) --- crates/ecstore/src/store/init.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index a89ef05f1..c88579c23 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -10339,9 +10339,17 @@ mod tests { } #[cfg(feature = "test-util")] - #[tokio::test] + #[test] #[serial_test::serial(storage_class_env)] - async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() { + fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() { + run_large_stack_async_test( + "v6-checkpoint-fence-loss", + v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case, + ); + } + + #[cfg(feature = "test-util")] + async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() { let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir"); let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await; From 146003a426e06ec837e14a2a5f8087582958d730 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 4 Sep 2026 21:40:07 +0800 Subject: [PATCH 18/19] refactor(odm): separate source backends from namespace mapping (#7129) refactor(odm): isolate read-only source backend contract --- .../on_demand_migration/source_client.rs | 268 +++++++++++++----- 1 file changed, 203 insertions(+), 65 deletions(-) diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 584cce320..4782e05f0 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -14,9 +14,9 @@ //! Outbound client for an on-demand migration source bucket. //! -//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared -//! remote builder and exposes the read-only surface the migration path -//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a +//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The +//! S3 backend uses the shared remote builder and exposes the surface the +//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a //! probe for admin validation). Every request carries the //! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and //! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of @@ -511,7 +511,7 @@ pub struct SourceObject { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SourcePage { pub objects: Vec, - /// Rolled-up prefixes, in the local namespace; always empty when the + /// Rolled-up prefixes, in the same namespace as `objects`; always empty when the /// request carried no delimiter. pub common_prefixes: Vec, pub is_truncated: bool, @@ -519,7 +519,8 @@ pub struct SourcePage { } /// One `ListObjectsV2` page request against the source. Keys are given in the -/// local namespace; `SourceClient` maps them through `source_prefix`. +/// local namespace at `SourceClient`, and in the source namespace at +/// `SourceBackend`; `SourceClient` maps them through `source_prefix`. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SourceListRequest<'a> { pub prefix: Option<&'a str>, @@ -575,8 +576,30 @@ impl Intercept for SourceProxyMarkerInterceptor { } } -pub struct SourceClient { +/// Read-only provider operations in the source bucket namespace. +/// +/// Implementations must preserve streaming, honor the requested range and +/// pagination cursor, and classify failures without including credentials. +/// `SourceClient` owns prefix mapping so every provider shares the same local +/// namespace. Continuation tokens are opaque and must never be prefix-mapped. +#[async_trait::async_trait] +pub trait SourceBackend: Send + Sync { + async fn head(&self, key: &str) -> Result; + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result; + async fn list(&self, request: &SourceListRequest<'_>) -> Result; + async fn tagging(&self, key: &str) -> Result, SourceError>; + /// Verify bucket access; `SourceClient` separately probes a filtered listing. + async fn probe(&self) -> Result<(), SourceError>; +} + +/// S3-compatible implementation, including request signing and anti-loop headers. +pub struct S3SourceBackend { client: S3Client, + bucket: String, +} + +pub struct SourceClient { + backend: Box, endpoint: String, bucket: String, source_prefix: Option, @@ -609,7 +632,10 @@ impl SourceClient { fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self { let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build()); Self { - client, + backend: Box::new(S3SourceBackend { + client, + bucket: spec.bucket.clone(), + }), endpoint, bucket: spec.bucket.clone(), source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()), @@ -652,35 +678,15 @@ impl SourceClient { } pub async fn head_object(&self, key: &str) -> Result { - let output = self - .client - .head_object() - .bucket(&self.bucket) - .key(self.source_key(key)) - .send() - .await - .map_err(classify_sdk_error)?; - source_head_from_head_output(output) + self.backend.head(&self.source_key(key)).await } - /// Streams the object; `range` is passed through as an HTTP `Range` - /// header and omitted entirely when `None`. + /// Streams the object, preserving an optional HTTP byte range. pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { - let range = range.map(range_header_value).transpose()?; - let output = self - .client - .get_object() - .bucket(&self.bucket) - .key(self.source_key(key)) - .set_range(range) - .send() - .await - .map_err(classify_sdk_error)?; - source_get_from_output(output) + self.backend.get(&self.source_key(key), range).await } - /// Lists one page under the local `prefix`. Keys are returned in the - /// local namespace; entries outside `source_prefix` are skipped. + /// Lists one page under the local prefix. pub async fn list_objects_v2( &self, prefix: Option<&str>, @@ -696,9 +702,81 @@ impl SourceClient { .await } - /// [`Self::list_objects_v2`] with the delimiter and start-after the - /// list-through merge needs (rustfs/backlog#2164). + /// Maps keys and common prefixes while leaving opaque cursors untouched. pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result { + let prefix = self.source_key(request.prefix.unwrap_or_default()); + let start_after = request.start_after.map(|key| self.source_key(key)); + let mut page = self + .backend + .list(&SourceListRequest { + prefix: Some(&prefix), + start_after: start_after.as_deref(), + ..*request + }) + .await?; + page.objects = page + .objects + .into_iter() + .filter_map(|object| self.local_object(object)) + .collect(); + page.common_prefixes = page + .common_prefixes + .into_iter() + .filter_map(|prefix| self.local_key(&prefix).map(str::to_string)) + .collect(); + Ok(page) + } + + fn local_object(&self, mut object: SourceObject) -> Option { + object.key = self.local_key(&object.key)?.to_string(); + Some(object) + } + + pub async fn get_object_tagging(&self, key: &str) -> Result, SourceError> { + self.backend.tagging(&self.source_key(key)).await + } + + pub async fn probe(&self) -> Result { + self.backend.probe().await?; + let page = self.list_objects_v2(None, None, 1).await?; + Ok(SourceProbe { + sample_object: page.objects.into_iter().next(), + has_more_objects: page.is_truncated, + }) + } +} + +#[async_trait::async_trait] +impl SourceBackend for S3SourceBackend { + async fn head(&self, key: &str) -> Result { + let output = self + .client + .head_object() + .bucket(&self.bucket) + .key(key) + .send() + .await + .map_err(classify_sdk_error)?; + source_head_from_head_output(output) + } + + /// Streams the object; `range` is passed through as an HTTP `Range` + /// header and omitted entirely when `None`. + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { + let range = range.map(range_header_value).transpose()?; + let output = self + .client + .get_object() + .bucket(&self.bucket) + .key(key) + .set_range(range) + .send() + .await + .map_err(classify_sdk_error)?; + source_get_from_output(output) + } + + async fn list(&self, request: &SourceListRequest<'_>) -> Result { // `start_after` is silently ignored by S3 once a continuation token is // present; refuse the ambiguous pair rather than list from the wrong // position. @@ -711,9 +789,9 @@ impl SourceClient { .client .list_objects_v2() .bucket(&self.bucket) - .prefix(self.source_key(request.prefix.unwrap_or_default())) + .prefix(request.prefix.unwrap_or_default()) .set_delimiter(request.delimiter.map(str::to_string)) - .set_start_after(request.start_after.map(|after| self.source_key(after))) + .set_start_after(request.start_after.map(str::to_string)) .set_continuation_token(request.continuation_token.map(str::to_string)) .max_keys(request.max_keys) .send() @@ -731,13 +809,13 @@ impl SourceClient { .contents .unwrap_or_default() .into_iter() - .filter_map(|object| self.source_object(object)) + .filter_map(s3_source_object) .collect(); let common_prefixes = output .common_prefixes .unwrap_or_default() .into_iter() - .filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string())) + .filter_map(|prefix| prefix.prefix) .collect(); Ok(SourcePage { @@ -748,48 +826,43 @@ impl SourceClient { }) } - fn source_object(&self, object: SdkObject) -> Option { - let key = self.local_key(object.key.as_deref()?)?.to_string(); - let etag = normalize_etag(object.e_tag); - let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); - Some(SourceObject { - key, - etag, - size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), - last_modified: system_time(object.last_modified), - storage_class: object.storage_class.map(|class| class.as_str().to_string()), - is_multipart_etag, - }) - } - - pub async fn get_object_tagging(&self, key: &str) -> Result, SourceError> { + async fn tagging(&self, key: &str) -> Result, SourceError> { let output = self .client .get_object_tagging() .bucket(&self.bucket) - .key(self.source_key(key)) + .key(key) .send() .await .map_err(classify_sdk_error)?; Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect()) } - /// Admin validation: HeadBucket plus a one-key listing under the prefix. - pub async fn probe(&self) -> Result { + async fn probe(&self) -> Result<(), SourceError> { self.client .head_bucket() .bucket(&self.bucket) .send() .await .map_err(classify_sdk_error)?; - let page = self.list_objects_v2(None, None, 1).await?; - Ok(SourceProbe { - sample_object: page.objects.into_iter().next(), - has_more_objects: page.is_truncated, - }) + Ok(()) } } +fn s3_source_object(object: SdkObject) -> Option { + let key = object.key?; + let etag = normalize_etag(object.e_tag); + let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); + Some(SourceObject { + key, + etag, + size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), + last_modified: system_time(object.last_modified), + storage_class: object.storage_class.map(|class| class.as_str().to_string()), + is_multipart_etag, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -1194,6 +1267,68 @@ mod tests { assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri); } + #[tokio::test] + async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() { + let body = r#" +truedata/opaque +data/photos/ +outside/ +"#; + let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await; + let first = client + .list_page(&SourceListRequest { + prefix: Some("photos/"), + delimiter: Some("/"), + start_after: Some("photos/a"), + max_keys: 2, + ..Default::default() + }) + .await + .expect("delimiter listing should succeed"); + assert_eq!(first.common_prefixes, vec!["photos/"]); + assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque")); + let second = client + .list_page(&SourceListRequest { + continuation_token: first.next_continuation_token.as_deref(), + max_keys: 2, + ..Default::default() + }) + .await + .expect("opaque continuation should succeed"); + assert_eq!(second.common_prefixes, first.common_prefixes); + let requests = recorded(&requests); + let query = |request: &RecordedRequest| { + Url::parse(&request.uri) + .expect("request URI") + .query_pairs() + .into_owned() + .collect::>() + }; + let first_query = query(&requests[0]); + assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/")); + assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a")); + assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/")); + let second_query = query(&requests[1]); + assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque")); + assert!(!second_query.contains_key("start-after")); + } + + #[tokio::test] + async fn list_page_rejects_ambiguous_cursor_before_sending() { + let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await; + let err = client + .list_page(&SourceListRequest { + start_after: Some("a"), + continuation_token: Some("opaque"), + max_keys: 1, + ..Default::default() + }) + .await + .expect_err("ambiguous list position must fail"); + assert!(matches!(err, SourceError::Other(_))); + assert!(recorded(&requests).is_empty(), "invalid request must never reach the source"); + } + #[tokio::test] async fn list_objects_v2_rejects_truncated_page_without_token() { let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await; @@ -1357,11 +1492,14 @@ mod tests { fn prefix_client(prefix: Option) -> SourceClient { SourceClient { - client: S3Client::from_conf( - aws_sdk_s3::Config::builder() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .build(), - ), + backend: Box::new(S3SourceBackend { + client: S3Client::from_conf( + aws_sdk_s3::Config::builder() + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) + .build(), + ), + bucket: "bucket".to_string(), + }), endpoint: "https://source.example.com".to_string(), bucket: "bucket".to_string(), source_prefix: prefix.filter(|prefix| !prefix.is_empty()), From 10ccf7c31a4d7d108b1d745e75d827897c6708d3 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Fri, 4 Sep 2026 23:03:11 +0800 Subject: [PATCH 19/19] fix(version): avoid deriving RustFS version from runtime working directory (#7118) Signed-off-by: cuishuang --- rustfs/src/version.rs | 168 ++---------------------------------------- 1 file changed, 6 insertions(+), 162 deletions(-) diff --git a/rustfs/src/version.rs b/rustfs/src/version.rs index d94adedaf..7384b47a7 100644 --- a/rustfs/src/version.rs +++ b/rustfs/src/version.rs @@ -14,8 +14,6 @@ use const_str::concat; use shadow_rs::shadow; -use std::path::Path; -use std::process::Command; shadow!(build); @@ -47,10 +45,6 @@ pub const DISPLAY_VERSION: &str = { type VersionParseResult = Result<(u32, u32, u32, Option), Box>; -fn build_version_override() -> Option<&'static str> { - BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty()) -} - fn version_ref(version: &str) -> String { if version.starts_with("refs/tags/") || version.starts_with('@') { version.to_string() @@ -61,91 +55,7 @@ fn version_ref(version: &str) -> String { #[allow(clippy::const_is_empty)] pub fn get_version() -> String { - if let Some(version) = build_version_override() { - return version_ref(version); - } - - // Get the latest tag - if let Ok(latest_tag) = get_latest_tag() { - // Check if current commit is newer than the latest tag - if is_head_newer_than_tag(&latest_tag) { - // If current commit is newer, increment the version number - if let Ok(new_version) = increment_version(&latest_tag) { - return format!("refs/tags/{new_version}"); - } - } - - // If current commit is the latest tag, or version increment failed, return current tag - return format!("refs/tags/{latest_tag}"); - } - - // If no tag exists, use original logic - if !build::TAG.is_empty() { - format!("refs/tags/{}", build::TAG) - } else if !build::SHORT_COMMIT.is_empty() { - format!("@{}", build::SHORT_COMMIT) - } else { - format!("refs/tags/{}", build::PKG_VERSION) - } -} - -/// Get the latest git tag -fn get_latest_tag() -> Result> { - let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?; - - if output.status.success() { - let tag = String::from_utf8(output.stdout)?; - Ok(tag.trim().to_string()) - } else { - Err("Failed to get latest tag".into()) - } -} - -/// Check if current HEAD is newer than specified tag -fn is_head_newer_than_tag(tag: &str) -> bool { - is_head_newer_than_tag_in(Path::new("."), tag) -} - -fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool { - let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output(); - let tag_commit = Command::new("git") - .current_dir(repo) - .args(["rev-list", "-n", "1", tag]) - .output(); - - let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else { - return false; - }; - - if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout { - return false; - } - - let output = Command::new("git") - .current_dir(repo) - .args(["merge-base", "--is-ancestor", tag, "HEAD"]) - .output(); - - match output { - Ok(result) => result.status.success(), - Err(_) => false, - } -} - -/// Increment version number (increase patch version) -fn increment_version(version: &str) -> Result> { - // Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19")) - let (major, minor, patch, pre_release) = parse_version(version)?; - - // If there's a pre-release identifier, increment the pre-release version number - if let Some(pre) = pre_release - && let Some(new_pre) = increment_pre_release(&pre) - { - return Ok(format!("{major}.{minor}.{patch}-{new_pre}")); - } - - // Otherwise increment patch version number - Ok(format!("{major}.{minor}.{}", patch + 1)) + version_ref(DISPLAY_VERSION) } /// Parse version number @@ -166,28 +76,6 @@ pub fn parse_version(version: &str) -> VersionParseResult { Ok((major, minor, patch, pre_release)) } -/// Increment pre-release version number -fn increment_pre_release(pre_release: &str) -> Option { - // Handle pre-release versions like "alpha.19" - let parts: Vec<&str> = pre_release.split('.').collect(); - if parts.len() == 2 - && let Ok(num) = parts[1].parse::() - { - return Some(format!("{}.{}", parts[0], num + 1)); - } - - // Handle pre-release versions like "alpha19" - if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) { - let prefix = &pre_release[..=pos]; - let suffix = &pre_release[pos + 1..]; - if let Ok(num) = suffix.parse::() { - return Some(format!("{prefix}{}", num + 1)); - } - } - - None -} - /// Clean version string - removes common prefixes pub fn clean_version(version: &str) -> String { version @@ -284,34 +172,6 @@ mod tests { use super::*; use tracing::debug; - fn run_git(repo: &Path, args: &[&str]) { - let status = Command::new("git").current_dir(repo).args(args).status().unwrap(); - assert!(status.success(), "git command failed: git {}", args.join(" ")); - } - - #[test] - fn test_is_head_newer_than_tag_requires_strict_descendant() { - let repo = tempfile::tempdir().unwrap(); - run_git(repo.path(), &["init", "--quiet"]); - run_git(repo.path(), &["config", "user.name", "RustFS Tests"]); - run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]); - run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]); - run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]); - - assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3")); - - run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]); - - assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3")); - } - - #[test] - fn build_version_override_is_used_for_current_version_when_set() { - if let Some(version) = build_version_override() { - assert_eq!(get_version(), version_ref(version)); - } - } - #[test] fn version_ref_keeps_existing_ref_prefixes() { assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3"); @@ -319,6 +179,11 @@ mod tests { assert_eq!(version_ref("@abc123"), "@abc123"); } + #[test] + fn get_version_uses_build_metadata() { + assert_eq!(get_version(), version_ref(DISPLAY_VERSION)); + } + #[test] fn test_parse_version() { // Test standard version parsing @@ -336,27 +201,6 @@ mod tests { assert_eq!(pre_release, Some("alpha.19".to_string())); } - #[test] - fn test_increment_pre_release() { - // Test alpha.19 -> alpha.20 - assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string())); - - // Test beta.5 -> beta.6 - assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string())); - - // Test unparsable case - assert_eq!(increment_pre_release("unknown"), None); - } - - #[test] - fn test_increment_version() { - // Test pre-release version increment - assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20"); - - // Test standard version increment - assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1"); - } - #[test] fn test_version_format() { // Test if version format starts with refs/tags/