diff --git a/.config/nextest.toml b/.config/nextest.toml index e97d7bdcc..8eb68deaf 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"'] # --- default profile (local): serialize the flaky groups, never retry -------- [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' +filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' [[profile.default.scripts]] @@ -183,7 +183,7 @@ fail-fast = false path = "junit.xml" [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' +filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' [[profile.ci.scripts]] diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 6ea3f15d1..06cbda8ae 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -75,6 +75,10 @@ pub mod bucket { delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, }; + #[cfg(feature = "test-util")] + pub use crate::bucket::lifecycle::transition_transaction::{ + TransitionTransactionRecoveryStats, recover_transition_transaction_records, + }; } pub mod evaluator { @@ -99,6 +103,17 @@ pub mod bucket { pub use crate::bucket::lifecycle::tier_delete_journal::{ persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, }; + + #[cfg(feature = "test-util")] + pub mod test_util { + /// Model a single-node, all-v6 fleet after its capability probe has completed. + /// + /// Call this only once while constructing an isolated test store, before any + /// tier-delete journal permit or background worker can be active. + pub fn install_all_v6_fleet_capability_proof() { + crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test(); + } + } } pub mod tier_last_day_stats { diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index c1e51a14c..9b175986e 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{ DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel, }; use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; -use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity}; +use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent}; use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop; use crate::bucket::object_lock::ObjectLockApi; use crate::bucket::versioning::VersioningApi as _; @@ -50,7 +50,10 @@ use crate::disk::error::DiskError; use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE}; use crate::error::Error; use crate::error::StorageError; -use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down}; +use crate::error::{ + is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found, + is_network_or_host_down, +}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; use crate::object_api::{ObjectEncryptionResolver, ReadPlan}; use crate::services::tier::{ @@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask { } } -async fn delete_free_version_remote_object( +async fn acquire_free_version_tier_lease( oi: &ObjectInfo, tier_config_mgr: &Arc>, -) -> Result<(), std::io::Error> { +) -> Result<(TierOperationLease, bool), std::io::Error> { let version_id_exact = validate_transition_remote_version(oi)?; let identity = tier_destination_id_from_metadata(&oi.user_defined)? .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?; - delete_object_from_remote_tier_idempotent_with_manager_and_identity( + let lease = + TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity) + .await + .map_err(std::io::Error::other)?; + Ok((lease, version_id_exact)) +} + +async fn delete_free_version_remote_object_with_lease( + oi: &ObjectInfo, + lease: &TierOperationLease, + version_id_exact: bool, +) -> Result<(), std::io::Error> { + delete_object_from_remote_tier_with_lease_idempotent( &oi.transitioned_object.name, &oi.transitioned_object.version_id, - &oi.transitioned_object.tier, - identity, - tier_config_mgr, + lease, version_id_exact, ) .await?; Ok(()) } +fn free_version_physical_topology_generation(api: &ECStore) -> String { + let mut hasher = Sha256::new(); + for pool in &api.pools { + hasher.update(pool.pool_idx.to_be_bytes()); + hasher.update(pool.disk_set.len().to_be_bytes()); + for set in &pool.disk_set { + hasher.update(set.set_index.to_be_bytes()); + } + } + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result { + if candidate.transitioned_object.tier != expected.transitioned_object.tier + || candidate.transitioned_object.name != expected.transitioned_object.name + { + return Ok(false); + } + let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)? + .ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?; + let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)? + .ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?; + if candidate_identity != expected_identity { + return Ok(false); + } + if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version remote version state is unknown", + )); + } + Ok(candidate.transition_version_state == expected.transition_version_state + && candidate.transitioned_object.version_id == expected.transitioned_object.version_id) +} + +async fn scan_exact_free_version_targets( + api: &ECStore, + oi: &ObjectInfo, + local_object: &str, +) -> std::io::Result, FileInfo)>> { + let mut targets = Vec::new(); + for pool in &api.pools { + for set in &pool.disk_set { + let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await { + Ok(Some(versions)) => versions, + Ok(None) => continue, + Err(err) if is_err_strict_volume_not_found(&err) => continue, + Err(err) => return Err(std::io::Error::other(err)), + }; + for version in versions.versions.iter().chain(versions.free_versions.iter()) { + let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true); + if free_version_remote_tuple_matches(&candidate, oi)? { + if candidate.transitioned_object.free_version { + // Data movement can leave the same remote tuple in + // several physical pools. Ordinary deletion assigns a + // fresh local free-version UUID to each copy, but all + // of those markers own the same idempotent remote + // DELETE. Consume them together while holding every + // physical object lock; treating their local UUIDs as + // conflicting would strand cleanup forever. + let mut actual = version.clone(); + actual.name = local_object.to_string(); + targets.push((Arc::clone(set), actual)); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "a live transitioned source still references the free-version remote tuple", + )); + } + } + } + } + } + Ok(targets) +} + +fn free_version_cleanup_fences_current( + topology_generation: &str, + api: &ECStore, + bucket_guard: &rustfs_lock::NamespaceLockGuard, + object_guards: &[crate::store::ObjectLockDiagGuard], + lease: &TierOperationLease, + cancel: &CancellationToken, + deadline: tokio::time::Instant, +) -> bool { + !cancel.is_cancelled() + && tokio::time::Instant::now() < deadline + && !bucket_guard.is_lock_lost() + && object_guards.iter().all(|guard| !guard.is_lock_lost()) + && lease.is_current_generation() + && free_version_physical_topology_generation(api) == topology_generation +} + +async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result { + const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30); + + let topology_generation = free_version_physical_topology_generation(&api); + let bucket_guard = api + .acquire_bucket_lifecycle_read_lock(&oi.bucket) + .await + .map_err(std::io::Error::other)?; + let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; + let local_object = encode_dir_object(&oi.name); + let object_guards = api + .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object) + .await + .map_err(std::io::Error::other)?; + let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?; + if targets.is_empty() { + return Ok(false); + } + + let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE; + if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version cleanup fence is invalid before remote delete", + )); + } + tokio::select! { + _ = cancel.cancelled() => { + return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); + } + result = tokio::time::timeout_at( + deadline, + delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact), + ) => { + result + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??; + } + } + if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { + // Remote DELETE is idempotent, but a changed fence makes the local + // outcome ambiguous. Keep every marker for a fully fenced retry. + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version cleanup fence changed after remote delete", + )); + } + + let mut first_error = None; + for (set, actual) in &targets { + let mut delete_request = FileInfo { + name: local_object.clone(), + version_id: actual.version_id, + ..Default::default() + }; + delete_request.set_tier_free_version(); + if let Err(err) = set + .delete_object_version(&oi.bucket, &local_object, &delete_request, false) + .await + && first_error.is_none() + { + first_error = Some(std::io::Error::other(err)); + } + } + let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?; + if !remaining.is_empty() { + return Err(first_error.unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version cleanup remained on at least one physical set", + ) + })); + } + if let Some(err) = first_error { + return Err(err); + } + Ok(true) +} + +#[cfg(all(test, feature = "test-util"))] +async fn delete_free_version_remote_object( + oi: &ObjectInfo, + tier_config_mgr: &Arc>, +) -> Result<(), std::io::Error> { + let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await +} + #[allow( dead_code, reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" @@ -618,8 +813,11 @@ where F: FnOnce() -> Fut, Fut: std::future::Future, { - delete_free_version_remote_object(oi, tier_config_mgr).await?; - Ok(delete_local().await) + let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?; + let result = delete_local().await; + drop(lease); + Ok(result) } struct NewerNoncurrentTask { @@ -690,6 +888,10 @@ impl ExpiryState { usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX) } + pub fn active_tasks(&self) -> usize { + usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX) + } + fn send_expiry_task(&self, wrkr: Sender>, task: ExpiryOpType) -> bool { let queued = wrkr.try_send(Some(task)).is_ok(); if queued { @@ -826,7 +1028,7 @@ impl ExpiryState { } pub async fn resize_workers(n: usize, api: Arc) { - let expiry_state = runtime_sources::expiry_state_handle(); + let expiry_state = api.ctx.expiry_state(); if n == expiry_state.read().await.tasks_tx.len() || n < 1 { return; } @@ -867,7 +1069,7 @@ impl ExpiryState { stats: Arc, recovery_notify: Arc, ) { - let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| { + let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| { static FALLBACK: std::sync::OnceLock = std::sync::OnceLock::new(); FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone() }); @@ -968,119 +1170,33 @@ impl ExpiryState { else if v.as_any().is::() { let v = v.as_any().downcast_ref::().expect("FreeVersionTask downcast failed"); let oi = v.0.clone(); - if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await { - recovery_notify.notify_one(); - debug!( - bucket = %oi.bucket, - object = %oi.name, - remote_object = %oi.transitioned_object.name, - remote_version_id = %oi.transitioned_object.version_id, - tier = %oi.transitioned_object.tier, - error = ?err, - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - reason = "remote_tier_delete_failed", - "Lifecycle worker skipped remote tier delete" - ); - continue; - } - - let local_object = encode_dir_object(&oi.name); - let mut fi = FileInfo { - name: local_object.clone(), - version_id: oi.version_id, - ..Default::default() - }; - // This removes an existing internal cleanup marker. Keeping - // `deleted` false makes duplicate tasks return not-found - // instead of creating an ordinary delete marker. - fi.set_tier_free_version(); - - let mut deleted_locally = false; - for pool in &api.pools { - let set = pool.get_disks_by_key(&local_object); - let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await { - Ok(lock) => lock, - Err(err) => { - recovery_notify.notify_one(); - debug!( - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, - pool_index = pool.pool_idx, - set_index = set.set_index, - error = ?err, - reason = "local_free_version_lock_failed", - "Lifecycle worker failed to create local free-version cleanup lock" - ); - continue; - } - }; - let _object_lock_guard = - match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await { - Ok(guard) => guard, - Err(err) => { - recovery_notify.notify_one(); - debug!( - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, - pool_index = pool.pool_idx, - set_index = set.set_index, - error = ?err, - reason = "local_free_version_lock_failed", - "Lifecycle worker failed to acquire local free-version cleanup lock" - ); - continue; - } - }; - match set - .delete_object_version(&oi.bucket, &local_object, &fi, false) - .await - { - Ok(()) => { - deleted_locally = true; - break; - } - Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue, - Err(err) => { - recovery_notify.notify_one(); - debug!( - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, - remote_object = %oi.transitioned_object.name, - remote_version_id = %oi.transitioned_object.version_id, - tier = %oi.transitioned_object.tier, - error = ?err, - reason = "local_free_version_delete_failed", - "Lifecycle worker failed local free-version cleanup" - ); - break; - } - } - } - - if !deleted_locally { - debug!( + match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await { + Ok(true) => {} + Ok(false) => debug!( event = EVENT_LIFECYCLE_WORKER_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, bucket = %oi.bucket, object = %oi.name, - remote_object = %oi.transitioned_object.name, - remote_version_id = %oi.transitioned_object.version_id, - tier = %oi.transitioned_object.tier, reason = "local_free_version_missing", - "Lifecycle worker could not find transitioned free version locally" - ); + "Lifecycle worker found that the exact free-version was already absent" + ), + Err(err) => { + recovery_notify.notify_one(); + debug!( + event = EVENT_LIFECYCLE_WORKER_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + remote_object = %oi.transitioned_object.name, + remote_version_id = %oi.transitioned_object.version_id, + tier = %oi.transitioned_object.tier, + error = ?err, + reason = "free_version_exact_cleanup_deferred", + "Lifecycle worker retained the exact free-version for a fenced retry" + ); + } } } else { @@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer( RecoveredFreeVersionEnqueueObserverGuard } -pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool { - let expiry_state = runtime_sources::expiry_state_handle(); +pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool { + let expiry_state = api.ctx.expiry_state(); let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await; #[cfg(test)] @@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc, started: &OnceLock<( } Some(tokio::spawn(async move { - let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default(); - let expiry_state = runtime_sources::expiry_state_handle(); + let cancel_token = api.ctx.background_cancel_token().unwrap_or_default(); + let expiry_state = api.ctx.expiry_state(); run_tier_free_version_recovery_loop( cancel_token, expiry_state, @@ -6229,9 +6345,18 @@ mod tests { rustfs_utils::crypto::hex(old_identity), ); oi.user_defined = Arc::new(metadata.clone()); + let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false)); delete_free_version_remote_object_then(&oi, &manager, { let local_delete_calls = Arc::clone(&local_delete_calls); + let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete); + let manager = manager.clone(); move || async move { + assert_eq!( + crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await, + 1, + "the identity-bound tier lease must span the exact local marker delete" + ); + lease_observed_during_local_delete.store(true, Ordering::Relaxed); local_delete_calls.fetch_add(1, Ordering::Relaxed); } }) @@ -6239,6 +6364,12 @@ mod tests { .expect("matching destination identity should allow idempotent remote cleanup"); assert_eq!(old_backend.remove_count().await, 1); assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1); + assert!(lease_observed_during_local_delete.load(Ordering::Relaxed)); + assert_eq!( + crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await, + 0, + "the tier lease should be released after the local marker delete completes" + ); let mut single_prefix_metadata = HashMap::new(); single_prefix_metadata.insert( @@ -6472,6 +6603,7 @@ mod tests { let state = ExpiryState::new(); let mut state = state.write().await; let je = Jentry { + persisted_version: 0, obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), @@ -6480,6 +6612,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; let err = state @@ -6620,6 +6753,7 @@ mod tests { let state = ExpiryState::new_with_unconsumed_worker_channel(1); let mut state = state.write().await; let je = Jentry { + persisted_version: 0, obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), @@ -6628,6 +6762,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; state @@ -6759,7 +6894,7 @@ mod tests { }; assert!( - super::enqueue_recovered_free_version(oi).await, + super::enqueue_recovered_free_version(&ecstore, oi).await, "the resized production worker queue should accept the task" ); stop_tx.send(None).await.expect("worker stop signal should be delivered"); @@ -6875,12 +7010,12 @@ mod tests { .await .expect("free-version task should reach the worker"); tokio::time::timeout(StdDuration::from_secs(30), async { - while remote_backend.remove_count().await == 0 { + while stats.active_tasks() == 0 { tokio::task::yield_now().await; } }) .await - .expect("worker should complete remote cleanup before taking the local lock"); + .expect("worker should mark the cleanup task active before the lock assertion"); let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async { while stats.active_tasks() != 0 { tokio::task::yield_now().await; @@ -6889,7 +7024,12 @@ mod tests { .await; assert!( completed_while_locked.is_err(), - "local cleanup must wait while a competing object writer owns the namespace lock" + "the cleanup task must wait while a competing object writer owns the namespace lock" + ); + assert_eq!( + remote_backend.remove_count().await, + 0, + "the remote tuple must not be deleted before the all-physical namespace fence is acquired" ); for disk_path in &disk_paths { assert!( @@ -6900,6 +7040,13 @@ mod tests { } drop(object_lock_guard); + tokio::time::timeout(StdDuration::from_secs(30), async { + while remote_backend.remove_count().await == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker should delete the remote tuple after acquiring the released namespace fence"); tx.send(None).await.expect("worker stop signal should be delivered"); worker.await.expect("free-version worker should stop cleanly"); @@ -6995,6 +7142,7 @@ mod tests { .next() .expect("seeded free version should be recoverable"); let stale_version_id = oi.version_id.expect("free version should have a concrete UUID"); + let ordinary_marker_mod_time = OffsetDateTime::now_utc(); for disk_path in &disk_paths { let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE); @@ -7017,7 +7165,7 @@ mod tests { name: object.to_string(), version_id: Some(stale_version_id), deleted: true, - mod_time: Some(OffsetDateTime::now_utc()), + mod_time: Some(ordinary_marker_mod_time), ..Default::default() }) .expect("same-ID ordinary marker should replace the stale free version"); @@ -7031,6 +7179,13 @@ mod tests { .expect("same-ID ordinary marker metadata should be written"); } + assert!( + !super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new()) + .await + .expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"), + "the stale free-version task must not report local cleanup" + ); + let state = ExpiryState::new(); let (stats, recovery_notify) = { let state = state.read().await; @@ -11522,7 +11677,7 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] - async fn journal_replay_rejects_unknown_version_state_before_backend_io() { + async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() { let (_disk_paths, ecstore) = setup_test_env().await; let (backend, _) = register_recovery_mock_tier(&ecstore).await; let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM") @@ -11530,6 +11685,7 @@ mod tests { .expect("mock tier lease should be available") .backend_identity(); let je = Jentry { + persisted_version: 0, obj_name: "remote/object".to_string(), version_id: "legacy-version".to_string(), tier_name: "WARM".to_string(), @@ -11538,25 +11694,28 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Unknown, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; + crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je) + .await + .expect("legacy unknown journal should remain byte-compatible and persistable"); let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je) .await - .expect_err("unknown journal state must fail before backend IO"); + .expect_err("legacy unknown journal must be quarantined before backend IO"); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); assert_eq!(backend.remove_count().await, 0); } #[cfg(feature = "test-util")] #[tokio::test] - async fn journal_replay_deletes_confirmed_exact_provider_token() { + async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() { let (_disk_paths, ecstore) = setup_test_env().await; let (backend, _) = register_recovery_mock_tier(&ecstore).await; let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM") .await .expect("mock tier lease should be available"); - let identity = lease.backend_identity(); backend .set_put_remote_version(Some("provider-version-token".to_string())) .await; @@ -11570,34 +11729,30 @@ mod tests { .expect("confirmed remote candidate should be seeded"); backend.set_remove_failure(true); backend.set_reject_non_empty_remote_versions(true); - let je = Jentry { - obj_name: "remote/object".to_string(), - version_id: "provider-version-token".to_string(), - tier_name: "WARM".to_string(), - backend_identity: Some(identity), - version_id_exact: true, - version_state: rustfs_filemeta::TransitionVersionState::Exact, - state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, - source: None, - }; - - crate::set_disk::cleanup_rejected_transition_upload_durably( + let err = crate::set_disk::cleanup_rejected_transition_upload_durably( &lease, - &je.obj_name, - &je.version_id, + "remote/object", + "provider-version-token", true, Some(ecstore.clone()), ) .await - .expect("failed immediate cleanup should remain durable in the journal"); - assert!(backend.contains(&je.obj_name).await); + .expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction"); + assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert!(backend.contains("remote/object").await); backend.set_remove_failure(false); - crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je) - .await - .expect("identity-bound exact journal must retry confirmed candidate cleanup"); + crate::set_disk::cleanup_rejected_transition_upload_durably( + &lease, + "remote/object", + "provider-version-token", + true, + Some(ecstore), + ) + .await + .expect("the transaction retry must delete the same confirmed candidate"); - assert!(!backend.contains(&je.obj_name).await); + assert!(!backend.contains("remote/object").await); assert_eq!(backend.exact_remove_count(), 2); assert_eq!( backend.remove_versions().await, @@ -11760,11 +11915,14 @@ mod tests { }; let mut recovery_rx = recovery_rx.lock().await; assert!( - super::enqueue_recovered_free_version(ObjectInfo { - bucket: "prefill".to_string(), - name: "prefill".to_string(), - ..Default::default() - }) + super::enqueue_recovered_free_version( + &ecstore, + ObjectInfo { + bucket: "prefill".to_string(), + name: "prefill".to_string(), + ..Default::default() + }, + ) .await, "the production recovery queue should accept its first task" ); diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs index 2383196bc..a8b6262fb 100644 --- a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum DurableIlmRecordKind { TierDeleteJournal, + TierDeleteDispatchManifest, TransitionTransaction, ManualTransitionJob, ManualTransitionScope, @@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm max_record_size: 64 * 1024, kind: DurableIlmRecordKind::TierDeleteJournal, }; +pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "tier-delete-journal-v6", + prefix: "ilm/tier-delete-journal-v6/", + max_record_size: 64 * 1024, + kind: DurableIlmRecordKind::TierDeleteJournal, +}; +pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "tier-delete-dispatch-manifest", + prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE, + kind: DurableIlmRecordKind::TierDeleteDispatchManifest, +}; pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { name: "transition-transaction", prefix: "ilm/transition-transactions/records", @@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace kind: DurableIlmRecordKind::ManualTransitionWorkerResult, }; -pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [ +pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [ TIER_DELETE_JOURNAL_NAMESPACE, + TIER_DELETE_JOURNAL_V6_NAMESPACE, + TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, TRANSITION_TRANSACTION_NAMESPACE, MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, @@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint { content_sha256: String, identity_sha256: String, committed: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + dispatch_identity_sha256: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + state: Option, + }, + TierDeleteDispatchManifest { + content_sha256: String, + identity_sha256: String, + state: tier_delete_journal::TierDeleteDispatchManifestState, }, TransitionTransaction { content_sha256: String, @@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint { pub(crate) fn content_sha256(&self) -> &str { match self { Self::TierDeleteJournal { content_sha256, .. } + | Self::TierDeleteDispatchManifest { content_sha256, .. } | Self::TransitionTransaction { content_sha256, .. } | Self::ManualTransitionJob { content_sha256, .. } | Self::ManualTransitionScope { content_sha256, .. } @@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint { } pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> { + for checkpoint in [self, next] { + if let Self::TierDeleteJournal { + committed, + dispatch_identity_sha256, + state, + .. + } = checkpoint + && (state.is_some() != dispatch_identity_sha256.is_some() + || state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed))) + { + return Err(Error::other("durable ILM tier delete journal checkpoint is invalid")); + } + } if self == next { if let Self::ManualTransitionJob { progress, @@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint { let valid = match (self, next) { ( Self::TierDeleteJournal { + content_sha256: previous_content, identity_sha256: previous_identity, committed: previous_committed, + dispatch_identity_sha256: previous_dispatch_identity, + state: previous_state, .. }, Self::TierDeleteJournal { + content_sha256: next_content, identity_sha256: next_identity, committed: next_committed, + dispatch_identity_sha256: next_dispatch_identity, + state: next_state, .. }, ) => { + use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared}; + + let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) { + (Some(previous), Some(next)) => previous == next, + (None, None) => true, + // Old receipts did not record the v6 dispatch binding. A + // byte-identical observation may adopt the stronger proof, + // but an in-flight mutation must fail closed instead of + // guessing which operation owned the journal. + (None, Some(_)) => previous_content == next_content, + (Some(_), None) => false, + }; + let state_is_monotonic = match (previous_state, next_state) { + (Some(previous), Some(next)) => { + previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed)) + } + (None, None) => previous_committed == next_committed || (!previous_committed && *next_committed), + (None, Some(_)) => previous_content == next_content, + (Some(_), None) => false, + }; + previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic + } + ( + Self::TierDeleteDispatchManifest { + identity_sha256: previous_identity, + state: previous_state, + .. + }, + Self::TierDeleteDispatchManifest { + identity_sha256: next_identity, + state: next_state, + .. + }, + ) => { + use tier_delete_journal::TierDeleteDispatchManifestState::{ + Aborted, Aborting, Completed, DispatchAuthorized, Preparing, + }; previous_identity == next_identity - && (previous_committed == next_committed || (!previous_committed && *next_committed)) + && matches!( + (previous_state, next_state), + (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed) + ) } ( Self::TransitionTransaction { @@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint { Err(Error::other("durable ILM record generation is not a monotonic successor")) } } + + /// Whether `self` is an older generation of the same immutable record + /// that can reach `terminal` through one or more valid state transitions. + /// This is deliberately broader than `validate_successor`, which remains + /// adjacent-only for receipt advancement. Terminal cleanup uses this only + /// after the exact terminal ETag and terminal receipt were committed, to + /// purge older object versions exposed by that deletion. + pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool { + if self == terminal || self.validate_successor(terminal).is_ok() { + return true; + } + match (self, terminal) { + ( + Self::TierDeleteJournal { + identity_sha256: previous_identity, + dispatch_identity_sha256: previous_dispatch, + state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared), + .. + }, + Self::TierDeleteJournal { + identity_sha256: terminal_identity, + dispatch_identity_sha256: terminal_dispatch, + state: Some(super::tier_sweeper::TierDeleteJournalState::Committed), + .. + }, + ) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch, + ( + Self::TierDeleteDispatchManifest { + identity_sha256: previous_identity, + state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing, + .. + }, + Self::TierDeleteDispatchManifest { + identity_sha256: terminal_identity, + state: + tier_delete_journal::TierDeleteDispatchManifestState::Aborted + | tier_delete_journal::TierDeleteDispatchManifestState::Completed, + .. + }, + ) => previous_identity == terminal_identity, + _ => false, + } + } } fn transition_state_distance( @@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result Result { + let (operation_id, identity_sha256, state) = + tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?; + ( + "operation_id", + hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), + DurableIlmRecordCheckpoint::TierDeleteDispatchManifest { + content_sha256, + identity_sha256, + state, }, ) } @@ -956,6 +1108,87 @@ mod tests { } } + #[test] + fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() { + use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing}; + + let operation_id = Uuid::new_v4(); + let checkpoint = |state| { + let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state); + let namespace = classify_durable_ilm_record(&path) + .expect("dispatch manifest namespace should classify") + .expect("dispatch manifest should be durable"); + assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE); + validate_durable_ilm_record(&path, &data) + .expect("dispatch manifest should validate") + .checkpoint + }; + + let preparing = checkpoint(Preparing); + let authorized = checkpoint(DispatchAuthorized); + let completed = checkpoint(Completed); + let aborting = checkpoint(Aborting); + let aborted = checkpoint(Aborted); + + preparing + .validate_successor(&authorized) + .expect("Preparing may become DispatchAuthorized"); + authorized + .validate_successor(&completed) + .expect("DispatchAuthorized may become Completed"); + preparing.validate_successor(&aborting).expect("Preparing may enter rollback"); + aborting.validate_successor(&aborted).expect("Aborting may become Aborted"); + assert!(authorized.validate_successor(&aborting).is_err()); + assert!(completed.validate_successor(&authorized).is_err()); + assert!(aborted.validate_successor(&preparing).is_err()); + } + + #[test] + fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() { + use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared}; + + let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: content.repeat(64), + identity_sha256: "i".repeat(64), + committed: state == Some(Committed), + dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)), + state, + }; + let prepared = checkpoint("a", Some("d"), Some(Prepared)); + let dispatched = checkpoint("b", Some("d"), Some(Dispatched)); + let committed = checkpoint("c", Some("d"), Some(Committed)); + prepared + .validate_successor(&dispatched) + .expect("Prepared may advance to Dispatched"); + dispatched + .validate_successor(&committed) + .expect("Dispatched may advance to Committed"); + assert!(prepared.validate_successor(&committed).is_err()); + assert!(dispatched.validate_successor(&prepared).is_err()); + + let rebound = checkpoint("b", Some("e"), Some(Dispatched)); + assert!(dispatched.validate_successor(&rebound).is_err()); + + let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({ + "kind": "tier_delete_journal", + "content_sha256": "a".repeat(64), + "identity_sha256": "i".repeat(64), + "committed": false + })) + .expect("legacy tier-delete checkpoint should remain decodable"); + legacy + .validate_successor(&prepared) + .expect("byte-identical legacy receipt may adopt the stronger v6 proof"); + let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: "z".repeat(64), + identity_sha256: "i".repeat(64), + committed: false, + dispatch_identity_sha256: None, + state: None, + }; + assert!(changed_legacy.validate_successor(&prepared).is_err()); + } + #[test] fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() { let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index f8e286e8f..a2290c8ea 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -34,6 +34,6 @@ pub mod tier_sweeper; pub mod transition_transaction; pub(crate) use durable_namespace::{ - DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record, - validate_durable_ilm_record, + DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord, + classify_durable_ilm_record, validate_durable_ilm_record, }; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 22bc2ca70..2f519ab55 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -12,45 +12,487 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{future::Future, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + future::Future, + sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use futures::StreamExt; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; +#[cfg(all(test, feature = "test-util"))] +use std::sync::atomic::AtomicUsize; + use crate::bucket::lifecycle::config_boundary; -use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE; +use crate::bucket::lifecycle::durable_namespace::{ + TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record, +}; use crate::bucket::lifecycle::runtime_boundary; use crate::bucket::lifecycle::tier_sweeper::{ - Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, - delete_confirmed_transition_candidate_exact_with_manager_and_identity, - delete_object_from_remote_tier_idempotent_with_manager_and_identity, + Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, + delete_confirmed_transition_candidate_exact_with_lease_idempotent, delete_object_from_remote_tier_with_lease_idempotent, }; use crate::disk::RUSTFS_META_BUCKET; -use crate::error::{Error, Result}; +use crate::error::{Error, Result, is_err_strict_volume_not_found}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; -use crate::services::tier::tier::tier_destination_id_from_metadata; +use crate::services::notification_sys::{ + TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, + tier_delete_journal_topology_generation, +}; +use crate::services::tier::tier::{TierConfigMgr, tier_destination_id_from_metadata}; use crate::storage_api_contracts::{ list::ListOperations as _, + namespace::NamespaceLocking as _, object::{DeletedObject, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete}, range::HTTPRangeSpec, }; use crate::store::ECStore; use rustfs_filemeta::FileInfo; -const LOG_COMPONENT_ECSTORE: &str = "ecstore"; -const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; -const EVENT_LIFECYCLE_TIER_DELETE_JOURNAL: &str = "lifecycle_tier_delete_journal"; +pub(crate) const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +pub(crate) const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; +pub(crate) const EVENT_LIFECYCLE_TIER_DELETE_JOURNAL: &str = "lifecycle_tier_delete_journal"; -pub const DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT: usize = 1_000; +// Keep one background pass small enough that a slow remote tier cannot hold +// the shared recovery worker for minutes. Subsequent passes resume from the +// returned marker, so this bounds latency without reducing eventual coverage. +pub const DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT: usize = 8; const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60); const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300); +const TIER_DELETE_REMOTE_DEADLINE: Duration = Duration::from_secs(30); +const TIER_DELETE_JOURNAL_ENTRY_RECOVERY_TIMEOUT: Duration = Duration::from_secs(90); +const TIER_DELETE_JOURNAL_RECOVERY_CONCURRENCY: usize = 4; +const TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_TIMEOUT: Duration = Duration::from_secs(120); +const TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_CONCURRENCY: usize = 4; +const TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY: usize = 32; +const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32; +const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16; +const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32; const TIER_DELETE_JOURNAL_VERSION: u8 = 2; const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3; const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4; const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5; -pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix; +// v6 is the first transaction journal whose Committed state is allowed to be +// the sole remote-cleanup owner. Readers capped at v5 reject this version and +// therefore cannot bypass the all-pool live-source proof during a rolling +// upgrade or downgrade. +// RUSTFS_COMPAT_TODO(backlog-2097-tier-delete-journal-v6): retain v1-v5 readers and the v6 downgrade fence for safe rollback. Remove after every supported rollback release preserves v6 sole-owner journals without remote deletion. +const TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION: u8 = 6; +/// Common list prefix covering both the byte-compatible v1-v5 namespace and +/// the operation-scoped v6 namespace. +pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal"; +pub(crate) const TIER_DELETE_JOURNAL_LEGACY_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix; +pub(crate) const TIER_DELETE_JOURNAL_V6_PREFIX: &str = TIER_DELETE_JOURNAL_V6_NAMESPACE.prefix; +pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_PREFIX: &str = "ilm/tier-delete-dispatch-manifests/"; +const TIER_DELETE_DISPATCH_MANIFEST_VERSION: u8 = 1; +pub(crate) const MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE: usize = 32 * 1024 * 1024; +const MAX_TIER_DELETE_DISPATCH_JOURNALS: usize = 200_000; + +fn valid_tier_delete_topology_generation(generation: &str) -> bool { + generation.len() == 64 && generation.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +pub(crate) fn tier_delete_source_matches_dispatch_scope( + source: &TierDeleteSourceIdentity, + bucket: &str, + persisted_prefix: &str, +) -> bool { + let logical_prefix = rustfs_utils::path::decode_dir_object(persisted_prefix); + source.bucket == bucket && !source.object.is_empty() && source.object.starts_with(&logical_prefix) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) enum TierDeleteDispatchManifestState { + Preparing, + DispatchAuthorized, + Aborting, + Aborted, + Completed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct TierDeleteDispatchManifest { + version: u8, + operation_id: uuid::Uuid, + bucket: String, + bucket_incarnation: uuid::Uuid, + prefix: String, + journal_names: Vec, + journal_set_sha256: String, + journal_count: u64, + topology_generation: String, + state: TierDeleteDispatchManifestState, +} + +impl TierDeleteDispatchManifest { + fn validate(&self, object_name: &str) -> Result<()> { + if self.version != TIER_DELETE_DISPATCH_MANIFEST_VERSION + || self.operation_id.is_nil() + || self.bucket.is_empty() + || self.bucket_incarnation.is_nil() + || self.journal_names.len() > MAX_TIER_DELETE_DISPATCH_JOURNALS + || self.journal_count != self.journal_names.len() as u64 + || !valid_tier_delete_topology_generation(&self.topology_generation) + { + return Err(Error::other("tier delete dispatch manifest is invalid")); + } + if !self.journal_names.windows(2).all(|pair| pair[0] < pair[1]) + || self.journal_names.iter().any(|name| { + let expected_prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", self.operation_id.simple()); + !name.starts_with(&expected_prefix) || !name.ends_with(".json") + }) + || tier_delete_dispatch_journal_set_digest(&self.journal_names) != self.journal_set_sha256 + || tier_delete_dispatch_manifest_object_name(&self.bucket, self.bucket_incarnation, &self.prefix) != object_name + { + return Err(Error::other("tier delete dispatch manifest binding is invalid")); + } + Ok(()) + } +} + +fn tier_delete_dispatch_journal_set_digest(names: &[String]) -> String { + let mut hasher = Sha256::new(); + for name in names { + hasher.update((name.len() as u64).to_be_bytes()); + hasher.update(name.as_bytes()); + } + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +fn tier_delete_dispatch_manifest_object_name(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(bucket.as_bytes()); + hasher.update([0]); + hasher.update(incarnation.as_bytes()); + hasher.update([0]); + hasher.update(prefix.as_bytes()); + format!( + "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{}.json", + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) + ) +} + +fn tier_delete_dispatch_operation_lock_name(manifest_object: &str) -> String { + format!("{manifest_object}.operation-lock") +} + +fn ensure_dispatch_lock_current( + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, +) -> Result<()> { + if bucket_fence.is_lock_lost() || operation_guard.is_lock_lost() { + return Err(Error::other("tier delete dispatch namespace fence was lost")); + } + Ok(()) +} + +fn dispatch_write_fences_current( + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + topology_generation: &str, +) -> bool { + !bucket_fence.is_lock_lost() + && !operation_guard.is_lock_lost() + && tier_delete_journal_fleet_proof_matches(fleet_proof) + && tier_delete_journal_topology_generation(fleet_proof) == topology_generation +} + +fn encode_tier_delete_dispatch_manifest(manifest: &TierDeleteDispatchManifest) -> Result> { + let data = serde_json::to_vec(manifest) + .map_err(|err| Error::other_with_context("encode tier delete dispatch manifest failed", err))?; + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch manifest is too large")); + } + Ok(data) +} + +fn decode_tier_delete_dispatch_manifest(data: &[u8], object_name: &str) -> Result { + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch manifest is too large")); + } + let manifest: TierDeleteDispatchManifest = serde_json::from_slice(data) + .map_err(|err| Error::other_with_context("decode tier delete dispatch manifest failed", err))?; + manifest.validate(object_name)?; + Ok(manifest) +} + +pub(crate) fn validate_tier_delete_dispatch_manifest_record( + object_name: &str, + data: &[u8], +) -> Result<(uuid::Uuid, String, TierDeleteDispatchManifestState)> { + let manifest = decode_tier_delete_dispatch_manifest(data, object_name)?; + let identity = serde_json::to_vec(&( + manifest.version, + manifest.operation_id, + &manifest.bucket, + manifest.bucket_incarnation, + &manifest.prefix, + &manifest.journal_names, + &manifest.journal_set_sha256, + manifest.journal_count, + &manifest.topology_generation, + )) + .map_err(Error::other)?; + Ok(( + manifest.operation_id, + rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), + manifest.state, + )) +} + +/// Return the fleet generation durably bound to a v6 manifest or journal. +/// Legacy v1-v5 records deliberately return `None` and retain their existing +/// cleanup compatibility behavior. +pub(crate) fn durable_ilm_v6_topology_generation(object_name: &str, data: &[u8]) -> Result> { + if object_name.starts_with(TIER_DELETE_DISPATCH_MANIFEST_PREFIX) { + return decode_tier_delete_dispatch_manifest(data, object_name).map(|manifest| Some(manifest.topology_generation)); + } + if object_name.starts_with(TIER_DELETE_JOURNAL_V6_PREFIX) { + let entry = decode_tier_delete_journal_entry(data)?; + if entry.persisted_version != TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION + || tier_delete_journal_object_name(&entry) != object_name + { + return Err(Error::other("tier delete journal v6 topology binding does not match its path")); + } + let generation = entry + .dispatch + .ok_or_else(|| Error::other("tier delete journal v6 topology binding is missing"))? + .topology_generation; + if !valid_tier_delete_topology_generation(&generation) { + return Err(Error::other("tier delete journal v6 topology generation is invalid")); + } + return Ok(Some(generation)); + } + Ok(None) +} + +#[cfg(test)] +pub(crate) fn test_tier_delete_dispatch_manifest_record( + operation_id: uuid::Uuid, + state: TierDeleteDispatchManifestState, +) -> (String, Vec) { + let bucket = "durable-manifest-bucket"; + let bucket_incarnation = uuid::Uuid::from_u128(1); + let prefix = "archive/"; + let object_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let manifest = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id, + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + journal_names: Vec::new(), + journal_set_sha256: tier_delete_dispatch_journal_set_digest(&[]), + journal_count: 0, + topology_generation: "a".repeat(64), + state, + }; + let data = encode_tier_delete_dispatch_manifest(&manifest).expect("test dispatch manifest should encode"); + (object_name, data) +} + +struct DispatchedJournalPermit { + manifest: TierDeleteDispatchManifest, + authorized_etag: String, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, +} + +struct TierDeleteDispatchAuthorizationInner { + manifest_object: String, + operation_id: uuid::Uuid, + bucket: String, + bucket_incarnation: uuid::Uuid, + prefix: String, + journal_set_sha256: String, + journal_entry_indexes: std::collections::BTreeMap, + entries: Arc<[Jentry]>, + topology_generation: String, + fleet_proof: TierDeleteJournalFleetProofToken, + mutation_started: AtomicBool, +} + +#[doc(hidden)] +#[derive(Clone)] +pub struct TierDeleteDispatchAuthorization(Arc); + +impl std::fmt::Debug for TierDeleteDispatchAuthorization { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TierDeleteDispatchAuthorization") + .field("operation_id", &self.0.operation_id) + .field("journal_count", &self.0.journal_entry_indexes.len()) + .field("mutation_started", &self.mutation_started()) + .finish() + } +} + +impl TierDeleteDispatchAuthorization { + fn ensure_fleet_proof_current(&self) -> Result<()> { + if !tier_delete_journal_fleet_proof_matches(&self.0.fleet_proof) + || tier_delete_journal_topology_generation(&self.0.fleet_proof) != self.0.topology_generation + { + return Err(Error::other("tier delete dispatch authorization fleet proof is stale")); + } + Ok(()) + } + + pub(crate) fn ensure_current(&self, bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> Result<()> { + if self.0.bucket != bucket || self.0.bucket_incarnation != incarnation || self.0.prefix != prefix { + return Err(Error::other( + "tier delete dispatch authorization is stale or belongs to another operation", + )); + } + self.ensure_fleet_proof_current() + } + + pub(crate) fn mark_mutation_started(&self, bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> Result<()> { + self.ensure_current(bucket, incarnation, prefix)?; + self.0.mutation_started.store(true, Ordering::Release); + Ok(()) + } + + pub(crate) fn mutation_started(&self) -> bool { + self.0.mutation_started.load(Ordering::Acquire) + } + + fn bound_journal_entry(&self, entry: &Jentry) -> Result<(String, Jentry)> { + // This method is called immediately before each authorized local + // source mutation. Re-check the fleet generation here because the + // caller may have waited for physical object locks after admission. + self.ensure_fleet_proof_current()?; + let source = entry + .source + .as_ref() + .filter(|source| source.has_stable_identity()) + .ok_or_else(|| Error::other("tier delete dispatch candidate has no stable source identity"))?; + if !tier_delete_source_matches_dispatch_scope(source, &self.0.bucket, &self.0.prefix) + || !entry.can_replace_tier_free_version() + { + return Err(Error::other("tier delete dispatch candidate is outside its authorized source scope")); + } + let mut bound = entry.clone(); + bound.persisted_version = 0; + bound.state = TierDeleteJournalState::Prepared; + bound.dispatch = Some(TierDeleteDispatchBinding { + operation_id: self.0.operation_id, + manifest_object: self.0.manifest_object.clone(), + journal_set_sha256: self.0.journal_set_sha256.clone(), + topology_generation: self.0.topology_generation.clone(), + }); + Ok((tier_delete_journal_object_name(&bound), bound)) + } + + pub(crate) fn authorizes_journal_entry(&self, entry: &Jentry) -> Result { + let (name, bound) = self.bound_journal_entry(entry)?; + Ok(self + .0 + .journal_entry_indexes + .get(&name) + .and_then(|index| self.0.entries.get(*index)) + .is_some_and(|expected| same_tier_delete_authorization_identity(expected, &bound))) + } + + pub(crate) fn authorized_journal_name(&self, entry: &Jentry) -> Result { + let (name, bound) = self.bound_journal_entry(entry)?; + if !self + .0 + .journal_entry_indexes + .get(&name) + .and_then(|index| self.0.entries.get(*index)) + .is_some_and(|expected| same_tier_delete_authorization_identity(expected, &bound)) + { + return Err(Error::other("tier delete dispatch does not authorize this exact cleanup identity")); + } + Ok(name) + } +} + +pub(crate) struct PreparedTierDeleteDispatch { + permit: Option, + predecessor_replay_required: bool, +} + +pub(crate) struct ActiveTierDeleteDispatch { + manifest: TierDeleteDispatchManifest, + authorized_etag: String, + entries: Arc<[Jentry]>, + authorization: TierDeleteDispatchAuthorization, + predecessor_replay_required: bool, +} + +impl PreparedTierDeleteDispatch { + pub(crate) fn consume(mut self, bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> Result { + let permit = self + .permit + .take() + .ok_or_else(|| Error::other("tier delete dispatch permit was already consumed"))?; + let manifest_object = tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix); + if permit.manifest.state != TierDeleteDispatchManifestState::DispatchAuthorized + || permit.manifest.bucket != bucket + || permit.manifest.bucket_incarnation != incarnation + || permit.manifest.prefix != prefix + || permit.entries.iter().map(tier_delete_journal_object_name).collect::>() != permit.manifest.journal_names + || !tier_delete_journal_fleet_proof_matches(&permit.fleet_proof) + || tier_delete_journal_topology_generation(&permit.fleet_proof) != permit.manifest.topology_generation + { + return Err(Error::other("tier delete dispatch permit validation failed")); + } + let entries = Arc::<[Jentry]>::from(permit.entries); + let journal_entry_indexes = permit + .manifest + .journal_names + .iter() + .cloned() + .enumerate() + .map(|(index, name)| (name, index)) + .collect(); + let authorization = TierDeleteDispatchAuthorization(Arc::new(TierDeleteDispatchAuthorizationInner { + manifest_object, + operation_id: permit.manifest.operation_id, + bucket: permit.manifest.bucket.clone(), + bucket_incarnation: permit.manifest.bucket_incarnation, + prefix: permit.manifest.prefix.clone(), + journal_set_sha256: permit.manifest.journal_set_sha256.clone(), + journal_entry_indexes, + entries: Arc::clone(&entries), + topology_generation: permit.manifest.topology_generation.clone(), + fleet_proof: permit.fleet_proof, + mutation_started: AtomicBool::new(false), + })); + Ok(ActiveTierDeleteDispatch { + manifest: permit.manifest, + authorized_etag: permit.authorized_etag, + entries, + authorization, + predecessor_replay_required: self.predecessor_replay_required, + }) + } +} + +impl ActiveTierDeleteDispatch { + pub(crate) fn authorization(&self) -> TierDeleteDispatchAuthorization { + self.authorization.clone() + } + + pub(crate) fn entries(&self) -> &[Jentry] { + &self.entries + } + + pub(crate) fn predecessor_replay_required(&self) -> bool { + self.predecessor_replay_required + } +} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -69,17 +511,36 @@ struct PersistedTierDeleteJournalEntry { state: Option, #[serde(default, skip_serializing_if = "Option::is_none")] source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + dispatch: Option, } impl PersistedTierDeleteJournalEntry { fn from_jentry(je: &Jentry) -> Result { validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?; let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown; - let version = if je.source.is_some() || je.state == TierDeleteJournalState::Prepared { + let version = if je.dispatch.is_some() { + if !je + .dispatch + .as_ref() + .is_some_and(|dispatch| valid_tier_delete_topology_generation(&dispatch.topology_generation)) + { + return Err(Error::other("tier delete v6 transaction has an invalid topology generation")); + } if je.backend_identity.is_none() { return Err(Error::other("tier delete transaction is missing its backend identity")); } + if !je.source.as_ref().is_some_and(TierDeleteSourceIdentity::has_stable_identity) { + return Err(Error::other("tier delete v6 transaction is missing its stable source identity")); + } + TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION + } else if je.persisted_version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION { + // A decoded v5 may be inspected or checkpointed by compatibility + // tooling, but new online transactions are never emitted as v5. TIER_DELETE_JOURNAL_TRANSACTION_VERSION + } else if je.source.is_some() || matches!(je.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) + { + return Err(Error::other("new sole-owner tier delete journal requires a dispatch manifest binding")); } else if legacy_unknown { if je.backend_identity.is_some() { TIER_DELETE_JOURNAL_VERSION @@ -100,10 +561,14 @@ impl PersistedTierDeleteJournalEntry { backend_identity: je.backend_identity, version_id_exact: je.version_id_exact.then_some(true), version_state: (!legacy_unknown).then_some(je.version_state), - state: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION).then_some(je.state), - source: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION) + state: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION || version == TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION) + .then_some(je.state), + source: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION || version == TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION) .then(|| je.source.clone()) .flatten(), + dispatch: (version == TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION) + .then(|| je.dispatch.clone()) + .flatten(), }) } @@ -119,19 +584,22 @@ impl PersistedTierDeleteJournalEntry { if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION && self.version != TIER_DELETE_JOURNAL_STATE_VERSION && self.version != TIER_DELETE_JOURNAL_TRANSACTION_VERSION + && self.version != TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION && self.version_id_exact.unwrap_or(false) { return Err(Error::other( "legacy tier delete journal entry has an unsupported exact version constraint", )); } - let (backend_identity, version_id_exact, version_state, state, source) = match self.version { + let persisted_version = self.version; + let (backend_identity, version_id_exact, version_state, state, source, dispatch) = match self.version { 1 => ( None, false, rustfs_filemeta::TransitionVersionState::Unknown, TierDeleteJournalState::Committed, None, + None, ), TIER_DELETE_JOURNAL_VERSION => ( Some( @@ -142,6 +610,7 @@ impl PersistedTierDeleteJournalEntry { rustfs_filemeta::TransitionVersionState::Unknown, TierDeleteJournalState::Committed, None, + None, ), TIER_DELETE_JOURNAL_EXACT_VERSION => { if self.version_id.is_empty() || self.version_id_exact != Some(true) { @@ -156,6 +625,7 @@ impl PersistedTierDeleteJournalEntry { rustfs_filemeta::TransitionVersionState::Exact, TierDeleteJournalState::Committed, None, + None, ) } TIER_DELETE_JOURNAL_STATE_VERSION => { @@ -173,34 +643,60 @@ impl PersistedTierDeleteJournalEntry { state, TierDeleteJournalState::Committed, None, + None, ) } - TIER_DELETE_JOURNAL_TRANSACTION_VERSION => { - let state = self - .state - .ok_or_else(|| Error::other("tier delete journal v5 entry is missing its state"))?; - let source = self - .source - .ok_or_else(|| Error::other("tier delete journal v5 entry is missing its source identity"))?; + TIER_DELETE_JOURNAL_TRANSACTION_VERSION | TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION => { + let version = self.version; + let state = self.state.ok_or_else(|| { + Error::other_with_context("tier delete journal entry is missing its state", format!("journal v{version}")) + })?; + let source = self.source.ok_or_else(|| { + Error::other_with_context( + "tier delete journal entry is missing its source identity", + format!("journal v{version}"), + ) + })?; let exact = self.version_id_exact.unwrap_or(false); - let version_state = self - .version_state - .ok_or_else(|| Error::other("tier delete journal v5 entry is missing its version state"))?; + let version_state = self.version_state.ok_or_else(|| { + Error::other_with_context( + "tier delete journal entry is missing its version state", + format!("journal v{version}"), + ) + })?; validate_version_state(version_state, &self.version_id, exact)?; + let dispatch = if version == TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION { + let dispatch = self + .dispatch + .ok_or_else(|| Error::other("tier delete journal v6 entry is missing its dispatch binding"))?; + if !valid_tier_delete_topology_generation(&dispatch.topology_generation) { + return Err(Error::other("tier delete journal v6 entry has an invalid topology generation")); + } + Some(dispatch) + } else { + if self.dispatch.is_some() { + return Err(Error::other("tier delete journal v5 entry has an unsupported dispatch binding")); + } + None + }; ( - Some( - self.backend_identity - .ok_or_else(|| Error::other("tier delete journal v5 entry is missing its backend identity"))?, - ), + Some(self.backend_identity.ok_or_else(|| { + Error::other_with_context( + "tier delete journal entry is missing its backend identity", + format!("journal v{version}"), + ) + })?), exact, version_state, state, Some(source), + dispatch, ) } version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))), }; Ok(Jentry { + persisted_version, obj_name: self.obj_name, version_id: self.version_id, tier_name: self.tier_name, @@ -209,6 +705,7 @@ impl PersistedTierDeleteJournalEntry { version_state, state, source, + dispatch, }) } } @@ -241,7 +738,39 @@ pub struct TierDeleteJournalRecoveryStats { pub truncated: bool, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TierDeleteDispatchManifestRecoveryStats { + pub scanned: usize, + pub advanced: usize, + pub deleted: usize, + pub retained: usize, + pub failed: usize, + pub next_marker: Option, + pub truncated: bool, +} + pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String { + let digest = tier_delete_journal_identity_digest(je); + if let Some(dispatch) = &je.dispatch { + return format!( + "{TIER_DELETE_JOURNAL_V6_PREFIX}{}/{}.json", + dispatch.operation_id.simple(), + rustfs_utils::crypto::hex(digest.as_ref()) + ); + } + format!("{TIER_DELETE_JOURNAL_LEGACY_PREFIX}{}.json", rustfs_utils::crypto::hex(digest.as_ref())) +} + +fn tier_delete_journal_v6_object_name(je: &Jentry, operation_id: uuid::Uuid) -> String { + let digest = tier_delete_journal_identity_digest(je); + format!( + "{TIER_DELETE_JOURNAL_V6_PREFIX}{}/{}.json", + operation_id.simple(), + rustfs_utils::crypto::hex(digest.as_ref()) + ) +} + +fn tier_delete_journal_identity_digest(je: &Jentry) -> impl AsRef<[u8]> { let mut hasher = Sha256::new(); hasher.update(je.tier_name.as_bytes()); hasher.update([0]); @@ -270,10 +799,7 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String { hasher.update([0]); hasher.update(source.mod_time.as_deref().unwrap_or_default().as_bytes()); } - format!( - "{TIER_DELETE_JOURNAL_PREFIX}{}.json", - rustfs_utils::crypto::hex(hasher.finalize().as_slice()) - ) + hasher.finalize() } pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result { @@ -287,6 +813,1949 @@ pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result> { .map_err(|err| Error::other(format!("encode tier delete journal failed: {err}"))) } +async fn read_tier_delete_dispatch_manifest( + api: Arc, + object_name: &str, +) -> Result> { + match config_boundary::read_config_with_metadata(api, object_name, &ObjectOptions::default()).await { + Ok((data, metadata)) => { + let etag = metadata + .etag + .ok_or_else(|| Error::other("tier delete dispatch manifest has no entity tag"))?; + Ok(Some((decode_tier_delete_dispatch_manifest(&data, object_name)?, etag))) + } + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => Ok(None), + Err(err) => Err(err), + } +} + +async fn read_tier_delete_journal_with_etag(api: Arc, name: &str) -> Result> { + match config_boundary::read_config_with_metadata(api, name, &ObjectOptions::default()).await { + Ok((data, metadata)) => { + let etag = metadata + .etag + .ok_or_else(|| Error::other("tier delete journal has no entity tag"))?; + Ok(Some((decode_tier_delete_journal_entry(&data)?, etag))) + } + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => Ok(None), + Err(err) => Err(err), + } +} + +#[cfg(all(test, feature = "test-util"))] +async fn save_config_if_none(api: Arc, name: &str, data: Vec) -> Result<()> { + save_config_if_none_fenced(api, name, data, &|| true).await +} + +fn ensure_durable_write_fence(fences_current: &impl Fn() -> bool, edge: &str) -> Result<()> { + if fences_current() { + Ok(()) + } else { + Err(Error::other_with_context("durable ILM write fence changed", edge.to_owned())) + } +} + +#[derive(Debug)] +struct DecommissionCheckpointTargetsIncompleteError { + source: Error, +} + +impl std::fmt::Display for DecommissionCheckpointTargetsIncompleteError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "decommission durable ILM checkpoint was not confirmed on every receipt-bearing target: {}", + self.source + ) + } +} + +impl std::error::Error for DecommissionCheckpointTargetsIncompleteError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +fn decommission_checkpoint_targets_incomplete(error: Error) -> Error { + Error::other(DecommissionCheckpointTargetsIncompleteError { source: error }) +} + +fn is_decommission_checkpoint_targets_incomplete(error: &Error) -> bool { + matches!(error, Error::Io(io_error) if io_error + .get_ref() + .is_some_and(|source| source.is::())) +} + +#[cfg(all(test, feature = "test-util"))] +#[derive(Clone, Copy, PartialEq, Eq)] +enum TierDeleteDispatchRollbackTestStage { + Delete, + Confirmation, +} + +#[cfg(all(test, feature = "test-util"))] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TierDeleteDispatchMemberReadTestStage { + Validation, + Authorized, + Completed, + Completion, +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteDispatchMemberReadTestState { + stage: TierDeleteDispatchMemberReadTestStage, + pause_arrived: tokio::sync::Notify, + pause_release: CancellationToken, + entry_count: AtomicUsize, + authorized_progress_count: AtomicUsize, + in_flight: AtomicUsize, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteDispatchMemberReadTestHook { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteDispatchMemberReadTestPermit { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_DISPATCH_MEMBER_READ_TEST_HOOK: OnceLock>>> = + OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteDispatchMemberReadTestHook { + pub(crate) fn install_pause(stage: TierDeleteDispatchMemberReadTestStage) -> Self { + let state = Arc::new(TierDeleteDispatchMemberReadTestState { + stage, + pause_arrived: tokio::sync::Notify::new(), + pause_release: CancellationToken::new(), + entry_count: AtomicUsize::new(0), + authorized_progress_count: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + }); + let mut slot = TIER_DELETE_DISPATCH_MEMBER_READ_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch member read test hook should not poison"); + assert!(slot.is_none(), "tier delete dispatch member read test hook must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_read_pause_count(&self, expected: usize) { + while self.state.entry_count.load(Ordering::Acquire) < expected { + self.state.pause_arrived.notified().await; + } + } + + pub(crate) fn release_all_reads(&self) { + self.state.pause_release.cancel(); + } + + pub(crate) fn entry_count(&self) -> usize { + self.state.entry_count.load(Ordering::Acquire) + } + + pub(crate) fn authorized_progress_count(&self) -> usize { + self.state.authorized_progress_count.load(Ordering::Acquire) + } + + pub(crate) fn in_flight(&self) -> usize { + self.state.in_flight.load(Ordering::Acquire) + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchMemberReadTestHook { + fn drop(&mut self) { + self.state.pause_release.cancel(); + let mut slot = TIER_DELETE_DISPATCH_MEMBER_READ_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch member read test hook should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchMemberReadTestPermit { + fn drop(&mut self) { + let previous = self.state.in_flight.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "tier delete dispatch member read test hook underflow"); + } +} + +#[cfg(all(test, feature = "test-util"))] +async fn tier_delete_dispatch_member_read_test_hook( + stage: TierDeleteDispatchMemberReadTestStage, +) -> Option { + let state = TIER_DELETE_DISPATCH_MEMBER_READ_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch member read test hook should not poison") + .clone(); + let state = state.filter(|state| state.stage == stage)?; + state.entry_count.fetch_add(1, Ordering::AcqRel); + state.in_flight.fetch_add(1, Ordering::AcqRel); + state.pause_arrived.notify_waiters(); + state.pause_release.cancelled().await; + Some(TierDeleteDispatchMemberReadTestPermit { state }) +} + +#[cfg(all(test, feature = "test-util"))] +fn tier_delete_dispatch_authorized_progress_test_observed() { + let state = TIER_DELETE_DISPATCH_MEMBER_READ_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch member read test hook should not poison") + .clone(); + if let Some(state) = state.filter(|state| state.stage == TierDeleteDispatchMemberReadTestStage::Authorized) { + state.authorized_progress_count.fetch_add(1, Ordering::AcqRel); + } +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteDispatchRollbackTestState { + pause_delete_name: Option, + pause_all_deletes: bool, + pause_all_except_delete_name: Option, + watch_delete_name: Option, + fail_delete_name: Option, + fail_confirmation_name: Option, + pause_arrived: tokio::sync::Notify, + pause_release: tokio::sync::Notify, + pause_all_release: CancellationToken, + watch_arrived: tokio::sync::Notify, + pause_seen: AtomicBool, + pause_count: AtomicUsize, + delete_entry_count: AtomicUsize, + watch_seen: AtomicBool, + in_flight: AtomicUsize, + max_in_flight: AtomicUsize, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteDispatchRollbackTestHook { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteDispatchRollbackTestPermit { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_DISPATCH_ROLLBACK_TEST_HOOK: OnceLock>>> = + OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteDispatchRollbackTestHook { + fn install(state: TierDeleteDispatchRollbackTestState) -> Self { + let state = Arc::new(state); + let mut slot = TIER_DELETE_DISPATCH_ROLLBACK_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch rollback test hook should not poison"); + assert!(slot.is_none(), "tier delete dispatch rollback test hook must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn install_slow_delete(pause_delete_name: &str, watch_delete_name: &str) -> Self { + Self::install(TierDeleteDispatchRollbackTestState { + pause_delete_name: Some(pause_delete_name.to_string()), + pause_all_deletes: false, + pause_all_except_delete_name: None, + watch_delete_name: Some(watch_delete_name.to_string()), + fail_delete_name: None, + fail_confirmation_name: None, + pause_arrived: tokio::sync::Notify::new(), + pause_release: tokio::sync::Notify::new(), + pause_all_release: CancellationToken::new(), + watch_arrived: tokio::sync::Notify::new(), + pause_seen: AtomicBool::new(false), + pause_count: AtomicUsize::new(0), + delete_entry_count: AtomicUsize::new(0), + watch_seen: AtomicBool::new(false), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + }) + } + + pub(crate) fn install_delete_failure(fail_delete_name: &str) -> Self { + Self::install(TierDeleteDispatchRollbackTestState { + pause_delete_name: None, + pause_all_deletes: false, + pause_all_except_delete_name: None, + watch_delete_name: None, + fail_delete_name: Some(fail_delete_name.to_string()), + fail_confirmation_name: None, + pause_arrived: tokio::sync::Notify::new(), + pause_release: tokio::sync::Notify::new(), + pause_all_release: CancellationToken::new(), + watch_arrived: tokio::sync::Notify::new(), + pause_seen: AtomicBool::new(false), + pause_count: AtomicUsize::new(0), + delete_entry_count: AtomicUsize::new(0), + watch_seen: AtomicBool::new(false), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + }) + } + + pub(crate) fn install_confirmation_failure(fail_confirmation_name: &str) -> Self { + Self::install(TierDeleteDispatchRollbackTestState { + pause_delete_name: None, + pause_all_deletes: false, + pause_all_except_delete_name: None, + watch_delete_name: None, + fail_delete_name: None, + fail_confirmation_name: Some(fail_confirmation_name.to_string()), + pause_arrived: tokio::sync::Notify::new(), + pause_release: tokio::sync::Notify::new(), + pause_all_release: CancellationToken::new(), + watch_arrived: tokio::sync::Notify::new(), + pause_seen: AtomicBool::new(false), + pause_count: AtomicUsize::new(0), + delete_entry_count: AtomicUsize::new(0), + watch_seen: AtomicBool::new(false), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + }) + } + + pub(crate) fn install_pause_all_deletes() -> Self { + Self::install(TierDeleteDispatchRollbackTestState { + pause_delete_name: None, + pause_all_deletes: true, + pause_all_except_delete_name: None, + watch_delete_name: None, + fail_delete_name: None, + fail_confirmation_name: None, + pause_arrived: tokio::sync::Notify::new(), + pause_release: tokio::sync::Notify::new(), + pause_all_release: CancellationToken::new(), + watch_arrived: tokio::sync::Notify::new(), + pause_seen: AtomicBool::new(false), + pause_count: AtomicUsize::new(0), + delete_entry_count: AtomicUsize::new(0), + watch_seen: AtomicBool::new(false), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + }) + } + + pub(crate) fn install_pause_all_except_delete(delete_name: &str) -> Self { + Self::install(TierDeleteDispatchRollbackTestState { + pause_delete_name: None, + pause_all_deletes: false, + pause_all_except_delete_name: Some(delete_name.to_string()), + watch_delete_name: None, + fail_delete_name: None, + fail_confirmation_name: None, + pause_arrived: tokio::sync::Notify::new(), + pause_release: tokio::sync::Notify::new(), + pause_all_release: CancellationToken::new(), + watch_arrived: tokio::sync::Notify::new(), + pause_seen: AtomicBool::new(false), + pause_count: AtomicUsize::new(0), + delete_entry_count: AtomicUsize::new(0), + watch_seen: AtomicBool::new(false), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + }) + } + + pub(crate) async fn wait_until_delete_paused(&self) { + while !self.state.pause_seen.load(Ordering::Acquire) { + self.state.pause_arrived.notified().await; + } + } + + pub(crate) async fn wait_until_delete_observed(&self) { + while !self.state.watch_seen.load(Ordering::Acquire) { + self.state.watch_arrived.notified().await; + } + } + + pub(crate) async fn wait_until_delete_pause_count(&self, expected: usize) { + while self.state.pause_count.load(Ordering::Acquire) < expected { + self.state.pause_arrived.notified().await; + } + } + + pub(crate) fn release_delete(&self) { + self.state.pause_release.notify_one(); + } + + pub(crate) fn release_all_deletes(&self) { + self.state.pause_all_release.cancel(); + } + + pub(crate) fn delete_entry_count(&self) -> usize { + self.state.delete_entry_count.load(Ordering::Acquire) + } + + pub(crate) fn max_in_flight(&self) -> usize { + self.state.max_in_flight.load(Ordering::Acquire) + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchRollbackTestHook { + fn drop(&mut self) { + self.state.pause_release.notify_one(); + self.state.pause_all_release.cancel(); + let mut slot = TIER_DELETE_DISPATCH_ROLLBACK_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch rollback test hook should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchRollbackTestPermit { + fn drop(&mut self) { + let previous = self.state.in_flight.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "tier delete dispatch rollback test hook underflow"); + } +} + +#[cfg(all(test, feature = "test-util"))] +async fn tier_delete_dispatch_rollback_test_hook( + stage: TierDeleteDispatchRollbackTestStage, + name: &str, +) -> Result> { + let state = TIER_DELETE_DISPATCH_ROLLBACK_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete dispatch rollback test hook should not poison") + .clone(); + let Some(state) = state else { + return Ok(None); + }; + let current = state.in_flight.fetch_add(1, Ordering::AcqRel) + 1; + state.max_in_flight.fetch_max(current, Ordering::AcqRel); + let permit = TierDeleteDispatchRollbackTestPermit { state: state.clone() }; + + if stage == TierDeleteDispatchRollbackTestStage::Delete { + state.delete_entry_count.fetch_add(1, Ordering::AcqRel); + } + + if stage == TierDeleteDispatchRollbackTestStage::Delete && state.watch_delete_name.as_deref() == Some(name) { + state.watch_seen.store(true, Ordering::Release); + state.watch_arrived.notify_one(); + } + let pause_all = state.pause_all_deletes + || state + .pause_all_except_delete_name + .as_deref() + .is_some_and(|except| except != name); + if stage == TierDeleteDispatchRollbackTestStage::Delete && pause_all { + state.pause_count.fetch_add(1, Ordering::AcqRel); + state.pause_arrived.notify_waiters(); + state.pause_all_release.cancelled().await; + } else if stage == TierDeleteDispatchRollbackTestStage::Delete && state.pause_delete_name.as_deref() == Some(name) { + state.pause_seen.store(true, Ordering::Release); + state.pause_count.store(1, Ordering::Release); + state.pause_arrived.notify_one(); + state.pause_release.notified().await; + } + let fail = match stage { + TierDeleteDispatchRollbackTestStage::Delete => state.fail_delete_name.as_deref() == Some(name), + TierDeleteDispatchRollbackTestStage::Confirmation => state.fail_confirmation_name.as_deref() == Some(name), + }; + if fail { + return Err(Error::other(match stage { + TierDeleteDispatchRollbackTestStage::Delete => "injected tier delete dispatch rollback delete failure", + TierDeleteDispatchRollbackTestStage::Confirmation => "injected tier delete dispatch rollback confirmation failure", + })); + } + Ok(Some(permit)) +} + +#[cfg(all(test, feature = "test-util"))] +struct DecommissionCheckpointTargetFailureState { + target_pool_index: usize, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct DecommissionCheckpointTargetFailureHook { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static DECOMMISSION_CHECKPOINT_TARGET_FAILURE_HOOK: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl DecommissionCheckpointTargetFailureHook { + pub(crate) fn install(target_pool_index: usize) -> Self { + let state = Arc::new(DecommissionCheckpointTargetFailureState { target_pool_index }); + let mut slot = DECOMMISSION_CHECKPOINT_TARGET_FAILURE_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission checkpoint target failure hook should not poison"); + assert!(slot.is_none(), "decommission checkpoint target failure hook must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for DecommissionCheckpointTargetFailureHook { + fn drop(&mut self) { + let mut slot = DECOMMISSION_CHECKPOINT_TARGET_FAILURE_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission checkpoint target failure hook should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(all(test, feature = "test-util"))] +fn fail_decommission_checkpoint_target_for_test(target_pool_index: usize) -> bool { + DECOMMISSION_CHECKPOINT_TARGET_FAILURE_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission checkpoint target failure hook should not poison") + .as_ref() + .is_some_and(|state| state.target_pool_index == target_pool_index) +} + +async fn save_config_if_none_fenced( + api: Arc, + name: &str, + data: Vec, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + ensure_durable_write_fence(fences_current, "before If-None-Match save")?; + let write = config_boundary::save_config_with_opts( + api.clone(), + name, + data.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await; + ensure_durable_write_fence(fences_current, "during If-None-Match save")?; + match write { + Ok(()) => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + Err(Error::PreconditionFailed) => { + let observed = config_boundary::read_config(api.clone(), name).await; + ensure_durable_write_fence(fences_current, "during If-None-Match confirmation read")?; + match observed { + Ok(observed) if observed == data => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + _ => Err(Error::PreconditionFailed), + } + } + Err(write_err) => { + let observed = config_boundary::read_config(api.clone(), name).await; + ensure_durable_write_fence(fences_current, "during ambiguous If-None-Match confirmation read")?; + match observed { + Ok(observed) if observed == data => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + _ => Err(write_err), + } + } + } +} + +async fn save_decommission_manifest_checkpoint_if_match( + api: Arc, + name: &str, + next_data: &[u8], + etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result { + let Some(targets) = api.decommission_durable_ilm_checkpoint_targets(name, next_data, etag).await? else { + return Ok(false); + }; + let encoded_name = rustfs_utils::path::encode_dir_object(name); + let mut first_write_error = None; + for target in &targets { + ensure_durable_write_fence(fences_current, "before decommission checkpoint namespace lock")?; + let guards = api + .acquire_data_movement_publication_write_locks( + RUSTFS_META_BUCKET, + &encoded_name, + target.source_pool_index, + target.target_pool_index, + true, + ) + .await?; + ensure_durable_write_fence(fences_current, "before decommission checkpoint capacity admission")?; + if guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("decommission checkpoint publication lock was lost")); + } + + let pool = api.pools[target.target_pool_index].clone(); + let (observed_data, observed_metadata) = config_boundary::read_config_with_metadata( + pool.clone(), + name, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await?; + if observed_data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("decommission checkpoint target exceeds the manifest size limit")); + } + let observed_etag = observed_metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("decommission checkpoint target is missing an ETag"))?; + if observed_data.as_slice() == next_data { + if api + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await + { + api.reconcile_decommission_capacity_after_equivalent_temporary_target( + target.capacity_owner, + target.target_pool_index, + next_data.len(), + ) + .await?; + } + if guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("decommission checkpoint publication lock was lost during capacity replay")); + } + continue; + } + if target.already_committed || target.target_etag.as_deref() != Some(observed_etag.as_str()) { + first_write_error = Some(Error::PreconditionFailed); + break; + } + + let mut opts = ObjectOptions { + max_parity: true, + no_lock: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(observed_etag), + ..Default::default() + }), + ..Default::default() + }; + for guard in &guards { + guard.add_namespace_lock_fence(&mut opts); + } + target.capacity_owner.apply_to(&mut opts); + #[cfg(all(test, feature = "test-util"))] + if fail_decommission_checkpoint_target_for_test(target.target_pool_index) { + first_write_error = Some(Error::other_with_context( + "injected decommission checkpoint target failure", + format!("target pool {}", target.target_pool_index), + )); + break; + } + let write_data = next_data.to_vec(); + let write = api + .run_decommission_capacity_temporary_mutation_with_capacity_lease( + target.target_pool_index, + Some(target.capacity_owner), + Some(next_data.len()), + |capacity_lease| async move { + let mut opts = opts; + if let Some(signal) = capacity_lease.as_ref() { + opts.add_namespace_lock_lost_signal(signal.clone()); + if signal.is_lost() { + return Err(Error::other("decommission checkpoint capacity lease was lost before save")); + } + } + let result = config_boundary::save_config_with_opts(pool, name, write_data, &opts).await; + if capacity_lease.as_ref().is_some_and(|signal| signal.is_lost()) { + return Err(Error::other("decommission checkpoint capacity lease was lost during save")); + } + result + }, + ) + .await; + let publication_lost = guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost); + if let Err(err) = ensure_durable_write_fence(fences_current, "during decommission checkpoint save") { + first_write_error = Some(err); + break; + } + if publication_lost { + first_write_error = Some(Error::other("decommission checkpoint publication lock was lost during save")); + break; + } + if let Err(err) = write { + first_write_error = Some(err); + break; + } + drop(guards); + } + + if let Some(write_error) = first_write_error { + // A target PUT may have committed even when its caller observed an + // error. Confirm every receipt-bearing target independently; never let + // one globally visible copy advance receipts for a partial update. + for target in &targets { + ensure_durable_write_fence(fences_current, "before decommission checkpoint confirmation")?; + let guards = api + .acquire_data_movement_publication_write_locks( + RUSTFS_META_BUCKET, + &encoded_name, + target.source_pool_index, + target.target_pool_index, + true, + ) + .await?; + let observed = config_boundary::read_config_with_metadata( + api.pools[target.target_pool_index].clone(), + name, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await; + let Ok((observed_data, _)) = observed else { + return Err(write_error); + }; + if observed_data.as_slice() != next_data { + return Err(write_error); + } + if api + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await + { + api.reconcile_decommission_capacity_after_equivalent_temporary_target( + target.capacity_owner, + target.target_pool_index, + next_data.len(), + ) + .await?; + } + if guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("decommission checkpoint publication lock was lost during confirmation")); + } + } + } + Ok(true) +} + +async fn save_config_if_match_fenced( + api: Arc, + name: &str, + data: Vec, + etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + ensure_durable_write_fence(fences_current, "before If-Match save")?; + match save_decommission_manifest_checkpoint_if_match(api.clone(), name, &data, etag, fences_current).await { + Ok(true) => { + ensure_durable_write_fence(fences_current, "during decommission If-Match save")?; + return record_durable_config_progress_fenced(api, name, &data, fences_current).await; + } + Ok(false) => {} + Err(err) => return Err(decommission_checkpoint_targets_incomplete(err)), + } + let write = config_boundary::save_config_with_opts( + api.clone(), + name, + data.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(etag.to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await; + ensure_durable_write_fence(fences_current, "during If-Match save")?; + match write { + Ok(()) => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + Err(Error::PreconditionFailed) => { + let observed = config_boundary::read_config(api.clone(), name).await; + ensure_durable_write_fence(fences_current, "during If-Match confirmation read")?; + match observed { + Ok(observed) if observed == data => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + _ => Err(Error::PreconditionFailed), + } + } + Err(write_err) => { + let observed = config_boundary::read_config(api.clone(), name).await; + ensure_durable_write_fence(fences_current, "during ambiguous If-Match confirmation read")?; + match observed { + Ok(observed) if observed == data => record_durable_config_progress_fenced(api, name, &data, fences_current).await, + _ => Err(write_err), + } + } + } +} + +async fn record_durable_config_progress_fenced( + api: Arc, + name: &str, + data: &[u8], + fences_current: &impl Fn() -> bool, +) -> Result<()> { + // Keep this receipt update retryable independently from the config write: + // both save helpers recognize an already-persisted byte-identical target + // and repair a crash between the quorum write and receipt advancement. + ensure_durable_write_fence(fences_current, "before decommission progress receipt")?; + let result = Box::pin(api.record_durable_ilm_decommission_progress(name, data)).await; + ensure_durable_write_fence(fences_current, "during decommission progress receipt")?; + result +} + +async fn record_tier_delete_journal_progress_fenced( + api: Arc, + name: &str, + entry: &Jentry, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + record_durable_config_progress_fenced(api, name, &encode_tier_delete_journal_entry(entry)?, fences_current).await +} + +async fn record_tier_delete_dispatch_manifest_progress_fenced( + api: Arc, + name: &str, + manifest: &TierDeleteDispatchManifest, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + record_durable_config_progress_fenced(api, name, &encode_tier_delete_dispatch_manifest(manifest)?, fences_current).await +} + +async fn delete_durable_config_if_match( + api: Arc, + name: &str, + current_data: &[u8], + current_etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed before terminal receipt")); + } + // Terminal receipt is the durable authorization to remove every copy a + // pool decommission may have staged. It must precede the first delete. + let target_pool_indices = Box::pin(api.record_durable_ilm_decommission_terminal_target_pools(name, current_data)).await?; + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed after terminal receipt")); + } + let terminal_record = validate_durable_ilm_record(name, current_data)?; + if let Some(target_pool_indices) = target_pool_indices { + for target_pool_idx in target_pool_indices { + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed before target cleanup")); + } + match config_boundary::delete_config(api.pools[target_pool_idx].clone(), name).await { + Ok(()) | Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => {} + Err(err) => return Err(err), + } + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed during target cleanup")); + } + } + // The source pool remains the decommission owner's durable checkpoint + // until its verified cleanup consumes the terminal receipt. A global + // delete here would erase that source after only proving target-copy + // cleanup, defeating restart recovery. + return Ok(()); + } + + // A logical config key may be visible from more than one pool while a + // decommission is active. Delete one byte-identical ETag generation at a + // time and strongly re-read after every attempt; never delete a successor + // operation that reuses a manifest path. + let mut etag = current_etag.to_string(); + for _ in 0..api.pools.len().saturating_add(3) { + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed before ETag deletion")); + } + let delete = config_boundary::delete_config_if_match(api.clone(), name, &etag).await; + let observed = match config_boundary::read_config_with_metadata(api.clone(), name, &ObjectOptions::default()).await { + Ok(observed) => Some(observed), + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => None, + Err(err) => return Err(err), + }; + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed during ETag deletion")); + } + let Some((observed_data, metadata)) = observed else { + return Ok(()); + }; + let observed_etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("durable ILM record has no entity tag during terminal cleanup"))?; + if observed_data != current_data { + let observed_record = validate_durable_ilm_record(name, &observed_data)?; + let same_terminal_history = observed_record.namespace == terminal_record.namespace + && observed_record.id_kind == terminal_record.id_kind + && observed_record.id == terminal_record.id + && observed_record + .checkpoint + .is_predecessor_of_terminal(&terminal_record.checkpoint); + if !same_terminal_history + || !matches!( + &delete, + Ok(()) | Err(Error::ConfigNotFound) | Err(Error::FileNotFound) | Err(Error::PreconditionFailed) + ) + { + return Err(Error::PreconditionFailed); + } + // The exact terminal ETag was removed and exposed an older + // generation of the same immutable operation. The operation lock + // (manifest) or operation-scoped v6 path (journal) excludes ABA, + // so purge the remaining historical versions before confirming. + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed before history purge")); + } + match config_boundary::delete_config(api.clone(), name).await { + Ok(()) | Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => { + if !fences_current() { + return Err(Error::other("durable ILM terminal cleanup fence changed during history purge")); + } + continue; + } + Err(err) => return Err(err), + } + } + match delete { + Ok(()) | Err(Error::ConfigNotFound) | Err(Error::FileNotFound) | Err(Error::PreconditionFailed) => { + etag = observed_etag; + } + Err(_) if observed_etag != etag => etag = observed_etag, + Err(err) => return Err(err), + } + } + Err(Error::other("durable ILM terminal cleanup retained too many visible pool generations")) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn install_test_tier_delete_dispatch_fixture( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec<(Jentry, Option)>, + manifest_state: TierDeleteDispatchManifestState, +) -> Result<(String, Vec)> { + let fleet_proof = + acquire_tier_delete_journal_fleet_proof().ok_or_else(|| Error::other("test dispatch fixture requires a fleet proof"))?; + let topology_generation = tier_delete_journal_topology_generation(&fleet_proof); + let manifest_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let operation_id = uuid::Uuid::new_v4(); + let mut states_by_name = std::collections::BTreeMap::new(); + let raw_entries = entries + .into_iter() + .map(|(entry, state)| { + let name = tier_delete_journal_v6_object_name(&entry, operation_id); + if states_by_name.insert(name, state).is_some() { + return Err(Error::other("test dispatch fixture contains a duplicate journal name")); + } + Ok(entry) + }) + .collect::>>()?; + let journal_names = states_by_name.keys().cloned().collect::>(); + let manifest = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id, + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + journal_set_sha256: tier_delete_dispatch_journal_set_digest(&journal_names), + journal_count: journal_names.len() as u64, + journal_names, + topology_generation, + state: manifest_state, + }; + let mut bound = bind_dispatch_entries( + raw_entries, + manifest.operation_id, + &manifest_name, + &manifest.journal_set_sha256, + &manifest.topology_generation, + )?; + save_config_if_none(api.clone(), &manifest_name, encode_tier_delete_dispatch_manifest(&manifest)?).await?; + for entry in &mut bound { + let name = tier_delete_journal_object_name(entry); + let Some(state) = states_by_name + .remove(&name) + .ok_or_else(|| Error::other("test dispatch fixture state is missing"))? + else { + continue; + }; + entry.state = state; + save_config_if_none(api.clone(), &name, encode_tier_delete_journal_entry(entry)?).await?; + } + Ok((manifest_name, bound)) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn test_tier_delete_dispatch_manifest_state( + api: Arc, + manifest_name: &str, +) -> Result> { + Ok(read_tier_delete_dispatch_manifest(api, manifest_name) + .await? + .map(|(manifest, _)| manifest.state)) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn test_tier_delete_dispatch_manifest_checkpoint( + api: Arc, + manifest_name: &str, + next_state: TierDeleteDispatchManifestState, +) -> Result<(Vec, String)> { + let (mut manifest, etag) = read_tier_delete_dispatch_manifest(api, manifest_name) + .await? + .ok_or(Error::ConfigNotFound)?; + manifest.state = next_state; + Ok((encode_tier_delete_dispatch_manifest(&manifest)?, etag)) +} + +fn bind_dispatch_entries( + entries: Vec, + operation_id: uuid::Uuid, + manifest_object: &str, + journal_set_sha256: &str, + topology_generation: &str, +) -> Result> { + let mut entries = entries; + entries.sort_by_key(|entry| tier_delete_journal_v6_object_name(entry, operation_id)); + let mut bound = Vec::with_capacity(entries.len()); + let mut last_name: Option = None; + for mut entry in entries { + if !entry.can_replace_tier_free_version() { + return Err(Error::other( + "tier delete dispatch contains a transitioned source without a stable exact cleanup identity", + )); + } + let name = tier_delete_journal_v6_object_name(&entry, operation_id); + entry.persisted_version = 0; + entry.state = TierDeleteJournalState::Prepared; + entry.dispatch = Some(TierDeleteDispatchBinding { + operation_id, + manifest_object: manifest_object.to_string(), + journal_set_sha256: journal_set_sha256.to_string(), + topology_generation: topology_generation.to_string(), + }); + debug_assert_eq!(tier_delete_journal_object_name(&entry), name); + if last_name.as_ref() == Some(&name) { + let previous = bound + .last() + .ok_or_else(|| Error::other("tier delete dispatch duplicate accounting failed"))?; + if !same_tier_delete_journal_identity(previous, &entry) { + return Err(Error::other("tier delete dispatch contains conflicting duplicate journal names")); + } + continue; + } + last_name = Some(name); + bound.push(entry); + } + Ok(bound) +} + +fn tier_delete_dispatch_desired_names(entries: &[Jentry], operation_id: uuid::Uuid) -> Result> { + if entries.len() > MAX_TIER_DELETE_DISPATCH_JOURNALS { + return Err(Error::other("tier delete dispatch contains too many journals")); + } + let mut names = Vec::with_capacity(entries.len()); + for entry in entries { + if !entry.can_replace_tier_free_version() { + return Err(Error::other( + "tier delete dispatch contains a transitioned source without a stable exact cleanup identity", + )); + } + names.push(tier_delete_journal_v6_object_name(entry, operation_id)); + } + names.sort(); + names.dedup(); + Ok(names) +} + +fn validate_bound_journal(manifest: &TierDeleteDispatchManifest, name: &str, entry: &Jentry) -> Result<()> { + if entry.persisted_version != TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION + || tier_delete_journal_object_name(entry) != name + || entry.dispatch.as_ref() + != Some(&TierDeleteDispatchBinding { + operation_id: manifest.operation_id, + manifest_object: tier_delete_dispatch_manifest_object_name( + &manifest.bucket, + manifest.bucket_incarnation, + &manifest.prefix, + ), + journal_set_sha256: manifest.journal_set_sha256.clone(), + topology_generation: manifest.topology_generation.clone(), + }) + { + return Err(Error::other("tier delete journal does not match its dispatch manifest")); + } + Ok(()) +} + +async fn load_complete_dispatch_journal_set( + api: Arc, + manifest: &TierDeleteDispatchManifest, + allowed_states: &[TierDeleteJournalState], + fences_current: &impl Fn() -> bool, +) -> Result> { + let mut reads = futures::stream::iter((0..manifest.journal_names.len()).map(|index| { + let api = api.clone(); + let name = manifest.journal_names[index].clone(); + async move { + let (entry, _) = read_tier_delete_journal_with_etag(api.clone(), &name) + .await? + .ok_or_else(|| Error::other("tier delete dispatch manifest references a missing journal"))?; + validate_bound_journal(manifest, &name, &entry)?; + if !allowed_states.contains(&entry.state) { + return Err(Error::other("tier delete dispatch journal has an invalid state")); + } + record_tier_delete_journal_progress_fenced(api, &name, &entry, fences_current).await?; + Ok::<_, Error>((index, entry)) + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY); + let mut entries = vec![None; manifest.journal_names.len()]; + let mut first_error = None; + while let Some(result) = reads.next().await { + match result { + Ok((index, entry)) => entries[index] = Some(entry), + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + } + if let Some(err) = first_error { + return Err(err); + } + entries + .into_iter() + .collect::>>() + .ok_or_else(|| Error::other("tier delete dispatch journal validation produced an incomplete set")) +} + +async fn persist_prepared_dispatch_journal( + api: Arc, + manifest: &TierDeleteDispatchManifest, + entry: &Jentry, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + let name = tier_delete_journal_object_name(entry); + for _ in 0..4 { + if let Some((current, _)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? { + validate_bound_journal(manifest, &name, ¤t)?; + if !same_tier_delete_journal_identity(¤t, entry) { + return Err(Error::other("tier delete journal key is occupied by another cleanup identity")); + } + record_tier_delete_journal_progress_fenced(api.clone(), &name, ¤t, fences_current).await?; + return match current.state { + TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched => Ok(()), + TierDeleteJournalState::Committed => { + Err(Error::other("a completed tier delete journal cannot be rebound to a preparing operation")) + } + }; + } + match save_config_if_none_fenced(api.clone(), &name, encode_tier_delete_journal_entry(entry)?, fences_current).await { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + Err(Error::other("tier delete journal changed repeatedly during prepare")) +} + +async fn dispatch_prepared_journal( + api: Arc, + manifest: &TierDeleteDispatchManifest, + expected: &Jentry, + fences_current: &impl Fn() -> bool, +) -> Result { + let name = tier_delete_journal_object_name(expected); + for _ in 0..4 { + let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) + .await? + .ok_or_else(|| Error::other("prepared tier delete journal disappeared before dispatch"))?; + validate_bound_journal(manifest, &name, ¤t)?; + if !same_tier_delete_journal_identity(¤t, expected) { + return Err(Error::other("prepared tier delete journal changed identity before dispatch")); + } + match current.state { + TierDeleteJournalState::Dispatched => { + record_tier_delete_journal_progress_fenced(api.clone(), &name, ¤t, fences_current).await?; + return Ok(current); + } + TierDeleteJournalState::Committed => { + record_tier_delete_journal_progress_fenced(api.clone(), &name, ¤t, fences_current).await?; + return Err(Error::other("prepared tier delete journal was committed before dispatch authorization")); + } + TierDeleteJournalState::Prepared => {} + } + current.state = TierDeleteJournalState::Dispatched; + match save_config_if_match_fenced(api.clone(), &name, encode_tier_delete_journal_entry(¤t)?, &etag, fences_current) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + Err(Error::other("tier delete journal changed repeatedly during dispatch")) +} + +fn ensure_tier_delete_dispatch_member_scan_fence(fences_current: &impl Fn() -> bool) -> Result<()> { + if fences_current() { + Ok(()) + } else { + Err(Error::other("tier delete dispatch member scan fence changed")) + } +} + +async fn validate_staged_dispatch_journal_set( + api: Arc, + manifest: &TierDeleteDispatchManifest, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let stopped = Arc::new(AtomicBool::new(false)); + let make_read = |name: String| { + let api = api.clone(); + let stopped = stopped.clone(); + async move { + if stopped.load(Ordering::Acquire) { + return Ok::<_, Error>(()); + } + let result = async { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + #[cfg(all(test, feature = "test-util"))] + let _test_permit = + tier_delete_dispatch_member_read_test_hook(TierDeleteDispatchMemberReadTestStage::Validation).await; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let observed = read_tier_delete_journal_with_etag(api, &name).await?; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let Some((entry, _)) = observed else { + return Ok(()); + }; + validate_bound_journal(manifest, &name, &entry)?; + if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { + return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); + } + Ok(()) + } + .await; + if result.is_err() { + stopped.store(true, Ordering::Release); + } + result + } + }; + let mut next = 0; + let mut reads = futures::stream::FuturesUnordered::new(); + while next < manifest.journal_names.len() && reads.len() < TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY { + reads.push(make_read(manifest.journal_names[next].clone())); + next += 1; + } + let mut first_error = None; + while let Some(result) = reads.next().await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + if first_error.is_none() && !stopped.load(Ordering::Acquire) { + if let Err(err) = ensure_tier_delete_dispatch_member_scan_fence(fences_current) { + stopped.store(true, Ordering::Release); + first_error = Some(err); + } else if next < manifest.journal_names.len() { + reads.push(make_read(manifest.journal_names[next].clone())); + next += 1; + } + } + } + match first_error { + Some(err) => Err(err), + None => Ok(()), + } +} + +async fn delete_staged_dispatch_journal_set( + api: Arc, + manifest: &TierDeleteDispatchManifest, + fences_current: &F, +) -> Result<()> +where + F: Fn() -> bool + Sync, +{ + // Validate the complete immutable set before deleting its first member so + // a corrupt binding or impossible Committed state quarantines the whole + // operation rather than producing a partial rollback. + validate_staged_dispatch_journal_set(api.clone(), manifest, fences_current).await?; + + // The validation barrier above must drain before this stream is created. + // Once deletion starts, stop admitting useful work after the first error + // but drain the at-most-bounded in-flight set so no detached mutation + // escapes the manifest operation/fleet fences. + let delete_stopped = Arc::new(AtomicBool::new(false)); + let mut deletes = futures::stream::iter(manifest.journal_names.iter().cloned().map(|name| { + let api = api.clone(); + let delete_stopped = delete_stopped.clone(); + async move { + if delete_stopped.load(Ordering::Acquire) { + return Ok(()); + } + let result = async { + #[cfg(all(test, feature = "test-util"))] + let _test_permit = + tier_delete_dispatch_rollback_test_hook(TierDeleteDispatchRollbackTestStage::Delete, &name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + let Some((entry, etag)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? else { + return Ok(()); + }; + validate_bound_journal(manifest, &name, &entry)?; + if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { + return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); + } + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + let data = encode_tier_delete_journal_entry(&entry)?; + match delete_durable_config_if_match(api, &name, &data, &etag, fences_current).await { + Ok(()) | Err(Error::ConfigNotFound) => Ok(()), + Err(Error::PreconditionFailed) => Err(Error::other("tier delete dispatch journal changed during rollback")), + Err(err) => Err(err), + } + } + .await; + if result.is_err() { + delete_stopped.store(true, Ordering::Release); + } + result + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY); + let mut first_error = None; + while let Some(result) = deletes.next().await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + } + drop(deletes); + if let Some(err) = first_error { + return Err(err); + } + + let confirmation_stopped = Arc::new(AtomicBool::new(false)); + let mut confirmations = futures::stream::iter(manifest.journal_names.iter().cloned().map(|name| { + let api = api.clone(); + let confirmation_stopped = confirmation_stopped.clone(); + async move { + if confirmation_stopped.load(Ordering::Acquire) { + return Ok(()); + } + let result = async { + #[cfg(all(test, feature = "test-util"))] + let _test_permit = + tier_delete_dispatch_rollback_test_hook(TierDeleteDispatchRollbackTestStage::Confirmation, &name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + if let Some((entry, _)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? { + let data = encode_tier_delete_journal_entry(&entry)?; + if !api.durable_ilm_terminal_receipt_covers_active_source(&name, &data).await? { + return Err(Error::other("tier delete dispatch rollback retained a journal")); + } + } + Ok(()) + } + .await; + if result.is_err() { + confirmation_stopped.store(true, Ordering::Release); + } + result + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY); + let mut first_error = None; + while let Some(result) = confirmations.next().await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + } + drop(confirmations); + match first_error { + Some(err) => Err(err), + None => Ok(()), + } +} + +async fn seal_and_rollback_preparing_dispatch( + api: Arc, + expected: &TierDeleteDispatchManifest, + fleet_proof: &TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, +) -> Result<()> { + let manifest_name = + tier_delete_dispatch_manifest_object_name(&expected.bucket, expected.bucket_incarnation, &expected.prefix); + let fences_current = + || dispatch_write_fences_current(bucket_fence, operation_guard, fleet_proof, &expected.topology_generation); + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + let Some((mut current, etag)) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? else { + return Ok(()); + }; + if current.operation_id != expected.operation_id + || current.journal_set_sha256 != expected.journal_set_sha256 + || current.state != TierDeleteDispatchManifestState::Preparing + { + return Err(Error::other( + "tier delete dispatch cannot be rolled back after authorization or identity change", + )); + } + current.state = TierDeleteDispatchManifestState::Aborting; + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + save_config_if_match_fenced( + api.clone(), + &manifest_name, + encode_tier_delete_dispatch_manifest(¤t)?, + &etag, + &fences_current, + ) + .await?; + let (mut sealed, sealed_etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + .await? + .ok_or_else(|| Error::other("sealed tier delete dispatch manifest disappeared"))?; + if sealed.operation_id != expected.operation_id || sealed.state != TierDeleteDispatchManifestState::Aborting { + return Err(Error::other("tier delete dispatch rollback seal changed")); + } + delete_staged_dispatch_journal_set(api.clone(), &sealed, &fences_current).await?; + sealed.state = TierDeleteDispatchManifestState::Aborted; + if !fences_current() { + return Err(Error::other("tier delete dispatch rollback fence changed")); + } + save_config_if_match_fenced( + api, + &manifest_name, + encode_tier_delete_dispatch_manifest(&sealed)?, + &sealed_etag, + &fences_current, + ) + .await +} + +async fn authorized_dispatch_permit( + api: Arc, + manifest_name: &str, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, +) -> Result { + let (manifest, authorized_etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) + .await? + .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared"))?; + if manifest.state != TierDeleteDispatchManifestState::DispatchAuthorized + || !tier_delete_journal_fleet_proof_matches(&fleet_proof) + || tier_delete_journal_topology_generation(&fleet_proof) != manifest.topology_generation + { + return Err(Error::other("tier delete dispatch authorization is stale or unconfirmed")); + } + let entries = { + let fences_current = + || dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation); + record_tier_delete_dispatch_manifest_progress_fenced(api.clone(), manifest_name, &manifest, &fences_current).await?; + load_complete_dispatch_journal_set( + api, + &manifest, + &[TierDeleteJournalState::Dispatched, TierDeleteJournalState::Committed], + &fences_current, + ) + .await? + }; + Ok(PreparedTierDeleteDispatch { + permit: Some(DispatchedJournalPermit { + manifest, + authorized_etag, + entries, + fleet_proof, + }), + predecessor_replay_required: false, + }) +} + +pub(crate) async fn prepare_tier_delete_dispatch( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + Box::pin(prepare_tier_delete_dispatch_inner( + api, + bucket, + bucket_incarnation, + prefix, + entries, + fleet_proof, + bucket_fence, + )) + .await +} + +async fn prepare_tier_delete_dispatch_inner( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + if bucket_incarnation.is_nil() || bucket_fence.is_lock_lost() || !tier_delete_journal_fleet_proof_matches(&fleet_proof) { + return Err(Error::other("tier delete journal v6 fleet capability is unavailable")); + } + let topology_generation = tier_delete_journal_topology_generation(&fleet_proof); + let manifest_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + + let (mut manifest, manifest_etag) = loop { + ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? { + Some((existing, etag)) => { + if existing.bucket != bucket + || existing.bucket_incarnation != bucket_incarnation + || existing.prefix != prefix + || existing.topology_generation != topology_generation + { + return Err(Error::other("an incompatible tier delete dispatch operation already owns this prefix")); + } + { + let fences_current = || { + dispatch_write_fences_current(bucket_fence, &operation_guard, &fleet_proof, &existing.topology_generation) + }; + record_tier_delete_dispatch_manifest_progress_fenced(api.clone(), &manifest_name, &existing, &fences_current) + .await?; + } + let desired_names = tier_delete_dispatch_desired_names(&entries, existing.operation_id)?; + let desired_digest = tier_delete_dispatch_journal_set_digest(&desired_names); + match existing.state { + TierDeleteDispatchManifestState::Preparing => { + if existing.journal_names != desired_names || existing.journal_set_sha256 != desired_digest { + return Err(Error::other("preparing tier delete dispatch no longer matches the complete source set")); + } + break (existing, etag); + } + TierDeleteDispatchManifestState::DispatchAuthorized => { + let predecessor_replay_required = desired_names + .iter() + .any(|name| existing.journal_names.binary_search(name).is_err()); + ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + let mut prepared = + authorized_dispatch_permit(api, &manifest_name, fleet_proof, bucket_fence, &operation_guard).await?; + prepared.predecessor_replay_required = predecessor_replay_required; + return Ok(prepared); + } + TierDeleteDispatchManifestState::Completed => { + return Err(Error::other("a completed tier delete dispatch is still awaiting durable cleanup")); + } + TierDeleteDispatchManifestState::Aborting => { + return Err(Error::other("a tier delete dispatch rollback is still in progress")); + } + TierDeleteDispatchManifestState::Aborted => { + for name in &existing.journal_names { + if read_tier_delete_journal_with_etag(api.clone(), name).await?.is_some() { + return Err(Error::other("an aborted tier delete dispatch still owns journal records")); + } + } + let data = encode_tier_delete_dispatch_manifest(&existing)?; + let fences_current = || { + !bucket_fence.is_lock_lost() + && !operation_guard.is_lock_lost() + && tier_delete_journal_fleet_proof_matches(&fleet_proof) + && tier_delete_journal_topology_generation(&fleet_proof) == existing.topology_generation + }; + match delete_durable_config_if_match(api.clone(), &manifest_name, &data, &etag, &fences_current).await { + Ok(()) | Err(Error::ConfigNotFound) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + } + } + None => { + let operation_id = uuid::Uuid::new_v4(); + let desired_names = tier_delete_dispatch_desired_names(&entries, operation_id)?; + let desired_digest = tier_delete_dispatch_journal_set_digest(&desired_names); + let new_manifest = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id, + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + journal_names: desired_names.clone(), + journal_set_sha256: desired_digest.clone(), + journal_count: desired_names.len() as u64, + topology_generation: topology_generation.clone(), + state: TierDeleteDispatchManifestState::Preparing, + }; + let fences_current = + || dispatch_write_fences_current(bucket_fence, &operation_guard, &fleet_proof, &topology_generation); + match save_config_if_none_fenced( + api.clone(), + &manifest_name, + encode_tier_delete_dispatch_manifest(&new_manifest)?, + &fences_current, + ) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + } + }; + + let bound = bind_dispatch_entries( + entries, + manifest.operation_id, + &manifest_name, + &manifest.journal_set_sha256, + &manifest.topology_generation, + )?; + let attempt = async { + let fences_current = + || dispatch_write_fences_current(bucket_fence, &operation_guard, &fleet_proof, &manifest.topology_generation); + let manifest_ref = &manifest; + let operation_guard_ref = &operation_guard; + let prepare_stopped = Arc::new(AtomicBool::new(false)); + let mut prepare_writes = futures::stream::iter((0..bound.len()).map(|index| { + let api = api.clone(); + let prepare_stopped = prepare_stopped.clone(); + let fences_current = &fences_current; + let manifest = manifest_ref; + let operation_guard = operation_guard_ref; + let entry = bound[index].clone(); + async move { + if prepare_stopped.load(Ordering::Acquire) { + return Ok(()); + } + let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { + Ok(()) => persist_prepared_dispatch_journal(api, manifest, &entry, fences_current).await, + Err(err) => Err(err), + }; + if result.is_err() { + prepare_stopped.store(true, Ordering::Release); + } + result + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY); + let mut prepare_error = None; + while let Some(result) = prepare_writes.next().await { + if let Err(err) = result + && prepare_error.is_none() + { + prepare_error = Some(err); + } + } + drop(prepare_writes); + if let Some(err) = prepare_error { + return Err(err); + } + + // Preserve the phase barrier: every Prepared write is durable before + // any journal is advanced to Dispatched. On the first failure, stop + // admitting new writes but drain the already-started bounded set + // before the caller seals and rolls the operation back. + let dispatch_stopped = Arc::new(AtomicBool::new(false)); + let mut dispatch_writes = futures::stream::iter((0..bound.len()).map(|index| { + let api = api.clone(); + let dispatch_stopped = dispatch_stopped.clone(); + let fences_current = &fences_current; + let manifest = manifest_ref; + let operation_guard = operation_guard_ref; + let entry = bound[index].clone(); + async move { + if dispatch_stopped.load(Ordering::Acquire) { + return Ok(()); + } + let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { + Ok(()) => dispatch_prepared_journal(api, manifest, &entry, fences_current) + .await + .map(|_| ()), + Err(err) => Err(err), + }; + if result.is_err() { + dispatch_stopped.store(true, Ordering::Release); + } + result + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_CAS_CONCURRENCY); + let mut dispatch_error = None; + while let Some(result) = dispatch_writes.next().await { + if let Err(err) = result + && dispatch_error.is_none() + { + dispatch_error = Some(err); + } + } + drop(dispatch_writes); + if let Some(err) = dispatch_error { + return Err(err); + } + ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + if !tier_delete_journal_fleet_proof_matches(&fleet_proof) + || tier_delete_journal_topology_generation(&fleet_proof) != manifest.topology_generation + { + return Err(Error::other("tier delete dispatch fleet proof changed before authorization")); + } + manifest.state = TierDeleteDispatchManifestState::DispatchAuthorized; + let authorized_data = encode_tier_delete_dispatch_manifest(&manifest)?; + match save_config_if_match_fenced(api.clone(), &manifest_name, authorized_data, &manifest_etag, &fences_current).await { + Ok(()) => {} + Err(Error::PreconditionFailed) => { + return Err(Error::other("tier delete dispatch manifest changed before authorization")); + } + Err(cas_err) if is_decommission_checkpoint_targets_incomplete(&cas_err) => return Err(cas_err), + Err(cas_err) => { + // The write may have reached quorum even if the client saw a + // timeout. Only a strong read confirming Authorized permits + // mutation; every other outcome is retained for recovery. + match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await { + Ok(Some((observed, _))) + if observed.operation_id == manifest.operation_id + && observed.state == TierDeleteDispatchManifestState::DispatchAuthorized => {} + _ => return Err(cas_err), + } + } + } + Ok(()) + } + .await; + + if let Err(err) = attempt { + let authorized = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + .await + .ok() + .flatten() + .is_some_and(|(current, _)| { + current.operation_id == manifest.operation_id + && current.state == TierDeleteDispatchManifestState::DispatchAuthorized + }); + if !authorized + && let Err(rollback_err) = + seal_and_rollback_preparing_dispatch(api.clone(), &manifest, &fleet_proof, bucket_fence, &operation_guard).await + { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + operation_id = %manifest.operation_id, + error = ?rollback_err, + "Tier delete dispatch preparation failed and its rollback remains durable for recovery" + ); + } + return Err(err); + } + + // Re-read the exact Authorized manifest and every bound journal before + // constructing the private one-shot permit. + ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + authorized_dispatch_permit(api, &manifest_name, fleet_proof, bucket_fence, &operation_guard).await +} + +async fn commit_dispatched_journal( + api: Arc, + manifest: &TierDeleteDispatchManifest, + expected: &Jentry, + authorization: &TierDeleteDispatchAuthorization, + fences_current: &impl Fn() -> bool, +) -> Result { + let name = tier_delete_journal_object_name(expected); + #[cfg(all(test, feature = "test-util"))] + let _test_permit = tier_delete_dispatch_member_read_test_hook(TierDeleteDispatchMemberReadTestStage::Completion).await; + authorization.ensure_current(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix)?; + for _ in 0..4 { + authorization.ensure_current(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix)?; + let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) + .await? + .ok_or_else(|| Error::other("dispatched tier delete journal disappeared before commit"))?; + validate_bound_journal(manifest, &name, ¤t)?; + if !same_tier_delete_journal_identity(¤t, expected) { + return Err(Error::other("dispatched tier delete journal changed identity before commit")); + } + match current.state { + TierDeleteJournalState::Committed => { + record_tier_delete_journal_progress_fenced(api.clone(), &name, ¤t, fences_current).await?; + authorization.ensure_current(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix)?; + return Ok(current); + } + TierDeleteJournalState::Prepared => { + return Err(Error::other("prepared tier delete journal was never dispatch-authorized")); + } + TierDeleteJournalState::Dispatched => {} + } + current.state = TierDeleteJournalState::Committed; + authorization.ensure_current(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix)?; + let result = + save_config_if_match_fenced(api.clone(), &name, encode_tier_delete_journal_entry(¤t)?, &etag, fences_current) + .await; + authorization.ensure_current(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix)?; + match result { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + Err(Error::other("tier delete journal changed repeatedly during commit")) +} + +pub(crate) async fn complete_tier_delete_dispatch( + api: Arc, + active: &ActiveTierDeleteDispatch, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result<()> { + active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; + if !active.authorization.mutation_started() { + return Err(Error::other("tier delete dispatch cannot commit before its local mutation starts")); + } + let manifest_name = tier_delete_dispatch_manifest_object_name( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + ); + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let fences_current = || { + !bucket_fence.is_lock_lost() + && !operation_guard.is_lock_lost() + && active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) + .is_ok() + }; + ensure_durable_write_fence(&fences_current, "before tier delete dispatch completion")?; + let make_commit = |index: usize| { + let api = api.clone(); + let fences_current = &fences_current; + async move { + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + commit_dispatched_journal(api, &active.manifest, &active.entries[index], &active.authorization, fences_current) + .await + .map(|entry| (index, entry)) + } + }; + let mut next = 0; + let mut commits = futures::stream::FuturesUnordered::new(); + while next < active.entries.len() && commits.len() < TIER_DELETE_DISPATCH_CAS_CONCURRENCY { + commits.push(make_commit(next)); + next += 1; + } + let mut committed_entries = vec![None; active.entries.len()]; + let mut first_error = None; + while let Some(result) = commits.next().await { + match result { + Ok((index, entry)) => committed_entries[index] = Some(entry), + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + // A failed member CAS leaves the Authorized manifest and every + // already-committed member recoverable. Stop admitting tail work, but + // drain the bounded in-flight set before releasing the operation lock. + if first_error.is_none() && next < active.entries.len() { + commits.push(make_commit(next)); + next += 1; + } + } + if let Some(err) = first_error { + return Err(err); + } + let committed_entries = committed_entries + .into_iter() + .collect::>>() + .ok_or_else(|| Error::other("tier delete dispatch completion omitted a journal"))?; + + active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; + let (mut current, etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + .await? + .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared before completion"))?; + if current.operation_id != active.manifest.operation_id + || current.journal_set_sha256 != active.manifest.journal_set_sha256 + || current.state != TierDeleteDispatchManifestState::DispatchAuthorized + || (etag != active.authorized_etag && current != active.manifest) + { + return Err(Error::other("tier delete dispatch manifest changed before completion")); + } + current.state = TierDeleteDispatchManifestState::Completed; + active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; + let completion = save_config_if_match_fenced( + api.clone(), + &manifest_name, + encode_tier_delete_dispatch_manifest(¤t)?, + &etag, + &fences_current, + ) + .await; + active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; + completion?; + + for entry in committed_entries { + if let Err(err) = enqueue_committed_tier_delete_journal_entry(&entry).await { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + operation_id = %current.operation_id, + error = ?err, + "Committed tier delete dispatch will be picked up by periodic recovery" + ); + } + } + + Ok(()) +} + pub fn record_tier_delete_journal_backend_identity( je: &mut Jentry, metadata: &std::collections::HashMap, @@ -310,11 +2779,59 @@ where >, { let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?; + if je.state == TierDeleteJournalState::Prepared && je.source.is_some() { + let name = tier_delete_journal_object_name(je); + for _ in 0..3 { + match config_boundary::save_config_with_opts( + api.clone(), + &name, + data.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) => { + let current = match config_boundary::read_config(api.clone(), &name).await { + Ok(current) => decode_tier_delete_journal_entry(¤t).map_err(std::io::Error::other)?, + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => continue, + Err(err) => return Err(std::io::Error::other(err)), + }; + if !same_tier_delete_journal_identity(¤t, je) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete journal key is occupied by a different transaction", + )); + } + // A replayed Prepare is idempotent. In particular, never + // regress a concurrently committed transaction back to + // Prepared. + return Ok(()); + } + Err(err) => return Err(std::io::Error::other(err)), + } + } + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete journal changed repeatedly during prepare", + )); + } config_boundary::save_config(api, &tier_delete_journal_object_name(je), data) .await .map_err(std::io::Error::other) } +#[allow( + dead_code, + reason = "legacy v1-v5 journal writer retained for rollback/source compatibility" +)] pub async fn commit_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> where S: ObjectIO< @@ -327,9 +2844,105 @@ where PutObjectReader = PutObjReader, >, { + let name = tier_delete_journal_object_name(je); let mut committed = je.clone(); committed.state = TierDeleteJournalState::Committed; - persist_tier_delete_journal_entry(api, &committed).await + let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?; + + for _ in 0..3 { + let (current, etag) = + match config_boundary::read_config_with_metadata(api.clone(), &name, &ObjectOptions::default()).await { + Ok((current, metadata)) => ( + Some(decode_tier_delete_journal_entry(¤t).map_err(std::io::Error::other)?), + metadata.etag, + ), + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => (None, None), + Err(err) => return Err(std::io::Error::other(err)), + }; + + if let Some(current) = current { + if !same_tier_delete_journal_identity(¤t, je) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete journal changed identity before commit", + )); + } + if current.state == TierDeleteJournalState::Committed { + return Ok(()); + } + let etag = etag.ok_or_else(|| std::io::Error::other("prepared tier delete journal has no entity tag"))?; + match config_boundary::save_config_with_opts( + api.clone(), + &name, + data.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => continue, + Err(err) => return Err(std::io::Error::other(err)), + } + } else { + // Another participant may have aborted the shared Prepared key + // while this participant committed locally. Recreate only this + // exact committed identity; its all-pool live-source gate still + // prevents an early remote deletion. + match config_boundary::save_config_with_opts( + api.clone(), + &name, + data.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(std::io::Error::other(err)), + } + } + } + Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete journal changed repeatedly during commit", + )) +} + +fn same_tier_delete_journal_identity(left: &Jentry, right: &Jentry) -> bool { + left.obj_name == right.obj_name + && left.version_id == right.version_id + && left.tier_name == right.tier_name + && left.backend_identity == right.backend_identity + && left.version_id_exact == right.version_id_exact + && left.version_state == right.version_state + && left.source == right.source + && left.dispatch == right.dispatch +} + +fn same_tier_delete_authorization_identity(expected: &Jentry, candidate: &Jentry) -> bool { + let mut candidate = candidate.clone(); + if let (Some(expected_source), Some(candidate_source)) = (expected.source.as_ref(), candidate.source.as_mut()) { + // Bucket versioning can change between the Authorized attempt and its + // replay. The persisted predecessor mode remains authoritative; every + // physical source, remote tuple, and dispatch field must still match. + candidate_source.versioned = expected_source.versioned; + candidate_source.version_suspended = expected_source.version_suspended; + } + same_tier_delete_journal_identity(expected, &candidate) } #[allow( @@ -350,6 +2963,10 @@ where remove_tier_delete_journal_entry(api, je).await } +#[allow( + dead_code, + reason = "legacy v1-v5 prepared-journal cleanup retained for rollback/source compatibility" +)] pub async fn abort_prepared_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> { let name = tier_delete_journal_object_name(je); let (data, metadata) = match config_boundary::read_config_with_metadata(api.clone(), &name, &ObjectOptions::default()).await { @@ -358,6 +2975,12 @@ pub async fn abort_prepared_tier_delete_journal_entry(api: Arc, je: &Je Err(err) => return Err(std::io::Error::other(err)), }; let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?; + if !same_tier_delete_journal_identity(¤t, je) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "prepared tier delete journal changed identity before abort", + )); + } if current.state != TierDeleteJournalState::Prepared { return Ok(()); } @@ -397,58 +3020,74 @@ where } pub async fn process_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> { - if je.state == TierDeleteJournalState::Prepared { - return reconcile_prepared_tier_delete_journal_entry(api, je).await; - } - process_committed_tier_delete_journal_entry(api, je).await -} - -async fn process_committed_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> { - if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "tier delete journal remote version state is unknown", - )); - } - let backend_identity = je - .backend_identity - .ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?; - if je.version_id_exact { - delete_confirmed_transition_candidate_exact_with_manager_and_identity( - &je.obj_name, - &je.version_id, - &je.tier_name, - backend_identity, - &api.tier_config_mgr(), - ) - .await?; - } else { - delete_object_from_remote_tier_idempotent_with_manager_and_identity( - &je.obj_name, - &je.version_id, - &je.tier_name, - backend_identity, - &api.tier_config_mgr(), - false, - ) - .await?; - } - let path = tier_delete_journal_object_name(je); - let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?; - let target_pool_indices = api - .record_durable_ilm_decommission_terminal_target_pools(&path, &data) + let journal_name = tier_delete_journal_object_name(je); + // The lock lives at a synthetic namespace key so config I/O on the journal + // object can still acquire its normal object lock. It serializes the + // strong-read -> remote DELETE -> ETag-delete transaction across recovery + // workers and nodes, avoiding avoidable duplicate backend calls. + let recovery_lock_name = format!("{journal_name}.recovery-lock"); + let recovery_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &recovery_lock_name) .await .map_err(std::io::Error::other)?; - if let Some(target_pool_indices) = target_pool_indices { - for target_pool_idx in target_pool_indices { - match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await { - Ok(()) | Err(Error::ConfigNotFound) => {} - Err(err) => return Err(std::io::Error::other(err)), - } - } - return Ok(()); + let _recovery_guard = recovery_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await + .map_err(std::io::Error::other)?; + let (current, journal_etag) = read_tier_delete_journal_with_etag(api.clone(), &journal_name) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "tier delete journal disappeared"))?; + if !same_tier_delete_journal_identity(¤t, je) { + metrics::counter!( + "rustfs_ilm_tier_delete_journal_quarantined_total", + "reason" => "identity_mismatch" + ) + .increment(1); + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "mismatched tier delete journal is quarantined", + )); + } + match current.persisted_version { + 1 | TIER_DELETE_JOURNAL_VERSION => { + metrics::counter!( + "rustfs_ilm_tier_delete_journal_quarantined_total", + "reason" => "legacy_unknown_version_state" + ) + .increment(1); + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "v1/v2 tier delete journal with unknown remote version state is quarantined", + )); + } + TIER_DELETE_JOURNAL_EXACT_VERSION | TIER_DELETE_JOURNAL_STATE_VERSION => { + return process_committed_v3_v4_journal(api, current, journal_etag).await; + } + TIER_DELETE_JOURNAL_TRANSACTION_VERSION => { + return process_v5_journal(api, current, journal_etag).await; + } + TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION => {} + _ => { + metrics::counter!( + "rustfs_ilm_tier_delete_journal_quarantined_total", + "reason" => "unsupported_version" + ) + .increment(1); + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "unsupported tier delete journal version is quarantined", + )); + } + } + let manifest = load_manifest_for_journal(api.clone(), &journal_name, ¤t) + .await + .map_err(std::io::Error::other)?; + match current.state { + TierDeleteJournalState::Prepared => reconcile_prepared_v6_journal(api, current, journal_etag, manifest).await, + TierDeleteJournalState::Dispatched => reconcile_dispatched_v6_journal(api, current, journal_etag, manifest).await, + TierDeleteJournalState::Committed => process_committed_v6_journal(api, current, journal_etag, manifest).await, } - remove_tier_delete_journal_entry(api, je).await } fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io::Result { @@ -474,7 +3113,7 @@ fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io }) } -async fn prepared_tier_delete_has_live_source( +async fn tier_delete_has_durable_source_or_free_version( api: &ECStore, source: &TierDeleteSourceIdentity, je: &Jentry, @@ -482,7 +3121,7 @@ async fn prepared_tier_delete_has_live_source( let lock_object = rustfs_utils::path::encode_dir_object(&source.object); let mut lock_opts = ObjectOptions::default(); let read_guards = api - .acquire_all_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts) + .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts) .await .map_err(std::io::Error::other)?; if api.ctx.lock_manager().is_disabled() { @@ -491,24 +3130,29 @@ async fn prepared_tier_delete_has_live_source( "tier delete journal recovery requires namespace locking", )); } - let mut has_live_source = false; + let mut has_durable_owner = false; for pool in &api.pools { - let set = pool.get_disks_by_key(&lock_object); - let Some(versions) = set - .load_file_info_versions_exact(&source.bucket, &source.object) - .await - .map_err(std::io::Error::other)? - else { - continue; - }; - for version in versions.versions.iter().filter(|version| !version.tier_free_version()) { - let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned); - if object_info_references_tier_delete(&info, je)? { - has_live_source = true; + for set in &pool.disk_set { + let versions = match set.load_file_info_versions_exact(&source.bucket, &source.object).await { + Ok(Some(versions)) => versions, + Ok(None) => continue, + // A quorum-confirmed missing physical volume proves absence in + // this set. Every other read/parse/quorum outcome is Unknown. + Err(err) if is_err_strict_volume_not_found(&err) => continue, + Err(err) => return Err(std::io::Error::other(err)), + }; + for version in versions.versions.iter().chain(versions.free_versions.iter()) { + let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned); + if object_info_references_tier_delete(&info, je)? { + has_durable_owner = true; + break; + } + } + if has_durable_owner { break; } } - if has_live_source { + if has_durable_owner { break; } } @@ -518,115 +3162,1239 @@ async fn prepared_tier_delete_has_live_source( "tier delete journal recovery object read lock was lost", )); } - Ok((has_live_source, read_guards)) + Ok((has_durable_owner, read_guards)) } -async fn reconcile_prepared_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> { - let journal_name = tier_delete_journal_object_name(je); - let (data, metadata) = config_boundary::read_config_with_metadata(api.clone(), &journal_name, &ObjectOptions::default()) +async fn delete_remote_tier_journal_target_with_lease( + journal: &Jentry, + lease: &crate::services::tier::tier::TierOperationLease, +) -> std::io::Result<()> { + let delete = async { + if journal.version_id_exact { + delete_confirmed_transition_candidate_exact_with_lease_idempotent(&journal.obj_name, &journal.version_id, lease).await + } else { + delete_object_from_remote_tier_with_lease_idempotent(&journal.obj_name, &journal.version_id, lease, false).await + } + }; + tokio::time::timeout(TIER_DELETE_REMOTE_DEADLINE, delete) .await - .map_err(std::io::Error::other)?; - let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?; - if tier_delete_journal_object_name(¤t) != journal_name { + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "remote tier delete exceeded its deadline"))??; + Ok(()) +} + +async fn process_committed_v3_v4_journal(api: Arc, current: Jentry, etag: String) -> std::io::Result<()> { + if current.state != TierDeleteJournalState::Committed + || current.version_state == rustfs_filemeta::TransitionVersionState::Unknown + { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - "prepared tier delete journal content does not match its object name", + "v3/v4 tier delete journal has no committed exact remote-version state", )); } - if current.state != TierDeleteJournalState::Prepared { + let backend_identity = current + .backend_identity + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v3/v4 journal has no backend identity"))?; + let lease = + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), ¤t.tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + delete_remote_tier_journal_target_with_lease(¤t, &lease).await?; + if !lease.is_current_generation() { return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, - "prepared tier delete journal changed before reconciliation", + "legacy tier generation changed after remote delete; journal retained", )); } - let Some(etag) = metadata.etag else { + + let path = tier_delete_journal_object_name(¤t); + let (observed, observed_etag) = read_tier_delete_journal_with_etag(api.clone(), &path) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "legacy journal disappeared"))?; + if observed_etag != etag + || observed.state != TierDeleteJournalState::Committed + || !same_tier_delete_journal_identity(&observed, ¤t) + { return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "prepared tier delete journal has no entity tag", + std::io::ErrorKind::WouldBlock, + "legacy tier delete journal changed during remote deletion", )); - }; + } + let observed_data = encode_tier_delete_journal_entry(&observed).map_err(std::io::Error::other)?; + let fences_current = || lease.is_current_generation(); + delete_durable_config_if_match(api, &path, &observed_data, &observed_etag, &fences_current) + .await + .map_err(std::io::Error::other) +} + +async fn process_v5_journal(api: Arc, mut current: Jentry, mut etag: String) -> std::io::Result<()> { let source = current .source .as_ref() - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?; - match prepared_tier_delete_has_live_source(&api, source, ¤t).await { - Ok((true, read_guards)) => { - if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) { + .filter(|source| source.has_stable_identity()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v5 journal has no stable source identity"))?; + let bucket_guard = api + .acquire_bucket_lifecycle_read_lock(&source.bucket) + .await + .map_err(std::io::Error::other)?; + let backend_identity = current + .backend_identity + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v5 journal has no backend identity"))?; + let lease = + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), ¤t.tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + let (present, guards) = tier_delete_has_durable_source_or_free_version(&api, source, ¤t).await?; + let fences_current = + || !bucket_guard.is_lock_lost() && guards.iter().all(|guard| !guard.is_lock_lost()) && lease.is_current_generation(); + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "v5 tier delete recovery fence changed", + )); + } + + match current.state { + TierDeleteJournalState::Prepared if present => { + let path = tier_delete_journal_object_name(¤t); + let result = config_boundary::delete_config_if_match(api, &path, &etag).await; + if !fences_current() { return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, - "tier delete journal recovery object read lock was lost before abort", + "v5 prepared journal fence changed during abort", )); } - let result = config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await; - drop(read_guards); - match result { - Ok(()) => Ok(()), + return match result { + Ok(()) | Err(Error::ConfigNotFound) => Ok(()), Err(Error::PreconditionFailed) => Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, - "prepared tier delete journal changed before abort", + "v5 prepared journal changed before abort", )), Err(err) => Err(std::io::Error::other(err)), - } + }; } - Ok((false, read_guards)) if source.has_stable_identity() => { - if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) { + TierDeleteJournalState::Dispatched | TierDeleteJournalState::Committed if present => { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "v5 tier delete journal retains a durable source or free-version owner", + )); + } + TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched => { + current.state = TierDeleteJournalState::Committed; + let path = tier_delete_journal_object_name(¤t); + save_config_if_match_fenced( + api.clone(), + &path, + encode_tier_delete_journal_entry(¤t).map_err(std::io::Error::other)?, + &etag, + &fences_current, + ) + .await + .map_err(std::io::Error::other)?; + let (observed, observed_etag) = read_tier_delete_journal_with_etag(api.clone(), &path) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "committed v5 journal disappeared"))?; + if observed.state != TierDeleteJournalState::Committed || !same_tier_delete_journal_identity(&observed, ¤t) { return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, - "tier delete journal recovery object read lock was lost before commit", + "v5 journal changed after committed CAS", )); } - let mut commit_opts = ObjectOptions::default(); - for signal in read_guards - .iter() - .filter_map(crate::store::ObjectLockDiagGuard::lock_lost_signal) - { - commit_opts.add_namespace_lock_lost_signal(signal); - } - let committed = - commit_prepared_tier_delete_journal_entry_if_current(api.clone(), current, etag, &commit_opts).await?; - // Keep namespace locks only through the journal CAS. Remote-tier IO - // must not block writers for the object during recovery. - drop(read_guards); - process_committed_tier_delete_journal_entry(api, &committed).await + current = observed; + etag = observed_etag; } - Ok((false, _read_guards)) => Err(std::io::Error::new( + TierDeleteJournalState::Committed => {} + } + + if !fences_current() { + return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, - "prepared tier delete journal source identity is not sufficient to confirm deletion", - )), - Err(err) => Err(err), + "v5 tier delete fence changed before remote deletion", + )); + } + delete_remote_tier_journal_target_with_lease(¤t, &lease).await?; + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "v5 tier delete fence changed after remote deletion; journal retained", + )); + } + let path = tier_delete_journal_object_name(¤t); + let (observed, observed_etag) = read_tier_delete_journal_with_etag(api.clone(), &path) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "committed v5 journal disappeared"))?; + if observed_etag != etag + || observed.state != TierDeleteJournalState::Committed + || !same_tier_delete_journal_identity(&observed, ¤t) + { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "v5 tier delete journal changed during remote deletion", + )); + } + let observed_data = encode_tier_delete_journal_entry(&observed).map_err(std::io::Error::other)?; + delete_durable_config_if_match(api, &path, &observed_data, &observed_etag, &fences_current) + .await + .map_err(std::io::Error::other) +} + +async fn load_manifest_for_journal( + api: Arc, + journal_name: &str, + journal: &Jentry, +) -> Result { + let binding = journal + .dispatch + .as_ref() + .ok_or_else(|| Error::other("tier delete journal v6 is missing its dispatch binding"))?; + let (manifest, _) = read_tier_delete_dispatch_manifest(api, &binding.manifest_object) + .await? + .ok_or_else(|| Error::other("tier delete dispatch manifest is missing"))?; + validate_bound_journal(&manifest, journal_name, journal)?; + if manifest.journal_names.binary_search(&journal_name.to_string()).is_err() { + return Err(Error::other("tier delete dispatch manifest does not contain its journal")); + } + Ok(manifest) +} + +fn acquire_matching_manifest_fleet_proof( + manifest: &TierDeleteDispatchManifest, +) -> std::io::Result { + let proof = acquire_tier_delete_journal_fleet_proof().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::WouldBlock, "tier delete journal v6 fleet capability is unavailable") + })?; + if tier_delete_journal_topology_generation(&proof) != manifest.topology_generation { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete journal topology generation changed", + )); + } + Ok(proof) +} + +async fn reconcile_prepared_v6_journal( + _api: Arc, + _current: Jentry, + _etag: String, + manifest: TierDeleteDispatchManifest, +) -> std::io::Result<()> { + // A Prepared journal is negative authorization evidence. Only the + // manifest coordinator, while holding the bucket WRITE lock, operation + // lock, and matching fleet proof, may seal rollback and remove the full + // immutable journal set. A single-journal worker must never punch a hole + // in that set even when it observes Aborting/Aborted. + Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + format!("prepared tier delete journal is owned by the {:?} manifest coordinator", manifest.state), + )) +} + +async fn reconcile_dispatched_v6_journal( + api: Arc, + current: Jentry, + etag: String, + manifest: TierDeleteDispatchManifest, +) -> std::io::Result<()> { + if !matches!( + manifest.state, + TierDeleteDispatchManifestState::DispatchAuthorized | TierDeleteDispatchManifestState::Completed + ) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "dispatched tier delete journal has no authorized manifest", + )); + } + let fleet_proof = acquire_matching_manifest_fleet_proof(&manifest)?; + let source = current + .source + .as_ref() + .filter(|source| source.has_stable_identity()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v6 journal has no stable source identity"))?; + let bucket_guard = api + .acquire_bucket_lifecycle_read_lock(&source.bucket) + .await + .map_err(std::io::Error::other)?; + let backend_identity = current + .backend_identity + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v6 journal has no backend identity"))?; + let lease = + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), ¤t.tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + let (present, guards) = tier_delete_has_durable_source_or_free_version(&api, source, ¤t).await?; + let fences_current = || journal_remote_fences_current(&bucket_guard, &guards, &lease, &fleet_proof, &manifest); + if present || !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "dispatched tier delete journal still has a source, free-version, or invalid fence", + )); + } + record_tier_delete_journal_progress_fenced( + api.clone(), + &tier_delete_journal_object_name(¤t), + ¤t, + &fences_current, + ) + .await + .map_err(std::io::Error::other)?; + let mut committed = current.clone(); + committed.state = TierDeleteJournalState::Committed; + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "dispatched tier delete journal fence changed before commit", + )); + } + let commit = save_config_if_match_fenced( + api.clone(), + &tier_delete_journal_object_name(&committed), + encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?, + &etag, + &fences_current, + ) + .await; + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "dispatched tier delete journal fence changed during commit; durable state retained", + )); + } + commit.map_err(std::io::Error::other)?; + drop(guards); + drop(lease); + drop(bucket_guard); + // The journal CAS is complete, but manifest completion belongs exclusively + // to the coordinator with its operation lock and bucket WRITE fence. + Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "committed tier delete journal waits for manifest coordinator completion", + )) +} + +fn journal_remote_fences_current( + bucket_guard: &rustfs_lock::NamespaceLockGuard, + guards: &[crate::store::ObjectLockDiagGuard], + lease: &crate::services::tier::tier::TierOperationLease, + fleet_proof: &TierDeleteJournalFleetProofToken, + manifest: &TierDeleteDispatchManifest, +) -> bool { + !bucket_guard.is_lock_lost() + && guards.iter().all(|guard| !guard.is_lock_lost()) + && lease.is_current_generation() + && tier_delete_journal_fleet_proof_matches(fleet_proof) + && tier_delete_journal_topology_generation(fleet_proof) == manifest.topology_generation +} + +async fn process_committed_v6_journal( + api: Arc, + current: Jentry, + etag: String, + manifest: TierDeleteDispatchManifest, +) -> std::io::Result<()> { + if manifest.state != TierDeleteDispatchManifestState::Completed { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "committed tier delete journal waits for its completed manifest", + )); + } + let fleet_proof = acquire_matching_manifest_fleet_proof(&manifest)?; + let source = current + .source + .as_ref() + .filter(|source| source.has_stable_identity()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v6 journal has no stable source identity"))?; + let bucket_guard = api + .acquire_bucket_lifecycle_read_lock(&source.bucket) + .await + .map_err(std::io::Error::other)?; + let backend_identity = current + .backend_identity + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "v6 journal has no backend identity"))?; + let lease = + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), ¤t.tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + let (present, guards) = tier_delete_has_durable_source_or_free_version(&api, source, ¤t).await?; + let fences_current = || journal_remote_fences_current(&bucket_guard, &guards, &lease, &fleet_proof, &manifest); + if present || !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "committed tier delete journal has a durable source owner or invalid fence", + )); + } + record_tier_delete_journal_progress_fenced( + api.clone(), + &tier_delete_journal_object_name(¤t), + ¤t, + &fences_current, + ) + .await + .map_err(std::io::Error::other)?; + + let remote_delete = async { + if current.version_id_exact { + delete_confirmed_transition_candidate_exact_with_lease_idempotent(¤t.obj_name, ¤t.version_id, &lease) + .await + } else { + delete_object_from_remote_tier_with_lease_idempotent(¤t.obj_name, ¤t.version_id, &lease, false).await + } + }; + tokio::time::timeout(TIER_DELETE_REMOTE_DEADLINE, remote_delete) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "remote tier delete exceeded its deadline"))??; + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete fence changed after remote deletion; durable journal retained", + )); + } + let path = tier_delete_journal_object_name(¤t); + let (observed, observed_etag) = read_tier_delete_journal_with_etag(api.clone(), &path) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "committed journal disappeared"))?; + if observed_etag != etag + || observed.state != TierDeleteJournalState::Committed + || !same_tier_delete_journal_identity(&observed, ¤t) + { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "committed tier delete journal changed during remote deletion", + )); + } + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete fence changed before committed journal cleanup; durable journal retained", + )); + } + let observed_data = encode_tier_delete_journal_entry(&observed).map_err(std::io::Error::other)?; + match delete_durable_config_if_match(api.clone(), &path, &observed_data, &observed_etag, &fences_current).await { + Ok(()) | Err(Error::ConfigNotFound) => {} + Err(err) => return Err(std::io::Error::other(err)), + } + if !fences_current() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier delete fence changed during committed journal cleanup", + )); + } + drop(guards); + drop(lease); + drop(bucket_guard); + // Completed manifest GC belongs to the manifest coordinator so a stale + // journal worker cannot erase the last durable operation record. + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TierDeleteDispatchManifestRecoveryOutcome { + Advanced, + Deleted, + Retained, +} + +fn manifest_recovery_fences_current( + bucket_guard: &rustfs_lock::NamespaceLockGuard, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + manifest: &TierDeleteDispatchManifest, + cancel_token: Option<&CancellationToken>, +) -> bool { + cancel_token.is_none_or(|token| !token.is_cancelled()) + && !bucket_guard.is_lock_lost() + && !operation_guard.is_lock_lost() + && tier_delete_journal_fleet_proof_matches(fleet_proof) + && tier_delete_journal_topology_generation(fleet_proof) == manifest.topology_generation +} + +async fn cas_tier_delete_dispatch_manifest_state( + api: Arc, + manifest_name: &str, + expected: &TierDeleteDispatchManifest, + expected_etag: &str, + next_state: TierDeleteDispatchManifestState, + fences_current: &impl Fn() -> bool, +) -> Result<(TierDeleteDispatchManifest, String)> { + if !fences_current() { + return Err(Error::other( + "tier delete dispatch manifest recovery fence changed before state transition", + )); + } + let mut next = expected.clone(); + next.state = next_state; + let write = save_config_if_match_fenced( + api.clone(), + manifest_name, + encode_tier_delete_dispatch_manifest(&next)?, + expected_etag, + fences_current, + ) + .await; + if write.as_ref().is_err_and(is_decommission_checkpoint_targets_incomplete) { + return Err(write.expect_err("the decommission checkpoint result should remain an error")); + } + let observed = read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await?; + if !fences_current() { + return Err(Error::other( + "tier delete dispatch manifest recovery fence changed during state transition", + )); + } + match observed { + Some((observed, etag)) if observed == next => Ok((observed, etag)), + _ => match write { + Ok(()) | Err(Error::PreconditionFailed) => { + Err(Error::other("tier delete dispatch manifest changed during state transition")) + } + Err(err) => Err(err), + }, } } -async fn commit_prepared_tier_delete_journal_entry_if_current( +async fn delete_tier_delete_dispatch_manifest_if_match_confirmed( api: Arc, - mut committed: Jentry, - etag: String, - lock_opts: &ObjectOptions, -) -> std::io::Result { - committed.state = TierDeleteJournalState::Committed; - let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?; - match config_boundary::save_config_with_opts( - api.clone(), - &tier_delete_journal_object_name(&committed), - data, - &ObjectOptions { - max_parity: true, - http_preconditions: Some(HTTPPreconditions { - if_match: Some(etag), - ..Default::default() - }), - namespace_lock_fence: lock_opts.namespace_lock_fence.clone(), - ..Default::default() - }, - ) - .await + manifest_name: &str, + manifest: &TierDeleteDispatchManifest, + etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + if !fences_current() { + return Err(Error::other("tier delete dispatch manifest recovery fence changed before deletion")); + } + let data = encode_tier_delete_dispatch_manifest(manifest)?; + let delete = delete_durable_config_if_match(api.clone(), manifest_name, &data, etag, fences_current).await; + let observed = read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch manifest recovery fence changed during deletion")); + } + match observed { + None => Ok(()), + Some((observed, _)) => { + let observed_data = encode_tier_delete_dispatch_manifest(&observed)?; + if api + .durable_ilm_terminal_receipt_covers_active_source(manifest_name, &observed_data) + .await? + { + Ok(()) + } else { + Err(Error::other_with_context( + "tier delete dispatch manifest changed during deletion", + format!("observed {:?}, delete result {:?}", observed.state, delete.as_ref().err()), + )) + } + } + } +} + +async fn authorized_dispatch_all_committed( + api: Arc, + manifest: &TierDeleteDispatchManifest, + fences_current: &impl Fn() -> bool, +) -> Result { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let stopped = Arc::new(AtomicBool::new(false)); + let make_read = |name: String| { + let api = api.clone(); + let stopped = stopped.clone(); + async move { + if stopped.load(Ordering::Acquire) { + return Ok::<_, Error>(TierDeleteJournalState::Committed); + } + let result = async { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + #[cfg(all(test, feature = "test-util"))] + let _test_permit = + tier_delete_dispatch_member_read_test_hook(TierDeleteDispatchMemberReadTestStage::Authorized).await; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let observed = read_tier_delete_journal_with_etag(api.clone(), &name).await?; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let (entry, _) = observed + .ok_or_else(|| Error::other("authorized tier delete dispatch manifest references a missing journal"))?; + validate_bound_journal(manifest, &name, &entry)?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_dispatch_authorized_progress_test_observed(); + record_tier_delete_journal_progress_fenced(api, &name, &entry, fences_current).await?; + Ok(entry.state) + } + .await; + if result.is_err() { + stopped.store(true, Ordering::Release); + } + result + } + }; + let mut next = 0; + let mut reads = futures::stream::FuturesUnordered::new(); + while next < manifest.journal_names.len() && reads.len() < TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY { + reads.push(make_read(manifest.journal_names[next].clone())); + next += 1; + } + let mut all_committed = true; + let mut first_error = None; + while let Some(result) = reads.next().await { + match result { + Ok(TierDeleteJournalState::Committed) => {} + Ok(TierDeleteJournalState::Dispatched) => all_committed = false, + Ok(TierDeleteJournalState::Prepared) if first_error.is_none() => { + first_error = Some(Error::other( + "authorized tier delete dispatch contains a journal that was never dispatched", + )); + } + Ok(TierDeleteJournalState::Prepared) => {} + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + if first_error.is_some() { + stopped.store(true, Ordering::Release); + } else if !stopped.load(Ordering::Acquire) { + if let Err(err) = ensure_tier_delete_dispatch_member_scan_fence(fences_current) { + stopped.store(true, Ordering::Release); + first_error = Some(err); + } else if next < manifest.journal_names.len() { + reads.push(make_read(manifest.journal_names[next].clone())); + next += 1; + } + } + } + match first_error { + Some(err) => Err(err), + None => Ok(all_committed), + } +} + +async fn completed_dispatch_has_present_journal( + api: Arc, + manifest: &TierDeleteDispatchManifest, + fences_current: &impl Fn() -> bool, +) -> Result { + for chunk in manifest.journal_names.chunks(TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY) { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let stopped = Arc::new(AtomicBool::new(false)); + let mut reads = futures::stream::iter(chunk.iter().cloned().map(|name| { + let api = api.clone(); + let stopped = stopped.clone(); + async move { + if stopped.load(Ordering::Acquire) { + return Ok::<_, Error>(None); + } + let result = async { + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + #[cfg(all(test, feature = "test-util"))] + let _test_permit = + tier_delete_dispatch_member_read_test_hook(TierDeleteDispatchMemberReadTestStage::Completed).await; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let observed = read_tier_delete_journal_with_etag(api, &name).await?; + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + let Some((entry, _)) = observed else { + return Ok(None); + }; + validate_bound_journal(manifest, &name, &entry)?; + if entry.state != TierDeleteJournalState::Committed { + return Err(Error::other("completed tier delete dispatch contains an uncommitted journal")); + } + Ok(Some((name, entry))) + } + .await; + if result.is_err() { + stopped.store(true, Ordering::Release); + } + result + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_MEMBER_READ_CONCURRENCY); + let mut present = None; + let mut first_error = None; + while let Some(result) = reads.next().await { + match result { + Ok(Some(found)) if present.is_none() => present = Some(found), + Ok(_) => {} + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + } + if let Some(err) = first_error { + return Err(err); + } + ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; + if let Some((name, entry)) = present { + record_tier_delete_journal_progress_fenced(api.clone(), &name, &entry, fences_current).await?; + return Ok(true); + } + } + Ok(false) +} + +async fn process_tier_delete_dispatch_manifest( + api: Arc, + manifest_name: &str, + observed_before_lock: &TierDeleteDispatchManifest, + cancel_token: Option<&CancellationToken>, +) -> Result { + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + // A stale/missing capability may inspect records but must never mutate or + // delete v6 evidence. The same proof is revalidated at every write edge. + let fleet_proof = acquire_matching_manifest_fleet_proof(observed_before_lock).map_err(Error::other)?; + let bucket_guard = api.acquire_bucket_lifecycle_write_lock(&observed_before_lock.bucket).await?; + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(manifest_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (mut current, mut etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) + .await? + .ok_or_else(|| Error::ConfigNotFound)?; + if current.bucket != observed_before_lock.bucket + || !manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) { - Ok(()) => Ok(committed), - Err(Error::PreconditionFailed) => Err(std::io::Error::new( - std::io::ErrorKind::WouldBlock, - "prepared tier delete journal changed before commit", - )), - Err(err) => Err(std::io::Error::other(err)), + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + return Err(Error::other("tier delete dispatch manifest recovery fence changed")); + } + { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + // A prior target PUT may have committed before its temporary capacity + // progress save. When recovery observes that checkpoint as current, + // no state CAS is left to trigger the normal already-committed path; + // reconcile the exact deterministic intent before any later journal + // mutation can be admitted. + let current_data = encode_tier_delete_dispatch_manifest(¤t)?; + let _ = save_decommission_manifest_checkpoint_if_match(api.clone(), manifest_name, ¤t_data, &etag, &fences_current) + .await?; + record_tier_delete_dispatch_manifest_progress_fenced(api.clone(), manifest_name, ¤t, &fences_current).await?; + } + + if current.state == TierDeleteDispatchManifestState::Preparing { + let next = { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + cas_tier_delete_dispatch_manifest_state( + api.clone(), + manifest_name, + ¤t, + &etag, + TierDeleteDispatchManifestState::Aborting, + &fences_current, + ) + .await? + }; + (current, etag) = next; + } + + if matches!( + current.state, + TierDeleteDispatchManifestState::Aborting | TierDeleteDispatchManifestState::Aborted + ) { + { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + delete_staged_dispatch_journal_set(api.clone(), ¤t, &fences_current).await?; + } + if current.state == TierDeleteDispatchManifestState::Aborting { + let next = { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + cas_tier_delete_dispatch_manifest_state( + api.clone(), + manifest_name, + ¤t, + &etag, + TierDeleteDispatchManifestState::Aborted, + &fences_current, + ) + .await? + }; + (current, etag) = next; + } + if !manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) { + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + return Err(Error::other("tier delete dispatch manifest recovery fence changed")); + } + validate_staged_dispatch_journal_set(api.clone(), ¤t, &|| { + manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) + }) + .await?; + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + delete_tier_delete_dispatch_manifest_if_match_confirmed(api, manifest_name, ¤t, &etag, &fences_current).await?; + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Deleted); + } + + if current.state == TierDeleteDispatchManifestState::DispatchAuthorized { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + if !authorized_dispatch_all_committed(api.clone(), ¤t, &fences_current).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let next = { + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + cas_tier_delete_dispatch_manifest_state( + api.clone(), + manifest_name, + ¤t, + &etag, + TierDeleteDispatchManifestState::Completed, + &fences_current, + ) + .await? + }; + (current, etag) = next; + if !current.journal_names.is_empty() { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced); + } + } + + if current.state != TierDeleteDispatchManifestState::Completed { + return Err(Error::other("tier delete dispatch manifest has an unsupported recovery state")); + } + let fences_current = + || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); + if completed_dispatch_has_present_journal(api.clone(), ¤t, &fences_current).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + if !manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) { + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + return Err(Error::other("tier delete dispatch manifest recovery fence changed")); + } + delete_tier_delete_dispatch_manifest_if_match_confirmed(api, manifest_name, ¤t, &etag, &fences_current).await?; + Ok(TierDeleteDispatchManifestRecoveryOutcome::Deleted) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn recover_test_tier_delete_dispatch_manifest(api: Arc, manifest_name: &str) -> Result<()> { + let (manifest, _) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) + .await? + .ok_or(Error::ConfigNotFound)?; + process_tier_delete_dispatch_manifest(api, manifest_name, &manifest, None) + .await + .map(|_| ()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TierDeleteDispatchManifestScanOutcome { + Advanced, + Deleted, + Retained, + Converged, + Failed, +} + +type TierDeleteDispatchManifestRecoveryKey = (uuid::Uuid, String); + +#[derive(Default)] +struct TierDeleteDispatchManifestRecoveryRegistry { + keys: HashSet, + active_by_store: HashMap, +} + +static TIER_DELETE_DISPATCH_MANIFEST_RECOVERIES: OnceLock> = OnceLock::new(); + +struct TierDeleteDispatchManifestRecoveryGuard { + key: TierDeleteDispatchManifestRecoveryKey, +} + +impl TierDeleteDispatchManifestRecoveryGuard { + fn try_acquire(store_id: uuid::Uuid, object_name: &str) -> Option { + let key = (store_id, object_name.to_string()); + let mut recoveries = TIER_DELETE_DISPATCH_MANIFEST_RECOVERIES + .get_or_init(|| Mutex::new(TierDeleteDispatchManifestRecoveryRegistry::default())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if recoveries.keys.contains(&key) + || recoveries.active_by_store.get(&store_id).copied().unwrap_or_default() + >= TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_CONCURRENCY + { + return None; + } + recoveries.keys.insert(key.clone()); + *recoveries.active_by_store.entry(store_id).or_default() += 1; + Some(Self { key }) + } +} + +impl Drop for TierDeleteDispatchManifestRecoveryGuard { + fn drop(&mut self) { + let mut recoveries = TIER_DELETE_DISPATCH_MANIFEST_RECOVERIES + .get_or_init(|| Mutex::new(TierDeleteDispatchManifestRecoveryRegistry::default())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !recoveries.keys.remove(&self.key) { + return; + } + let store_id = self.key.0; + let remove_store = if let Some(active) = recoveries.active_by_store.get_mut(&store_id) { + *active = active.saturating_sub(1); + *active == 0 + } else { + false + }; + if remove_store { + recoveries.active_by_store.remove(&store_id); + } + } +} + +async fn recover_tier_delete_dispatch_manifest_object( + api: Arc, + object_name: String, + cancel_token: Option, +) -> TierDeleteDispatchManifestScanOutcome { + if cancel_token.as_ref().is_some_and(CancellationToken::is_cancelled) { + return TierDeleteDispatchManifestScanOutcome::Retained; + } + let data = match config_boundary::read_config(api.clone(), &object_name).await { + Ok(data) => data, + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => { + return TierDeleteDispatchManifestScanOutcome::Converged; + } + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %object_name, + error = ?err, + "Failed to read tier delete dispatch manifest" + ); + return TierDeleteDispatchManifestScanOutcome::Failed; + } + }; + let manifest = match decode_tier_delete_dispatch_manifest(&data, &object_name) { + Ok(manifest) => manifest, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %object_name, + error = ?err, + "Invalid tier delete dispatch manifest is quarantined" + ); + return TierDeleteDispatchManifestScanOutcome::Failed; + } + }; + match api + .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data) + .await + { + Ok(true) => { + // The remaining copy is the decommission source checkpoint. Its + // target-side terminal receipt makes the logical operation + // complete; only decommission verification may remove it. + return TierDeleteDispatchManifestScanOutcome::Retained; + } + Ok(false) => {} + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %object_name, + operation_id = %manifest.operation_id, + error = ?err, + "Tier delete dispatch terminal source proof will retry later" + ); + return TierDeleteDispatchManifestScanOutcome::Failed; + } + } + let result = process_tier_delete_dispatch_manifest(api, &object_name, &manifest, cancel_token.as_ref()).await; + if cancel_token.as_ref().is_some_and(CancellationToken::is_cancelled) { + // Cancellation is cooperative: every admitted mutation above has + // already returned while the operation/fleet guards and registry + // permit were still held. Durable partial progress is intentionally + // retained for a fresh store instance to replay. + return TierDeleteDispatchManifestScanOutcome::Retained; + } + match result { + Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced) => TierDeleteDispatchManifestScanOutcome::Advanced, + Ok(TierDeleteDispatchManifestRecoveryOutcome::Deleted) => TierDeleteDispatchManifestScanOutcome::Deleted, + Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained) => TierDeleteDispatchManifestScanOutcome::Retained, + Err(Error::ConfigNotFound) => TierDeleteDispatchManifestScanOutcome::Converged, + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %object_name, + operation_id = %manifest.operation_id, + state = ?manifest.state, + error = ?err, + "Tier delete dispatch manifest recovery will retry later" + ); + TierDeleteDispatchManifestScanOutcome::Failed + } + } +} + +async fn schedule_tier_delete_dispatch_manifest_recovery( + api: Arc, + object_name: String, + recovery_timeout: Duration, +) -> TierDeleteDispatchManifestScanOutcome { + let Some(guard) = TierDeleteDispatchManifestRecoveryGuard::try_acquire(api.id, &object_name) else { + // A prior page timed out while this manifest was still running, or the + // per-store worker budget is full. Do not queue an operation-lock or + // worker-permit waiter: the durable record will be observed by a later + // page after an active worker converges or releases its permit. + return TierDeleteDispatchManifestScanOutcome::Retained; + }; + let log_name = object_name.clone(); + let cancel_token = api.ctx.background_cancel_token(); + let mut recovery = tokio::spawn(async move { + let _guard = guard; + // Never drop an in-flight recovery on shutdown: lower config writes + // may hand their physical commit to a detached task. The token is a + // cooperative admission fence, so already-started mutations drain + // under the same bucket/operation/fleet guards and registry permit. + recover_tier_delete_dispatch_manifest_object(api, object_name, cancel_token).await + }); + match tokio::time::timeout(recovery_timeout, &mut recovery).await { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %log_name, + error = ?err, + "Tier delete dispatch manifest recovery task failed and will retry later" + ); + TierDeleteDispatchManifestScanOutcome::Failed + } + Err(_) => { + // Dropping a Tokio JoinHandle detaches rather than cancels the + // task. The per-store/object guard keeps later pages from queuing + // duplicates while this bounded worker continues toward the tail + // of a large manifest under the same operation/fleet fences. + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + manifest_object = %log_name, + timeout_seconds = recovery_timeout.as_secs_f64(), + "Tier delete dispatch manifest recovery exceeded its page budget and continues in the background" + ); + TierDeleteDispatchManifestScanOutcome::Failed + } + } +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn recover_test_tier_delete_dispatch_manifest_with_page_budget( + api: Arc, + manifest_name: &str, + recovery_timeout: Duration, +) -> bool { + matches!( + schedule_tier_delete_dispatch_manifest_recovery(api, manifest_name.to_string(), recovery_timeout).await, + TierDeleteDispatchManifestScanOutcome::Failed + ) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn tier_delete_dispatch_manifest_recovery_inflight_for_test(api: &ECStore, manifest_name: &str) -> bool { + TIER_DELETE_DISPATCH_MANIFEST_RECOVERIES + .get_or_init(|| Mutex::new(TierDeleteDispatchManifestRecoveryRegistry::default())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .keys + .contains(&(api.id, manifest_name.to_string())) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn tier_delete_dispatch_manifest_recovery_count_for_test(api: &ECStore) -> usize { + TIER_DELETE_DISPATCH_MANIFEST_RECOVERIES + .get_or_init(|| Mutex::new(TierDeleteDispatchManifestRecoveryRegistry::default())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active_by_store + .get(&api.id) + .copied() + .unwrap_or_default() +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn tier_delete_dispatch_manifest_operation_lock_held_for_test(api: Arc, manifest_name: &str) -> bool { + let Ok(lock) = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(manifest_name)) + .await + else { + return false; + }; + lock.get_write_lock(Duration::from_millis(20)).await.is_err() +} + +pub async fn recover_tier_delete_dispatch_manifests( + api: Arc, + limit: usize, + marker: Option, +) -> Result { + if limit == 0 { + return Err(Error::other("tier delete dispatch manifest recovery limit must be greater than zero")); + } + let list = api + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + marker, + None, + i32::try_from(limit).unwrap_or(i32::MAX), + false, + None, + false, + ) + .await?; + let mut stats = TierDeleteDispatchManifestRecoveryStats { + scanned: 0, + advanced: 0, + deleted: 0, + retained: 0, + failed: 0, + next_marker: list.next_continuation_token, + truncated: list.is_truncated, + }; + let mut recoveries = futures::stream::iter(list.objects.into_iter().map(|object| { + let api = api.clone(); + async move { + schedule_tier_delete_dispatch_manifest_recovery(api, object.name, TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_TIMEOUT) + .await + } + })) + .buffer_unordered(TIER_DELETE_DISPATCH_MANIFEST_RECOVERY_CONCURRENCY); + while let Some(outcome) = recoveries.next().await { + stats.scanned += 1; + match outcome { + TierDeleteDispatchManifestScanOutcome::Advanced => stats.advanced += 1, + TierDeleteDispatchManifestScanOutcome::Deleted => stats.deleted += 1, + TierDeleteDispatchManifestScanOutcome::Retained => stats.retained += 1, + TierDeleteDispatchManifestScanOutcome::Failed => stats.failed += 1, + TierDeleteDispatchManifestScanOutcome::Converged => {} + } + } + Ok(stats) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TierDeleteJournalEntryRecoveryOutcome { + Deleted, + Converged, + Retained, + Failed, +} + +async fn recover_tier_delete_journal_entry(api: Arc, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome { + let data = match config_boundary::read_config(api.clone(), &object_name).await { + Ok(data) => data, + Err(Error::ConfigNotFound) => return TierDeleteJournalEntryRecoveryOutcome::Converged, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?err, + "Failed to read tier delete journal entry" + ); + return TierDeleteJournalEntryRecoveryOutcome::Failed; + } + }; + + let je = match decode_tier_delete_journal_entry(&data) { + Ok(je) => je, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?err, + "Failed to decode tier delete journal entry" + ); + return TierDeleteJournalEntryRecoveryOutcome::Failed; + } + }; + + if tier_delete_journal_object_name(&je) != object_name { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + "Tier delete journal content does not match its object name and will be retained" + ); + return TierDeleteJournalEntryRecoveryOutcome::Failed; + } + + match api + .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data) + .await + { + Ok(true) => return TierDeleteJournalEntryRecoveryOutcome::Retained, + Ok(false) => {} + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?err, + "Tier delete journal terminal source proof will retry later" + ); + return TierDeleteJournalEntryRecoveryOutcome::Failed; + } + } + + if je.backend_identity.is_none() { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + remote_object = %je.obj_name, + remote_version_id = %je.version_id, + tier = %je.tier_name, + "Legacy tier delete journal entry has no durable backend identity and will be retained" + ); + return TierDeleteJournalEntryRecoveryOutcome::Failed; + } + + match process_tier_delete_journal_entry(api, &je).await { + Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Deleted, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // Another recovery worker may have completed the same journal + // after this page was listed but before we acquired its recovery + // lock. The durable obligation has converged. + TierDeleteJournalEntryRecoveryOutcome::Converged + } + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + remote_object = %je.obj_name, + remote_version_id = %je.version_id, + tier = %je.tier_name, + error = ?err, + "Tier delete journal recovery will retry later" + ); + TierDeleteJournalEntryRecoveryOutcome::Failed + } } } @@ -644,7 +4412,7 @@ pub async fn recover_tier_delete_journal_entries( .list_objects_v2( RUSTFS_META_BUCKET, TIER_DELETE_JOURNAL_PREFIX, - marker.clone(), + marker, None, i32::try_from(limit).unwrap_or(i32::MAX), false, @@ -660,85 +4428,39 @@ pub async fn recover_tier_delete_journal_entries( next_marker: list.next_continuation_token, truncated: list.is_truncated, }; + let mut recoveries = futures::stream::iter(list.objects.into_iter().map(|object| { + let api = api.clone(); + async move { + let object_name = object.name; + match tokio::time::timeout( + TIER_DELETE_JOURNAL_ENTRY_RECOVERY_TIMEOUT, + recover_tier_delete_journal_entry(api, object_name.clone()), + ) + .await + { + Ok(outcome) => outcome, + Err(_) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + timeout_seconds = TIER_DELETE_JOURNAL_ENTRY_RECOVERY_TIMEOUT.as_secs(), + "Tier delete journal entry recovery timed out and will retry after marker rotation" + ); + TierDeleteJournalEntryRecoveryOutcome::Failed + } + } + } + })) + .buffer_unordered(TIER_DELETE_JOURNAL_RECOVERY_CONCURRENCY); - for object in list.objects { + while let Some(outcome) = recoveries.next().await { stats.scanned += 1; - let data = match config_boundary::read_config(api.clone(), &object.name).await { - Ok(data) => data, - Err(Error::ConfigNotFound) => continue, - Err(err) => { - stats.failed += 1; - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - journal_object = %object.name, - error = ?err, - "Failed to read tier delete journal entry" - ); - continue; - } - }; - - let je = match decode_tier_delete_journal_entry(&data) { - Ok(je) => je, - Err(err) => { - stats.failed += 1; - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - journal_object = %object.name, - error = ?err, - "Failed to decode tier delete journal entry" - ); - continue; - } - }; - - if tier_delete_journal_object_name(&je) != object.name { - stats.failed += 1; - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - journal_object = %object.name, - "Tier delete journal content does not match its object name and will be retained" - ); - continue; - } - - if je.backend_identity.is_none() { - stats.failed += 1; - warn!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - journal_object = %object.name, - remote_object = %je.obj_name, - remote_version_id = %je.version_id, - tier = %je.tier_name, - "Legacy tier delete journal entry has no durable backend identity and will be retained" - ); - continue; - } - - match process_tier_delete_journal_entry(api.clone(), &je).await { - Ok(()) => stats.deleted += 1, - Err(err) => { - stats.failed += 1; - debug!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - journal_object = %object.name, - remote_object = %je.obj_name, - remote_version_id = %je.version_id, - tier = %je.tier_name, - error = ?err, - "Tier delete journal recovery will retry later" - ); - } + match outcome { + TierDeleteJournalEntryRecoveryOutcome::Deleted => stats.deleted += 1, + TierDeleteJournalEntryRecoveryOutcome::Failed => stats.failed += 1, + TierDeleteJournalEntryRecoveryOutcome::Converged | TierDeleteJournalEntryRecoveryOutcome::Retained => {} } } @@ -749,6 +4471,7 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_tok let mut interval = tokio::time::interval(TIER_DELETE_JOURNAL_RECOVERY_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut marker: Option = None; + let mut manifest_marker: Option = None; loop { #[cfg(test)] @@ -765,6 +4488,45 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_tok _ = interval.tick() => {}, } + let manifest_recovery = recover_tier_delete_dispatch_manifests( + api.clone(), + DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, + manifest_marker.clone(), + ); + let Some(manifest_result) = + await_tier_delete_journal_recovery(&cancel_token, TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT, manifest_recovery).await + else { + return; + }; + match manifest_result { + Ok(stats) => { + manifest_marker = stats.next_marker; + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + scanned = stats.scanned, + advanced = stats.advanced, + deleted = stats.deleted, + retained = stats.retained, + failed = stats.failed, + truncated = stats.truncated, + next_marker = ?manifest_marker, + "Reconciled tier delete dispatch manifests" + ); + } + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + next_marker = ?manifest_marker, + error = ?err, + "Failed to recover tier delete dispatch manifests" + ); + } + } + let recovery = recover_tier_delete_journal_entries(api.clone(), DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, marker.clone()); let Some(result) = @@ -824,11 +4586,15 @@ where #[cfg(test)] mod tests { use super::{ - TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery, - decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, object_info_references_tier_delete, - record_tier_delete_journal_backend_identity, tier_delete_journal_object_name, + TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, + TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX, + await_tier_delete_journal_recovery, decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, + object_info_references_tier_delete, record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, + same_tier_delete_journal_identity, tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, + }; + use crate::bucket::lifecycle::tier_sweeper::{ + Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, }; - use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity}; use crate::error::Result; use crate::object_api::ObjectInfo; use std::time::Duration; @@ -836,6 +4602,7 @@ mod tests { fn journal_entry() -> Jentry { Jentry { + persisted_version: 0, obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), @@ -844,6 +4611,37 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, + } + } + + fn bound_v6_journal_entry(state: TierDeleteJournalState) -> Jentry { + let operation_id = uuid::Uuid::new_v4(); + Jentry { + persisted_version: 0, + obj_name: "remote/object".to_string(), + version_id: "remote-version".to_string(), + tier_name: "WARM".to_string(), + backend_identity: Some([7; 32]), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state, + source: Some(TierDeleteSourceIdentity { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: Some("version".to_string()), + versioned: true, + version_suspended: false, + data_dir: Some("data-dir".to_string()), + etag: Some("etag".to_string()), + mod_time: Some("mod-time".to_string()), + }), + dispatch: Some(TierDeleteDispatchBinding { + operation_id, + manifest_object: "ilm/tier-delete-dispatch-manifests/manifest.json".to_string(), + journal_set_sha256: "7".repeat(64), + topology_generation: "8".repeat(64), + }), } } @@ -873,29 +4671,123 @@ mod tests { } #[test] - fn tier_delete_transaction_roundtrips_prepared_source_identity() { - let mut je = journal_entry(); - je.state = TierDeleteJournalState::Prepared; - je.source = Some(TierDeleteSourceIdentity { + fn dispatch_authorization_identity_normalizes_source_mode_name_collision() { + let original = bound_v6_journal_entry(TierDeleteJournalState::Dispatched); + let mut changed = original.clone(); + let source = changed.source.as_mut().expect("bound journal should carry a source identity"); + source.versioned = false; + source.version_suspended = true; + + assert_eq!(tier_delete_journal_object_name(&original), tier_delete_journal_object_name(&changed)); + assert!(!same_tier_delete_journal_identity(&original, &changed)); + assert!(same_tier_delete_authorization_identity(&original, &changed)); + } + + #[test] + fn dispatch_scope_compares_persisted_directory_names_logically() { + let source = TierDeleteSourceIdentity { bucket: "bucket".to_string(), - object: "object".to_string(), + object: "directory/".to_string(), version_id: Some("version".to_string()), versioned: true, version_suspended: false, data_dir: Some("data-dir".to_string()), etag: Some("etag".to_string()), mod_time: Some("mod-time".to_string()), - }); + }; + let persisted_prefix = rustfs_utils::path::encode_dir_object("directory/"); + + assert!(tier_delete_source_matches_dispatch_scope(&source, "bucket", &persisted_prefix)); + assert!(!tier_delete_source_matches_dispatch_scope(&source, "other-bucket", &persisted_prefix,)); + assert!(!tier_delete_source_matches_dispatch_scope(&source, "bucket", "other/")); + let mut empty_source = source; + empty_source.object.clear(); + assert!(!tier_delete_source_matches_dispatch_scope(&empty_source, "bucket", "")); + } + + #[test] + fn tier_delete_v6_dispatch_roundtrips_bound_source_identity() { + let je = bound_v6_journal_entry(TierDeleteJournalState::Prepared); let encoded = encode_tier_delete_journal_entry(&je).expect("prepared transaction should encode"); let value: serde_json::Value = serde_json::from_slice(&encoded).expect("transaction should be JSON"); - assert_eq!(value["version"], serde_json::json!(5)); + assert_eq!(value["version"], serde_json::json!(TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION)); assert_eq!(value["state"], serde_json::json!("Prepared")); assert!(value["source"].is_object()); let decoded = decode_tier_delete_journal_entry(&encoded).expect("prepared transaction should decode"); + assert_eq!(decoded.persisted_version, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION); assert_eq!(decoded.state, TierDeleteJournalState::Prepared); assert_eq!(decoded.source, je.source); + assert_eq!(decoded.dispatch, je.dispatch); + } + + #[test] + fn v6_operation_path_isolated_from_v5_and_successor_operations() { + let first = bound_v6_journal_entry(TierDeleteJournalState::Prepared); + let first_path = tier_delete_journal_object_name(&first); + let first_operation = first.dispatch.as_ref().expect("v6 entry should be bound").operation_id; + assert!(first_path.starts_with(&format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", first_operation.simple()))); + + let mut legacy = first.clone(); + legacy.persisted_version = TIER_DELETE_JOURNAL_TRANSACTION_VERSION; + legacy.dispatch = None; + let legacy_path = tier_delete_journal_object_name(&legacy); + assert!(legacy_path.starts_with(TIER_DELETE_JOURNAL_LEGACY_PREFIX)); + assert_ne!(legacy_path, first_path, "an old v5 writer must not address a v6 record"); + + let mut successor = first; + successor + .dispatch + .as_mut() + .expect("successor should remain bound") + .operation_id = uuid::Uuid::new_v4(); + let successor_path = tier_delete_journal_object_name(&successor); + assert_ne!( + successor_path, first_path, + "a retry after terminal rollback must not reuse the previous operation receipt key" + ); + } + + #[test] + fn sole_owner_transaction_is_rejected_by_previous_v5_reader_and_v5_remains_readable() { + let je = bound_v6_journal_entry(TierDeleteJournalState::Prepared); + + let encoded = encode_tier_delete_journal_entry(&je).expect("sole-owner transaction should encode"); + let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("transaction should be JSON"); + let written_version = persisted["version"].as_u64().expect("journal version should be numeric"); + assert_eq!(written_version, u64::from(TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION)); + assert!( + written_version > 5, + "the previous v5 reader must reject sole-owner records before remote cleanup" + ); + let previous: super::PersistedTierDeleteJournalEntry = + serde_json::from_slice(&encoded).expect("the previous reader should parse the shared JSON shape"); + let previous_decode = if previous.version > super::TIER_DELETE_JOURNAL_TRANSACTION_VERSION { + Err(format!("unsupported tier delete journal version {}", previous.version)) + } else { + previous.into_jentry().map_err(|err| err.to_string()) + }; + assert_eq!( + previous_decode.expect_err("the previous v5 decoder must fail closed on v6"), + "unsupported tier delete journal version 6" + ); + + persisted["version"] = serde_json::json!(5); + persisted + .as_object_mut() + .expect("journal must be an object") + .remove("dispatch"); + let legacy_v5 = serde_json::to_vec(&persisted).expect("v5 compatibility fixture should encode"); + let decoded = decode_tier_delete_journal_entry(&legacy_v5).expect("the new reader must retain v5 compatibility"); + assert_eq!(decoded.persisted_version, TIER_DELETE_JOURNAL_TRANSACTION_VERSION); + assert_eq!(decoded.state, TierDeleteJournalState::Prepared); + assert_eq!(decoded.source, je.source); + let checkpointed = encode_tier_delete_journal_entry(&decoded).expect("compatibility tooling may checkpoint v5"); + let checkpointed: serde_json::Value = + serde_json::from_slice(&checkpointed).expect("checkpointed transaction should be JSON"); + assert_eq!(checkpointed["version"], serde_json::json!(TIER_DELETE_JOURNAL_TRANSACTION_VERSION)); + assert!(checkpointed.get("dispatch").is_none()); } #[test] diff --git a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs index 0bcae9dc6..d00c74e5a 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs @@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel( return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into()); } - let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?; + let page = + list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?; let mut stats = FreeVersionRecoveryStats { scanned: 0, enqueued: 0, @@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel( return Err(tier_free_version_recovery_cancelled()); } retry_cursor.visit(&oi); - if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) { + if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) { let (bucket_marker, object_marker) = retry_cursor.retry_markers(); stats.truncated = true; stats.next_bucket_marker = bucket_marker; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 0d622576d..bfbefe2ae 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -255,6 +255,7 @@ impl ObjSweeper { } if del_tier { return Some(Jentry { + persisted_version: 0, obj_name: self.remote_object.clone(), version_id: self.transition_version_id.clone(), tier_name: self.transition_tier.clone(), @@ -266,6 +267,7 @@ impl ObjSweeper { version_state: self.transition_version_state, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, }); } None @@ -298,9 +300,19 @@ impl ObjSweeper { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub(crate) enum TierDeleteJournalState { Prepared, + Dispatched, Committed, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TierDeleteDispatchBinding { + pub(crate) operation_id: Uuid, + pub(crate) manifest_object: String, + pub(crate) journal_set_sha256: String, + pub(crate) topology_generation: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub(crate) struct TierDeleteSourceIdentity { @@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity { #[derive(Debug, Clone)] #[allow(unused_assignments)] pub struct Jentry { + /// On-disk format version when decoded. Newly constructed entries use 0; + /// the encoder chooses their format from the durable ownership fields. + /// Recovery uses this value to quarantine v1-v5 without rewriting them. + pub(crate) persisted_version: u8, pub(crate) obj_name: String, pub(crate) version_id: String, pub(crate) tier_name: String, @@ -350,6 +366,23 @@ pub struct Jentry { pub(crate) version_state: rustfs_filemeta::TransitionVersionState, pub(crate) state: TierDeleteJournalState, pub(crate) source: Option, + pub(crate) dispatch: Option, +} + +impl Jentry { + /// Whether this prepared transaction is eligible to become the sole + /// cleanup owner for its transitioned source. The caller may use this to + /// decide whether to persist it, but must not set `skip_free_version` + /// until persistence succeeds. + pub(crate) fn can_replace_tier_free_version(&self) -> bool { + self.state == TierDeleteJournalState::Prepared + && self.backend_identity.is_some() + && self.version_state != rustfs_filemeta::TransitionVersionState::Unknown + && self + .source + .as_ref() + .is_some_and(TierDeleteSourceIdentity::has_stable_identity) + } } impl ExpiryOp for Jentry { @@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry( } Some(Jentry { + persisted_version: 0, obj_name: transitioned.name.clone(), version_id: transitioned.version_id.clone(), tier_name: transitioned.tier.clone(), @@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry( version_state: transition_version_state, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, }) } @@ -673,17 +708,73 @@ mod test { use rustfs_s3_client::signer_error::invalid_utf8_header_error; use super::{ - CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, - RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity, - delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity, - is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, - should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, + CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry, + RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity, + delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent, + delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error, + is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure, + transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, }; use crate::storage_api_contracts::lifecycle::TransitionedObject; use rustfs_filemeta::TransitionVersionState; use std::io::{Error, ErrorKind}; use std::time::{Duration, Instant}; + fn stable_prepared_journal() -> Jentry { + Jentry { + persisted_version: 0, + obj_name: "remote/object".to_string(), + version_id: "remote-version".to_string(), + tier_name: "WARM".to_string(), + backend_identity: Some([7; 32]), + version_id_exact: true, + version_state: TransitionVersionState::Exact, + state: TierDeleteJournalState::Prepared, + source: Some(TierDeleteSourceIdentity { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: Some(uuid::Uuid::new_v4().to_string()), + versioned: true, + version_suspended: false, + data_dir: None, + etag: None, + mod_time: None, + }), + dispatch: None, + } + } + + #[test] + fn only_stable_prepared_journal_can_replace_tier_free_version() { + let stable = stable_prepared_journal(); + assert!(stable.can_replace_tier_free_version()); + + let mut committed = stable.clone(); + committed.state = TierDeleteJournalState::Committed; + assert!(!committed.can_replace_tier_free_version()); + + let mut unbound = stable.clone(); + unbound.backend_identity = None; + assert!(!unbound.can_replace_tier_free_version()); + + let mut unknown = stable.clone(); + unknown.version_state = TransitionVersionState::Unknown; + assert!(!unknown.can_replace_tier_free_version()); + + let mut unstable = stable; + unstable.source = Some(TierDeleteSourceIdentity { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + versioned: false, + version_suspended: false, + data_dir: None, + etag: Some("etag-only".to_string()), + mod_time: None, + }); + assert!(!unstable.can_replace_tier_free_version()); + } + #[test] fn signer_header_error_detection_matches_utf8_failures() { let err = Error::new( diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index c03a69e6e..3b1ae83fc 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{ ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, - TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient, - tier_mutation_control_service_client::TierMutationControlServiceClient, + TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, + node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient, }; pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS}; use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection}; @@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome { pub applied: bool, } +#[derive(Debug, thiserror::Error)] +#[error("{message}")] +struct TierMutationDefinitelyRejected { + message: String, +} + +fn tier_mutation_definitely_rejected_error(message: String) -> Error { + Error::other(TierMutationDefinitelyRejected { message }) +} + +#[cfg(test)] +pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error { + tier_mutation_definitely_rejected_error(message.to_string()) +} + +pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool { + matches!( + error, + Error::Io(io_error) + if io_error + .get_ref() + .is_some_and(|source| source.downcast_ref::().is_some()) + ) +} + fn validate_tier_mutation_response_proof( version: u32, phase: TierMutationRpcPhase, @@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof( canonical_payload: &[u8], response: &TierMutationControlResponse, ) -> Result<()> { + if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE { + return Err(Error::other("peer tier mutation response proof exceeds size limit")); + } + if response + .error_info + .as_ref() + .is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE) + { + return Err(Error::other("peer tier mutation error response exceeds size limit")); + } let canonical_response = rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput { version, @@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof( state: response.state, applied: response.applied, error_info: response.error_info.as_deref(), + failure_class: response.failure_class, }) .map_err(|_| Error::other("tier mutation response length cannot be represented"))?; verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof) @@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len: TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE, TierMutationRpcPhase::Abort => { if payload_len == 0 { - return Ok(()); + return Err(Error::other("tier mutation abort payload is empty")); } - return Err(Error::other("tier mutation abort payload must be empty")); + rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE } _ => return Err(Error::other("tier mutation rpc phase is unsupported")), }; @@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str { } } -fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error { - Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase))) +fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error { + let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)); + let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}"); + // RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class. + if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION + && status.code() == tonic::Code::FailedPrecondition + && status.message().as_bytes() == legacy_rejection.as_bytes() + { + return tier_mutation_definitely_rejected_error(message); + } + Error::other(message) +} + +fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option) -> Error { + let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string()); + if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION + && TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected) + { + return tier_mutation_definitely_rejected_error(message); + } + // Missing/zero, unknown, and explicit Ambiguous are deliberately the same + // fail-closed result: the coordinator must include this peer in Abort. + Error::other(message) } impl PeerRestClient { @@ -1315,8 +1372,12 @@ impl PeerRestClient { .await } - pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result { - self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new()) + pub async fn abort_tier_mutation( + &self, + mutation_id: Uuid, + canonical_prepare_payload: Bytes, + ) -> Result { + self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload) .await } @@ -1350,7 +1411,7 @@ impl PeerRestClient { client .prepare_tier_mutation(request) .await - .map_err(|status| tier_mutation_control_status_error(phase, status))? + .map_err(|status| tier_mutation_control_status_error(phase, version, status))? .into_inner() } TierMutationRpcPhase::Commit => { @@ -1363,7 +1424,7 @@ impl PeerRestClient { client .commit_tier_mutation(request) .await - .map_err(|status| tier_mutation_control_status_error(phase, status))? + .map_err(|status| tier_mutation_control_status_error(phase, version, status))? .into_inner() } TierMutationRpcPhase::Abort => { @@ -1376,18 +1437,19 @@ impl PeerRestClient { client .abort_tier_mutation(request) .await - .map_err(|status| tier_mutation_control_status_error(phase, status))? + .map_err(|status| tier_mutation_control_status_error(phase, version, status))? .into_inner() } _ => return Err(Error::other("tier mutation rpc phase is unsupported")), }; validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?; if !response.success { - return Err(Error::other( - response - .error_info - .unwrap_or_else(|| "peer tier mutation failed without an error".to_string()), - )); + return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info)); + } + if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION + && response.failure_class != TierMutationFailureClass::Unspecified as i32 + { + return Err(Error::other("successful peer tier mutation response carried a failure class")); } let state = decode_tier_mutation_peer_state(response.state)?; Ok(PeerTierMutationOutcome { @@ -3332,6 +3394,7 @@ mod tests { state: i32, applied: bool, error_info: Option<&'a str>, + failure_class: i32, } fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse { @@ -3345,6 +3408,7 @@ mod tests { state: input.state, applied: input.applied, error_info: input.error_info, + failure_class: input.failure_class, }) .expect("small tier mutation response should encode"); let response_proof = @@ -3355,6 +3419,7 @@ mod tests { applied: input.applied, error_info: input.error_info.map(str::to_string), response_proof: response_proof.into(), + failure_class: input.failure_class, } } @@ -3372,6 +3437,7 @@ mod tests { state: TierMutationPeerState::Prepared as i32, applied: true, error_info: None, + failure_class: TierMutationFailureClass::Unspecified as i32, }); validate_tier_mutation_response_proof( rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, @@ -3396,6 +3462,10 @@ mod tests { applied: false, ..response.clone() }, + TierMutationControlResponse { + failure_class: TierMutationFailureClass::Ambiguous as i32, + ..response.clone() + }, ] { let err = validate_tier_mutation_response_proof( rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, @@ -3419,6 +3489,44 @@ mod tests { assert!(err.to_string().contains("invalid tier mutation response proof")); } + #[test] + fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() { + let mutation_id = Uuid::new_v4(); + let payload = b"tier-mutation-prepare"; + let oversized_proof = TierMutationControlResponse { + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: None, + response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(), + failure_class: TierMutationFailureClass::Ambiguous as i32, + }; + let err = validate_tier_mutation_response_proof( + rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + payload, + &oversized_proof, + ) + .expect_err("oversized proof must fail before cryptographic verification"); + assert!(err.to_string().contains("response proof exceeds size limit")); + + let oversized_error = TierMutationControlResponse { + response_proof: Bytes::new(), + error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)), + ..oversized_proof + }; + let err = validate_tier_mutation_response_proof( + rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + payload, + &oversized_error, + ) + .expect_err("oversized error detail must fail before proof construction"); + assert!(err.to_string().contains("error response exceeds size limit")); + } + #[test] fn tier_mutation_peer_state_decode_fails_closed() { assert_eq!( @@ -3463,8 +3571,17 @@ mod tests { ) .is_err() ); - validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit"); - assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err()); + assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err()); + validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit"); + validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE) + .expect("max abort payload should fit"); + assert!( + validate_tier_mutation_payload_len( + TierMutationRpcPhase::Abort, + rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1, + ) + .is_err() + ); } #[test] @@ -3479,7 +3596,7 @@ mod tests { tonic::Status::deadline_exceeded("peer tier mutation control timed out"), tonic::Status::unavailable("peer tier mutation control unavailable"), ] { - let err = tier_mutation_control_status_error(phase, status); + let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status); let rendered = err.to_string(); assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}"); assert!( @@ -3493,6 +3610,61 @@ mod tests { } } + #[test] + fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() { + let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION; + let exact = format!("unsupported tier mutation peer protocol version: {version}"); + let rejected = tier_mutation_control_status_error( + TierMutationRpcPhase::Prepare, + version, + tonic::Status::failed_precondition(exact.clone()), + ); + assert!(tier_mutation_error_is_definitely_rejected(&rejected)); + + for status in [ + tonic::Status::failed_precondition(format!("{exact}.")), + tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)), + tonic::Status::invalid_argument(exact.clone()), + tonic::Status::unimplemented(exact), + ] { + let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status); + assert!( + !tier_mutation_error_is_definitely_rejected(&ambiguous), + "near-text, wrong-code, and Unimplemented failures must remain ambiguous" + ); + } + } + + #[test] + fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() { + let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION; + let rejected = tier_mutation_failed_response_error( + version, + TierMutationFailureClass::PreDispatchRejected as i32, + Some("rejected".to_string()), + ); + assert!(tier_mutation_error_is_definitely_rejected(&rejected)); + + for failure_class in [ + TierMutationFailureClass::Unspecified as i32, + TierMutationFailureClass::Ambiguous as i32, + 99, + ] { + let ambiguous = tier_mutation_failed_response_error(version, failure_class, None); + assert!( + !tier_mutation_error_is_definitely_rejected(&ambiguous), + "missing, unknown, and explicit ambiguous classes must trigger Abort fanout" + ); + } + + let v3_ignores_v4_class = tier_mutation_failed_response_error( + rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, + TierMutationFailureClass::PreDispatchRejected as i32, + Some("legacy failure".to_string()), + ); + assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class)); + } + #[tokio::test] async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() { let client = test_peer_client(); diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 0bc96dabb..e8f257433 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -18,7 +18,8 @@ use crate::bucket::utils::is_meta_bucketname; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ - DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord, + DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, + ValidatedDurableIlmRecord, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in, @@ -26,6 +27,7 @@ use crate::bucket::{ }, classify_durable_ilm_record, get_expiry_configs, lifecycle::IlmAction, + tier_delete_journal::durable_ilm_v6_topology_generation, validate_durable_ilm_record, }, metadata_sys, @@ -49,6 +51,9 @@ use crate::error::{ use crate::layout::endpoints::EndpointServerPools; use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions}; use crate::runtime::sources as runtime_sources; +use crate::services::notification_sys::{ + acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation, +}; use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::storage_api_contracts::{ @@ -2384,10 +2389,12 @@ struct DecommissionDurableIlmReceipt { id: String, checkpoint: DurableIlmRecordCheckpoint, terminal_checkpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + fleet_topology_generation: Option, } impl DecommissionDurableIlmReceipt { - fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self { + fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option) -> Self { Self { source_path: path.to_string(), namespace: record.namespace.to_string(), @@ -2395,6 +2402,7 @@ impl DecommissionDurableIlmReceipt { id: record.id.clone(), checkpoint: record.checkpoint.clone(), terminal_checkpoint: None, + fleet_topology_generation, } } @@ -2433,6 +2441,16 @@ impl DecommissionDurableIlmReceipt { )) })?; } + if self + .fleet_topology_generation + .as_deref() + .is_some_and(|generation| !is_sha256_checksum(generation)) + { + return Err(Error::other_with_context( + "receipt fleet topology generation is invalid", + format!("source path `{}` {}", self.source_path, self.context()), + )); + } Ok(()) } @@ -2541,6 +2559,21 @@ fn merge_decommission_durable_ilm_receipts( id: existing.id.clone(), checkpoint, terminal_checkpoint, + fleet_topology_generation: match ( + existing.fleet_topology_generation.as_ref(), + incoming.fleet_topology_generation.as_ref(), + ) { + (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => { + Some(existing_generation.clone()) + } + (None, None) => None, + _ => { + return Err(Error::other_with_context( + "durable ILM receipt fleet topology conflict", + format!("source path `{}` {}", existing.source_path, existing.context()), + )); + } + }, }; merged.validate()?; Ok(merged) @@ -7050,6 +7083,15 @@ pub(crate) struct DecommissionCapacityOwner { pub(crate) mutation_id: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DecommissionDurableIlmCheckpointTarget { + pub(crate) source_pool_index: usize, + pub(crate) target_pool_index: usize, + pub(crate) capacity_owner: DecommissionCapacityOwner, + pub(crate) already_committed: bool, + pub(crate) target_etag: Option, +} + impl DecommissionCapacityOwner { pub(crate) fn apply_to(self, opts: &mut ObjectOptions) { opts.src_pool_idx = self.source_pool_index; @@ -7457,6 +7499,7 @@ impl DecommissionCapacityLockOrderBarrier { Self { state } } + #[cfg(feature = "test-util")] pub(crate) async fn wait_until_owner_paused(&self) { tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified()) .await @@ -7469,6 +7512,7 @@ impl DecommissionCapacityLockOrderBarrier { .expect("external mutation should release capacity before waiting for the object namespace"); } + #[cfg(feature = "test-util")] pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) { tokio::time::timeout( std::time::Duration::from_secs(30), @@ -7478,6 +7522,7 @@ impl DecommissionCapacityLockOrderBarrier { .expect("external object mutation should acquire its no-active capacity probe"); } + #[cfg(feature = "test-util")] pub(crate) async fn wait_until_external_object_commit_phase_started(&self) { tokio::time::timeout( std::time::Duration::from_secs(30), @@ -7502,24 +7547,29 @@ impl DecommissionCapacityLockOrderBarrier { .expect("external heal should attempt the target namespace lock after capacity admission"); } + #[cfg(feature = "test-util")] pub(crate) fn release_owner(&self) { self.state.owner_release.notify_one(); } + #[cfg(feature = "test-util")] pub(crate) fn pause_external_object_commit_phase(&self) { self.state.external_object_commit_phase_paused.store(true, Ordering::Release); } + #[cfg(feature = "test-util")] pub(crate) fn release_external_object_commit_phase(&self) { self.state.external_object_commit_phase_release.notify_one(); } + #[cfg(feature = "test-util")] pub(crate) fn pause_external_object_capacity_probe(&self) { self.state .external_object_capacity_probe_paused .store(true, Ordering::Release); } + #[cfg(feature = "test-util")] pub(crate) fn release_external_object_capacity_probe(&self) { self.state.external_object_capacity_probe_release.notify_one(); } @@ -8315,6 +8365,122 @@ impl ECStore { .await } + /// Finish the capacity transaction for an identity-preserving temporary + /// replacement whose target bytes are already durably present. This is + /// the crash-recovery half of `run_decommission_capacity_temporary_mutation`: + /// it never writes the target again and only resolves a pending intent + /// owned by the exact deterministic mutation id. + pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target( + &self, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + expected_data_bytes: usize, + ) -> Result<()> { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (pool_meta_guard, mut snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed") + .await?; + let source_pool_index = owner.source_pool_index; + let mutation_id = owner + .mutation_id + .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?; + let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = { + let reservation = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_cleanup_owner(owner)) + .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?; + let target = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?; + ( + target.layout, + target.pending_physical_bytes, + target.pending_mutation_id, + target + .temporary_mutations + .iter() + .any(|mutation| mutation.mutation_id == mutation_id), + ) + }; + if pending_physical_bytes == 0 { + // Either the successful attempt already saved its progress, or a + // byte-neutral replacement had no inflight delta to record. + ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?; + return Ok(()); + } + if pending_mutation_id != Some(mutation_id) { + return Err(decommission_capacity_blocked_error( + "equivalent temporary target pending intent belongs to another mutation", + )); + } + if already_reconciled { + return Err(decommission_capacity_blocked_error( + "equivalent temporary target has both pending and reconciled state", + )); + } + let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?; + if pending_physical_bytes < expected_target_physical_bytes { + return Err(decommission_capacity_blocked_error( + "equivalent temporary target pending capacity is smaller than the committed checkpoint", + )); + } + resolve_decommission_target_pending( + &mut snapshot, + source_pool_index, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + )?; + // The replacement is byte-non-growing and its exact bytes were read + // before this call, so no new physical delta is inferred on replay. + // A prior successful progress save would have taken the idempotent + // pending==0 return above. + record_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + 0, + mutation_id, + OffsetDateTime::now_utc(), + )?; + let outcome = snapshot + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + pool_meta_guard.lock_lost_signal(), + &[source_pool_index], + ) + .await?; + ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?; + let persisted_info = outcome + .committed + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .cloned() + .ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent temporary target reconciliation"))?; + { + let mut pool_meta = self.pool_meta.write().await; + let pool_count = pool_meta.pools.len(); + pool_meta.version = pool_meta.version.max(outcome.committed.version); + let info = pool_meta + .pools + .get_mut(source_pool_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + info.capacity_reservation = persisted_info.capacity_reservation; + info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + } + ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?; + outcome.disarm(); + Ok(()) + } + pub(crate) async fn has_decommission_capacity_temporary_mutation_state( &self, target_pool_index: usize, @@ -12847,6 +13013,7 @@ impl ECStore { let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id); let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; let mut proof = None::; + let mut nonterminal_receipt_found = false; for pool_idx in 0..self.pools.len() { if pool_idx == source_pool_idx { continue; @@ -12890,23 +13057,35 @@ impl ECStore { receipt.context() ))); } - source_record - .checkpoint - .validate_successor(&receipt.checkpoint) - .map_err(|err| { - Error::other(format!( - "terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}", - source_record.context() - )) - })?; - if receipt.terminal_checkpoint.is_some() { + if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() { + if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) { + return Err(Error::other_with_context( + "terminal durable ILM decommission receipt does not cover source", + format!("path `{path}` {}", source_record.context()), + )); + } proof = Some(match proof { Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?, None => receipt, }); + } else { + source_record + .checkpoint + .validate_successor(&receipt.checkpoint) + .map_err(|err| { + Error::other_with_context( + "durable ILM decommission receipt does not cover source", + format!("path `{path}` {}: {err}", source_record.context()), + ) + })?; + nonterminal_receipt_found = true; } } - Ok(proof) + // A terminal receipt on one target must not hide another target copy + // whose receipt was installed later and has not reached terminal yet. + // Returning no proof makes recovery advance every outstanding copy + // before source cleanup can treat the operation as complete. + if nonterminal_receipt_found { Ok(None) } else { Ok(proof) } } async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { @@ -13050,6 +13229,7 @@ impl ECStore { pool_idx: usize, receipt_path: &str, record: &ValidatedDurableIlmRecord, + fleet_topology_generation: Option<&str>, terminal: bool, ) -> Result { let stage = if terminal { "terminal" } else { "progress" }; @@ -13085,6 +13265,12 @@ impl ECStore { )) })?; Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?; + if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation { + return Err(Error::other_with_context( + "durable ILM decommission receipt fleet topology mismatch", + format!("path `{}` {}", receipt.source_path, receipt.context()), + )); + } receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| { Error::other(format!( "{stage} durable ILM record generation mismatch at path `{}` {}: {err}", @@ -13197,6 +13383,7 @@ impl ECStore { let stage = if terminal { "terminal" } else { "progress" }; let record = validate_durable_ilm_record(path, data) .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?; + let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?; let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::>(); let mut terminal_target_pool_indices = Vec::new(); for (source_pool_idx, run_token) in active_runs { @@ -13205,7 +13392,13 @@ impl ECStore { for pool_idx in 0..self.pools.len() { if pool_idx != source_pool_idx { let found = self - .advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal) + .advance_durable_ilm_decommission_receipt( + pool_idx, + &receipt_path, + &record, + fleet_topology_generation.as_deref(), + terminal, + ) .await?; receipt_found |= found; if terminal @@ -13227,6 +13420,240 @@ impl ECStore { Ok(Some(terminal_target_pool_indices)) } + /// Resolve the exact receipt-bearing target copies on which a v6 dispatch + /// manifest may advance while a decommission reservation is active. + /// + /// This is intentionally not a general capacity bypass. The target copy + /// must still be covered by the active source reservation and its durable + /// receipt, the ETag must be the caller's exact CAS generation, and the + /// replacement must be a byte-non-growing adjacent manifest checkpoint. + pub(crate) async fn decommission_durable_ilm_checkpoint_targets( + &self, + path: &str, + next_data: &[u8], + expected_etag: &str, + ) -> Result>> { + let active_runs = { + let pool_meta = self.pool_meta.read().await; + let mut active_runs = Vec::new(); + for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() { + let Some(info) = pool + .decommission + .as_ref() + .filter(|info| info.has_decommission_state() && !info.complete) + else { + continue; + }; + let Some(start_time) = info.start_time else { + continue; + }; + let reservation = info.capacity_reservation.clone().ok_or_else(|| { + decommission_capacity_blocked_error(format!( + "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint" + )) + })?; + if !reservation.active() { + return Err(decommission_capacity_blocked_error(format!( + "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint" + ))); + } + active_runs.push(( + source_pool_index, + decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time), + reservation, + )); + } + active_runs + }; + if active_runs.is_empty() { + return Ok(None); + } + + let next_record = validate_durable_ilm_record(path, next_data)?; + if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name { + return Ok(None); + } + let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?; + + let mut targets = Vec::::new(); + let mut missing_source_receipts = 0usize; + let mut stale_target_etag_mismatch = false; + for (source_pool_index, run_token, reservation) in active_runs { + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id); + let mut source_receipt_found = false; + for allocation in &reservation.targets { + let receipt_data = match read_config_limited_preserve_empty( + self.pools[allocation.pool_index].clone(), + &receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok(data) => data, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + continue; + } + Err(err) => return Err(err), + }; + let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?; + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?; + if receipt.source_path != path + || receipt.namespace != next_record.namespace + || receipt.id_kind != next_record.id_kind + || receipt.id != next_record.id + { + return Err(Error::other_with_context( + "durable ILM checkpoint receipt identity does not authorize source", + format!("path `{path}` {}", next_record.context()), + )); + } + if receipt.fleet_topology_generation != next_fleet_topology_generation { + return Err(Error::other_with_context( + "durable ILM checkpoint receipt fleet topology does not authorize source", + format!("path `{path}` {}", next_record.context()), + )); + } + receipt + .checkpoint + .validate_successor(&next_record.checkpoint) + .map_err(|err| { + Error::other_with_context( + "durable ILM checkpoint receipt is not a predecessor of the requested generation", + format!( + "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}", + allocation.pool_index, + next_record.context(), + receipt.checkpoint, + next_record.checkpoint + ), + ) + })?; + + let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata( + self.pools[allocation.pool_index].clone(), + path, + TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size, + ) + .await?; + let target_record = validate_durable_ilm_record(path, &target_data)?; + if target_record.namespace != next_record.namespace + || target_record.id_kind != next_record.id_kind + || target_record.id != next_record.id + { + return Err(Error::other_with_context( + "durable ILM checkpoint target identity does not match source", + format!("path `{path}` {}", next_record.context()), + )); + } + let already_committed = target_data.as_slice() == next_data; + let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| { + Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`")) + })?; + if already_committed { + receipt + .checkpoint + .validate_successor(&target_record.checkpoint) + .map_err(|err| { + Error::other_with_context( + "durable ILM checkpoint receipt is not a predecessor of the committed target generation", + format!("path `{path}` {}: {err}", next_record.context()), + ) + })?; + } else { + if target_record.checkpoint != receipt.checkpoint { + return Err(Error::other_with_context( + "durable ILM checkpoint target is not the receipt generation", + format!("path `{path}` {}", next_record.context()), + )); + } + target_record + .checkpoint + .validate_successor(&next_record.checkpoint) + .map_err(|err| { + Error::other_with_context( + "durable ILM checkpoint target is not a predecessor of the requested generation", + format!("path `{path}` {}: {err}", next_record.context()), + ) + })?; + if next_data.len() > target_data.len() { + return Err(decommission_capacity_blocked_error(format!( + "durable ILM checkpoint update at `{path}` grows from {} to {} bytes", + target_data.len(), + next_data.len() + ))); + } + stale_target_etag_mismatch |= target_etag != expected_etag; + } + + source_receipt_found = true; + if let Some(existing) = targets + .iter_mut() + .find(|target| target.target_pool_index == allocation.pool_index) + { + if existing.already_committed != already_committed { + return Err(Error::other_with_context( + "durable ILM checkpoint target state changed during authorization", + format!("path `{path}`"), + )); + } + continue; + } + let base_owner = DecommissionCapacityOwner { + source_pool_index, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + }; + let mutation_id = decommission_capacity_mutation_id( + base_owner, + RUSTFS_META_BUCKET, + path, + Some(next_record.checkpoint.content_sha256()), + false, + None, + ); + targets.push(DecommissionDurableIlmCheckpointTarget { + source_pool_index, + target_pool_index: allocation.pool_index, + capacity_owner: base_owner.with_mutation_id(mutation_id), + already_committed, + target_etag: Some(target_etag), + }); + } + if !source_receipt_found { + missing_source_receipts = missing_source_receipts.saturating_add(1); + } + } + + if missing_source_receipts > 0 { + return Err(decommission_capacity_blocked_error(format!( + "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)" + ))); + } + if targets.is_empty() { + return Err(decommission_capacity_blocked_error(format!( + "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target" + ))); + } + if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) { + return Err(Error::PreconditionFailed); + } + // After a partial multi-target commit, an aggregate read may return + // the ETag of the already-advanced target while another authorized + // target still has the predecessor ETag. The exact committed bytes + // plus every target's receipt/checkpoint proof authorize repairing + // that predecessor with its own target-local CAS. Without an exact + // committed target, retain the caller-ETag requirement above. + targets.sort_unstable_by_key(|target| target.target_pool_index); + Ok(Some(targets)) + } + pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> { self.advance_durable_ilm_decommission_receipts(path, data, false) .await @@ -13248,6 +13675,66 @@ impl ECStore { self.advance_durable_ilm_decommission_receipts(path, data, true).await } + /// Return true only when `data` is the exact copy still owned by an active + /// decommission source and a target-side terminal receipt authorizes that + /// source's later verified cleanup. Lifecycle recovery may then regard the + /// logical record as terminal without deleting the source checkpoint. + pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result { + let namespace = classify_durable_ilm_record(path)? + .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?; + let source_record = validate_durable_ilm_record(path, data)?; + let active_runs = { + let pool_meta = self.pool_meta.read().await; + pool_meta + .pools + .iter() + .enumerate() + .filter_map(|(source_pool_index, pool)| { + pool.decommission + .as_ref() + .filter(|info| info.has_decommission_state() && !info.complete) + .and_then(|info| info.start_time) + .map(|start_time| { + (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)) + }) + }) + .collect::>() + }; + let mut covered_active_source = false; + for (source_pool_index, run_token) in active_runs { + let source_data = + match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size) + .await + { + Ok(source_data) => source_data, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + continue; + } + Err(err) => return Err(err), + }; + if source_data.as_slice() != data { + continue; + } + if self + .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token) + .await? + .is_some() + { + covered_active_source = true; + } else { + // Every active source that still stores this exact generation + // needs complete terminal receipt coverage. One covered source + // cannot authorize lifecycle recovery to skip another. + return Ok(false); + } + } + Ok(covered_active_source) + } + async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? { match delete_config(self.pools[pool_idx].clone(), &receipt_path).await { @@ -13316,12 +13803,20 @@ impl ECStore { .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?; let source_record = validate_durable_ilm_record(path, &source) .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?; + let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?; let target = self .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context()) .await?; let manifest_receipt = if let Some((target_pool_idx, target)) = target { let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?; - let receipt = DecommissionDurableIlmReceipt::new(path, &target_record); + let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?; + if target_fleet_topology_generation != source_fleet_topology_generation { + return Err(Error::other_with_context( + "target durable ILM fleet topology generation differs from source", + format!("path `{path}` {}", source_record.context()), + )); + } + let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation); self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token) .await?; receipt @@ -13335,9 +13830,34 @@ impl ECStore { )) })? }; + if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation { + return Err(Error::other_with_context( + "terminal durable ILM receipt fleet topology generation does not cover source", + format!("path `{path}` {}", source_record.context()), + )); + } + let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() { + let proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?; + if tier_delete_journal_topology_generation(&proof) != expected_generation + || !tier_delete_journal_fleet_proof_matches(&proof) + { + return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup")); + } + Some(proof) + } else { + None + }; self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token) .await?; + if fleet_proof + .as_ref() + .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof)) + { + return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup")); + } + let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( source_set, RUSTFS_META_BUCKET, @@ -13354,7 +13874,16 @@ impl ECStore { source_record.context() )) }); - resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path) + let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path); + if fleet_proof + .as_ref() + .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof)) + { + return Err(Error::other( + "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification", + )); + } + cleanup_result } #[cfg(all(test, feature = "test-util"))] @@ -13390,7 +13919,26 @@ impl ECStore { record: &ValidatedDurableIlmRecord, terminal: bool, ) -> Result { - let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record); + let fleet_topology_generation = match read_config_limited_preserve_empty( + self.pools[target_pool_idx].clone(), + source_path, + classify_durable_ilm_record(source_path)? + .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))? + .max_record_size, + ) + .await + { + Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + None + } + Err(err) => return Err(err), + }; + let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation); if terminal { receipt.terminal_checkpoint = Some(record.checkpoint.clone()); } @@ -17361,11 +17909,15 @@ mod pools_tests { content_sha256: "b".repeat(64), identity_sha256: "c".repeat(64), committed: false, + dispatch_identity_sha256: None, + state: None, }; let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal { content_sha256: "d".repeat(64), identity_sha256: "c".repeat(64), committed: true, + dispatch_identity_sha256: None, + state: None, }; let incoming = DecommissionDurableIlmReceipt { source_path, @@ -17374,6 +17926,7 @@ mod pools_tests { id: operation_id, checkpoint: checkpoint.clone(), terminal_checkpoint: None, + fleet_topology_generation: None, }; let existing = DecommissionDurableIlmReceipt { terminal_checkpoint: Some(terminal_checkpoint.clone()), @@ -17384,7 +17937,24 @@ mod pools_tests { .expect("retry receipt must merge with a terminal receipt"); assert_eq!(merged.checkpoint, checkpoint); - assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint)); + assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone())); + + let topology_bound = DecommissionDurableIlmReceipt { + fleet_topology_generation: Some("e".repeat(64)), + ..incoming + }; + let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound) + .expect_err("a topology-bound v6 receipt must not mask an unbound receipt") + .to_string(); + assert!(mixed_error.contains("fleet topology conflict")); + + let topology_existing = DecommissionDurableIlmReceipt { + terminal_checkpoint: Some(terminal_checkpoint), + ..topology_bound.clone() + }; + let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound) + .expect("receipts bound to the same fleet topology should merge"); + assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64))); } #[test] @@ -17409,7 +17979,7 @@ mod pools_tests { assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE); let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate"); let expected_checkpoint = record.checkpoint.clone(); - let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record); + let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None); receipt.terminal_checkpoint = Some(record.checkpoint); let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit"); diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 371a3f326..ab51167e8 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -37,8 +37,9 @@ use crate::{ runtime::instance::{InstanceContext, bootstrap_ctx}, runtime::sources as runtime_sources, set_disk::{PreparedGetObjectMetadata, SetDisks}, - store::init_format::{ - check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum, + store::{ + RemoteTuplePublicationFence, + init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum}, }, }; use futures::{ @@ -625,6 +626,19 @@ impl Sets { .put_object_with_old_current_size(bucket, object, data, opts) .await } + + pub(crate) async fn put_object_with_old_current_size_for_data_movement( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + publication_fence: RemoteTuplePublicationFence, + ) -> Result<(ObjectInfo, Option)> { + self.get_disks_by_key(object) + .put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence) + .await + } } #[async_trait::async_trait] diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index ee6f1df48..6d65fa714 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -27,7 +27,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{HTTPPreconditions, ObjectOperations as _}, }; -use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence}; +use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence}; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; @@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc) { flag.store(false, Ordering::Relaxed); } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] struct DataMovementMultipartAbortBarrierState { bucket: String, object: String, @@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct DataMovementMultipartAbortBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl DataMovementMultipartAbortBarrier { pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(DataMovementMultipartAbortBarrierState { @@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for DataMovementMultipartAbortBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) { let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object( capacity_owner: Option, ) -> Result<()> { let source = rd.object_info.clone(); - let _mutation_fence = store + let mutation_fence = store .acquire_decommission_object_mutation_fence(&bucket, &source.name) .await?; let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source) @@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object( op_label, None, capacity_owner, - Some(&_mutation_fence), + Some(mutation_fence), ) .await } @@ -1588,8 +1588,9 @@ async fn migrate_object_inner( op_label: &str, lock_lost_signal: Option>, capacity_owner: Option, - mutation_fence: Option<&ObjectLockDiagGuard>, + mutation_fence: Option, ) -> Result<()> { + let mut mutation_fence = mutation_fence; let object_info = rd.object_info.clone(); let capacity_owner = capacity_owner.map(|owner| { let version_id = object_info.version_id.map(|version_id| version_id.to_string()); @@ -1605,6 +1606,13 @@ async fn migrate_object_inner( }); owner.with_mutation_id(mutation_id) }); + // Capture the exact source/tier identity before any client-paced read, but + // defer both the tier lease and source/target write locks to the final + // publication. Decommission already owns main's fixed-domain mutation + // fence, so reacquiring that domain as a write lock would self-deadlock. + let remote_tuple_publication_fence = store + .acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false) + .await?; let has_part_checksums = object_info .parts .iter() @@ -1656,8 +1664,8 @@ async fn migrate_object_inner( } let mut cleanup_opts = data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); - if let Some(fence) = mutation_fence { - fence.add_namespace_lock_fence(&mut cleanup_opts); + if let Some(anchor) = mutation_fence.as_ref() { + anchor.guard().add_namespace_lock_fence(&mut cleanup_opts); } if let Some(fence) = multipart_mutation_fence.as_ref() { fence.add_namespace_lock_fence(&mut cleanup_opts); @@ -1684,7 +1692,12 @@ async fn migrate_object_inner( } } let (res, target_pool_idx, expected_bucket_incarnation_id) = match store - .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence) + .handle_new_multipart_upload_with_pool_idx( + &bucket, + &object_info.name, + &new_multipart_opts, + mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard), + ) .await { Ok(res) => res, @@ -1797,15 +1810,20 @@ async fn migrate_object_inner( if let Some(fence) = multipart_mutation_fence.as_ref() { fence.add_namespace_lock_fence(&mut complete_multipart_opts); } + let remote_tuple_publication_fence = match mutation_fence.take() { + Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?, + None => remote_tuple_publication_fence, + }; if let Err(err) = store .clone() - .complete_multipart_upload_for_data_movement( - (target_pool_idx, mutation_fence), + .complete_multipart_upload_for_data_movement_with_publication_fence( + target_pool_idx, &bucket, &object_info.name, &res.upload_id, parts, &complete_multipart_opts, + remote_tuple_publication_fence, ) .await { @@ -1849,8 +1867,8 @@ async fn migrate_object_inner( if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) { let mut abort_opts = data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); - if let Some(fence) = mutation_fence { - fence.add_namespace_lock_fence(&mut abort_opts); + if let Some(anchor) = mutation_fence.as_ref() { + anchor.guard().add_namespace_lock_fence(&mut abort_opts); } if let Some(fence) = multipart_mutation_fence.as_ref() { fence.add_namespace_lock_fence(&mut abort_opts); @@ -1923,12 +1941,12 @@ async fn migrate_object_inner( if let Err(primary_err) = multipart_result { if should_abort_multipart_upload(&abort_multipart_flag) { - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await; let mut abort_opts = data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); - if let Some(fence) = mutation_fence { - fence.add_namespace_lock_fence(&mut abort_opts); + if let Some(anchor) = mutation_fence.as_ref() { + anchor.guard().add_namespace_lock_fence(&mut abort_opts); } if let Some(fence) = multipart_mutation_fence.as_ref() { fence.add_namespace_lock_fence(&mut abort_opts); @@ -1982,8 +2000,18 @@ async fn migrate_object_inner( if let Some(signal) = lock_lost_signal { put_opts.add_namespace_lock_lost_signal(signal); } + let remote_tuple_publication_fence = match mutation_fence.take() { + Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?, + None => remote_tuple_publication_fence, + }; let (target_pool_idx, put_result) = store - .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence) + .put_object_for_data_movement_with_publication_fence( + &bucket, + &object_info.name, + &mut data, + &put_opts, + remote_tuple_publication_fence, + ) .await .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?; if let Err(err) = put_result { diff --git a/crates/ecstore/src/error/conversion_roundtrip_tests.rs b/crates/ecstore/src/error/conversion_roundtrip_tests.rs index 6602ad76f..9a01d16a2 100644 --- a/crates/ecstore/src/error/conversion_roundtrip_tests.rs +++ b/crates/ecstore/src/error/conversion_roundtrip_tests.rs @@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() { assert_eq!(count, 3); assert_eq!(err, Some(DiskError::other("can not get client"))); } + +#[test] +fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() { + let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a"); + let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b"); + + assert_eq!(first, second, "diagnostic identity must not split quorum buckets"); + let StorageError::Io(io_error) = first else { + panic!("stable context must remain an io error"); + }; + assert_eq!(io_error.to_string(), "tier mutation intent changed"); + let context = io_error.get_ref().expect("stable context must remain downcastable"); + assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a"); +} diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 3f5035634..52018d6b4 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -23,6 +23,36 @@ use s3s::S3ErrorCode; pub type Error = StorageError; pub type Result = core::result::Result; +/// Keeps high-cardinality diagnostic detail in the error source while making +/// the rendered `io::Error` stable for quorum aggregation. +#[derive(Debug)] +struct StableIoContextError { + message: &'static str, + source: Box, +} + +impl std::fmt::Display for StableIoContextError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.message) + } +} + +impl std::error::Error for StableIoContextError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +pub(crate) fn stable_io_error(message: &'static str, source: E) -> std::io::Error +where + E: Into>, +{ + std::io::Error::other(StableIoContextError { + message, + source: source.into(), + }) +} + /// Storage layer error type covering disk, volume, bucket, object, multipart, /// erasure-coding, and operational error conditions. /// @@ -264,6 +294,13 @@ impl StorageError { StorageError::Io(std::io::Error::other(error)) } + pub(crate) fn other_with_context(message: &'static str, source: E) -> Self + where + E: Into>, + { + StorageError::Io(stable_io_error(message, source)) + } + pub fn is_not_found(&self) -> bool { matches!( self, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 88b648b49..e5cd0dd75 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase { #[doc(hidden)] #[derive(Default)] pub struct LifecycleDeleteAllJournalState { - prepared: HashMap, mutation_started: bool, } impl Debug for LifecycleDeleteAllJournalState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LifecycleDeleteAllJournalState") - .field("prepared_count", &self.prepared.len()) .field("mutation_started", &self.mutation_started) .finish() } } impl LifecycleDeleteAllJournalState { - pub(crate) fn contains(&self, name: &str) -> bool { - self.prepared.contains_key(name) - } - - pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) { - self.prepared.insert(name, entry); - } - - pub(crate) fn prepared_entries(&self) -> Vec { - self.prepared.values().cloned().collect() - } - pub(crate) fn mark_mutation_started(&mut self) { self.mutation_started = true; } + #[cfg(test)] pub(crate) fn mutation_started(&self) -> bool { self.mutation_started } @@ -706,6 +693,12 @@ pub struct ObjectOptions { pub lifecycle_delete_all: Option, #[doc(hidden)] pub lifecycle_delete_all_journal: Option>>, + /// Whole-operation authorization created only by consuming a validated + /// v6 dispatch-manifest permit. Clones share the authorization, not the + /// one-shot permit itself. + #[doc(hidden)] + pub tier_delete_dispatch_authorization: + Option, /// RustFS-only compare-and-set condition checked under the object write lock. pub expected_current_version_id: Option, /// Persisted bucket incarnation observed before authorization. @@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions { .field("version_id", &self.version_id.is_some()) .field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some()) .field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some()) + .field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some()) .field("expected_current_version_id", &self.expected_current_version_id.is_some()) .field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id) .field("no_lock", &self.no_lock) @@ -972,7 +966,7 @@ impl ObjectOptions { } #[cfg(test)] - pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) { + pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) { self.namespace_lock_fence .get_or_insert_with(NamespaceLockFence::new) .extend(fence); diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 19660dd84..b6eafe086 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics; use rustfs_madmin::net::NetInfo; use rustfs_madmin::{ItemState, ServerProperties, StorageInfo}; use rustfs_utils::XHost; +use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher}; use std::future::Future; use std::hash::{Hash, Hasher}; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; use std::time::{Duration, Instant, SystemTime}; use tokio::time::{sleep, timeout}; use tokio_util::sync::CancellationToken; @@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2; +const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3; +type CrossPoolFencePolicyResult = Result>; + +fn cross_pool_fence_policy_results( + peer_epochs: BTreeMap, + minimum_version: u32, +) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) { + let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION { + Ok(peer_epochs.clone()) + } else { + Err(Error::other("tier delete journal v6 policy capability version is unsupported")) + }; + (Ok(peer_epochs), journal_result) +} #[derive(Clone, Debug)] pub struct ScannerPublicationLeaseGrant { @@ -107,15 +125,91 @@ struct FleetCapabilityProof { topology_fingerprint: String, peer_epochs: Arc>, expires_at: Instant, + generation: Arc, } impl FleetCapabilityProof { + fn new(topology_fingerprint: String, peer_epochs: Arc>, expires_at: Instant) -> Self { + Self { + topology_fingerprint, + peer_epochs, + expires_at, + generation: FleetCapabilityProofGeneration::fresh(), + } + } + fn token(&self) -> FleetCapabilityProofToken { FleetCapabilityProofToken { topology_fingerprint: self.topology_fingerprint.clone(), peer_epochs: self.peer_epochs.clone(), } } + + #[cfg(any(test, feature = "test-util"))] + fn with_fresh_generation(&self) -> Self { + Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at) + } +} + +/// Admission generation for effects that must not straddle a fleet-proof +/// replacement. Revocation is deliberately non-blocking: it closes admission +/// immediately, while the proof slot withholds the successor generation until +/// every admitted operation has drained. +#[derive(Default)] +struct FleetCapabilityProofGeneration { + accepting: AtomicBool, + active: AtomicUsize, +} + +impl FleetCapabilityProofGeneration { + fn fresh() -> Arc { + Arc::new(Self { + accepting: AtomicBool::new(true), + active: AtomicUsize::new(0), + }) + } + + fn try_acquire(self: &Arc) -> Option { + if !self.accepting.load(Ordering::Acquire) { + return None; + } + self.active.fetch_add(1, Ordering::AcqRel); + if self.accepting.load(Ordering::Acquire) { + Some(FleetCapabilityProofPermit { + generation: Arc::clone(self), + }) + } else { + self.release(); + None + } + } + + fn revoke(&self) { + self.accepting.store(false, Ordering::Release); + } + + fn is_accepting(&self) -> bool { + self.accepting.load(Ordering::Acquire) + } + + fn is_drained(&self) -> bool { + self.active.load(Ordering::Acquire) == 0 + } + + fn release(&self) { + let previous = self.active.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "fleet capability permit count underflow"); + } +} + +struct FleetCapabilityProofPermit { + generation: Arc, +} + +impl Drop for FleetCapabilityProofPermit { + fn drop(&mut self) { + self.generation.release(); + } } #[derive(Clone, PartialEq, Eq)] @@ -127,6 +221,7 @@ struct FleetCapabilityProofToken { #[derive(Default)] struct FleetCapabilityProofState { proof: Option, + draining_generation: Option>, topology_conflict: bool, } @@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken); #[derive(Clone, PartialEq, Eq)] pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken); +/// A point-in-time proof that every current storage member implements the v6 +/// dispatch-manifest policy. It intentionally has no `Clone` implementation: +/// one acquisition authorizes one manifest construction attempt. +pub(crate) struct TierDeleteJournalFleetProofToken { + token: FleetCapabilityProofToken, + _permit: FleetCapabilityProofPermit, +} + static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); +static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { @@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock, proof: Option) { - slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof; +fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock { + TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) +} + +fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) { + if let Some(proof) = state.proof.take() { + proof.generation.revoke(); + if !proof.generation.is_drained() { + state.draining_generation = Some(proof.generation); + } + } + if state + .draining_generation + .as_ref() + .is_some_and(|generation| generation.is_drained()) + { + state.draining_generation = None; + } +} + +fn revoke_fleet_capability_proof(slot: &std::sync::RwLock) { + let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); + revoke_fleet_capability_proof_state(&mut state); +} + +fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock) { + let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); + state.topology_conflict = true; + revoke_fleet_capability_proof_state(&mut state); } fn publish_fleet_capability_probe_result( @@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result( match result { Ok(peer_epochs) => { let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); - let peer_epochs = state + if let Some(current) = state .proof - .as_ref() + .as_mut() .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs) - .map(|proof| Arc::clone(&proof.peer_epochs)) - .unwrap_or_else(|| Arc::new(peer_epochs)); - state.proof = Some(FleetCapabilityProof { - topology_fingerprint: topology_fingerprint.to_string(), - peer_epochs, - expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL, - }); + { + current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL; + return None; + } + + if let Some(previous) = state.proof.take() { + previous.generation.revoke(); + if !previous.generation.is_drained() { + state.draining_generation = Some(previous.generation); + } + } + if state + .draining_generation + .as_ref() + .is_some_and(|generation| generation.is_drained()) + { + state.draining_generation = None; + } + if state.draining_generation.is_some() { + return Some(Error::other( + "fleet capability proof successor waits for the previous generation to drain", + )); + } + state.proof = Some(FleetCapabilityProof::new( + topology_fingerprint.to_string(), + Arc::new(peer_epochs), + observed_at + REMOTE_VERSION_STATE_PROOF_TTL, + )); None } Err(err) => { - replace_fleet_capability_proof(slot, None); + revoke_fleet_capability_proof(slot); Some(err) } } @@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0) } -#[cfg(test)] +pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let state = tier_delete_journal_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now()) +} + +fn acquire_tier_delete_journal_fleet_proof_from( + state: &FleetCapabilityProofState, + expected_topology: &str, + now: Instant, +) -> Option { + let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?; + let permit = state.proof.as_ref()?.generation.try_acquire()?; + Some(TierDeleteJournalFleetProofToken { token, _permit: permit }) +} + +pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool { + let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { + return false; + }; + let state = tier_delete_journal_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) +} + +fn tier_delete_journal_fleet_proof_matches_at( + state: &FleetCapabilityProofState, + proof: &TierDeleteJournalFleetProofToken, + expected_topology: &str, + now: Instant, +) -> bool { + proof._permit.generation.is_accepting() + && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now) + && state + .proof + .as_ref() + .is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation)) +} + +pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String { + stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint) +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool { + let state = tier_delete_journal_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained()) + || state + .draining_generation + .as_ref() + .is_some_and(|generation| !generation.is_drained()) +} + +fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-tier-delete-journal-topology-v1\0"); + hasher.update(topology_fingerprint.as_bytes()); + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +#[cfg(any(test, feature = "test-util"))] pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() { let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY .get() @@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() { let mut state = cross_pool_fence_fleet_proof_slot() .write() .unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) { + state.proof.clone() + } else { + Some(FleetCapabilityProof::new( + topology, + Arc::new(BTreeMap::new()), + now + Duration::from_secs(60 * 60), + )) + }; state.topology_conflict = false; - state.proof = Some(FleetCapabilityProof { - topology_fingerprint: topology, - peer_epochs: Arc::new(BTreeMap::new()), - expires_at: Instant::now() + Duration::from_secs(60 * 60), - }); + state.proof = proof.clone(); + drop(state); + let mut journal_state = tier_delete_journal_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + debug_assert!( + journal_state + .proof + .as_ref() + .is_none_or(|current| current.generation.is_drained()) + ); + journal_state.topology_conflict = false; + journal_state.draining_generation = None; + journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation); } #[cfg(test)] pub(crate) struct CrossPoolFenceFleetProofGuard { previous_proof: Option, previous_topology_conflict: bool, + previous_journal_proof: Option, + previous_journal_topology_conflict: bool, } #[cfg(test)] @@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard { let mut state = cross_pool_fence_fleet_proof_slot() .write() .unwrap_or_else(std::sync::PoisonError::into_inner); - state.proof = self.previous_proof.take(); + state.proof = self + .previous_proof + .take() + .as_ref() + .map(FleetCapabilityProof::with_fresh_generation); + state.draining_generation = None; state.topology_conflict = self.previous_topology_conflict; + drop(state); + let mut journal_state = tier_delete_journal_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + journal_state.proof = self + .previous_journal_proof + .take() + .as_ref() + .map(FleetCapabilityProof::with_fresh_generation); + journal_state.draining_generation = None; + journal_state.topology_conflict = self.previous_journal_topology_conflict; } } @@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF let mut state = cross_pool_fence_fleet_proof_slot() .write() .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut journal_state = tier_delete_journal_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); let guard = CrossPoolFenceFleetProofGuard { previous_proof: state.proof.clone(), previous_topology_conflict: state.topology_conflict, + previous_journal_proof: journal_state.proof.clone(), + previous_journal_topology_conflict: journal_state.topology_conflict, }; - state.proof = None; + if let Some(proof) = state.proof.take() { + proof.generation.revoke(); + if !proof.generation.is_drained() { + state.draining_generation = Some(proof.generation); + } + } state.topology_conflict = true; + if let Some(proof) = journal_state.proof.take() { + proof.generation.revoke(); + if !proof.generation.is_drained() { + journal_state.draining_generation = Some(proof.generation); + } + } + journal_state.topology_conflict = true; guard } @@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool { let Some(current) = state.proof.as_ref() else { return false; }; - state.proof = Some(FleetCapabilityProof { - topology_fingerprint: current.topology_fingerprint.clone(), - peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()), - expires_at: current.expires_at, - }); + let proof = FleetCapabilityProof::new( + current.topology_fingerprint.clone(), + Arc::new(current.peer_epochs.as_ref().clone()), + current.expires_at, + ); + state.proof = Some(proof.clone()); + drop(state); + let mut journal_state = tier_delete_journal_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + journal_state.topology_conflict = false; + if let Some(previous) = journal_state.proof.take() { + previous.generation.revoke(); + if !previous.generation.is_drained() { + journal_state.draining_generation = Some(previous.generation); + } + } + if journal_state + .draining_generation + .as_ref() + .is_some_and(|generation| generation.is_drained()) + { + journal_state.draining_generation = None; + } + if journal_state.draining_generation.is_none() { + journal_state.proof = Some(proof.with_fresh_generation()); + } true } @@ -291,15 +584,22 @@ fn fleet_capability_proof_matches( return false; }; let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner); - if state.topology_conflict { - return false; - } - state.proof.as_ref().is_some_and(|current| { - current.topology_fingerprint == *expected_topology - && current.topology_fingerprint == proof.topology_fingerprint - && Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs) - && Instant::now() < current.expires_at - }) + fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now()) +} + +fn fleet_capability_proof_matches_at( + state: &FleetCapabilityProofState, + proof: &FleetCapabilityProofToken, + expected_topology: &str, + now: Instant, +) -> bool { + !state.topology_conflict + && state.proof.as_ref().is_some_and(|current| { + current.topology_fingerprint == expected_topology + && current.topology_fingerprint == proof.topology_fingerprint + && Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs) + && now < current.expires_at + }) } fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool { @@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard; #[cfg(test)] impl Drop for RemoteVersionStateFleetProofGuard { fn drop(&mut self) { - replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); + revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot()); } } @@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, pe pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { - for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] { - let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); - state.topology_conflict = true; - state.proof = None; + for slot in [ + remote_version_state_fleet_proof_slot(), + cross_pool_fence_fleet_proof_slot(), + tier_delete_journal_fleet_proof_slot(), + ] { + mark_fleet_capability_topology_conflict(slot); } } return; @@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } None => Err(Error::other("remote version state fleet capability notification system is unavailable")), }; - let fence_result = match get_global_notification_sys() { + let fence_probe = match get_global_notification_sys() { Some(notification_sys) => timeout( REMOTE_VERSION_STATE_PROBE_TIMEOUT, notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), @@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), }; + let (fence_result, journal_result) = match fence_probe { + Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version), + Err(err) => { + let message = err.to_string(); + (Err(Error::other(message.clone())), Err(Error::other(message))) + } + }; let topology_conflict = remote_version_state_fleet_proof_slot() .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .topology_conflict; if topology_conflict { - replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); - replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None); + revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot()); + revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot()); + revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot()); } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, @@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { event = EVENT_NOTIFICATION_CAPABILITY_PROBE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_NOTIFICATION, - capability = "cross_pool_fence_v2", + capability = "cross_pool_fence", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + tier_delete_journal_fleet_proof_slot(), + &topology_fingerprint, + journal_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "tier_delete_journal_v6_policy", state = "failed_closed", error = %err, "notification capability probe" @@ -483,7 +811,7 @@ impl NotificationSys { Ok(peer_epochs) } - async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result> { + async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap, u32)> { if self.peer_clients.len() != self.peer_topology_hosts.len() { return Err(Error::other("cross-pool fence capability fleet membership is incomplete")); } @@ -494,14 +822,21 @@ impl NotificationSys { client.probe_cross_pool_fence(topology_fingerprint.to_string()).await }); let mut peer_epochs = BTreeMap::new(); + let mut minimum_version = u32::MAX; for result in join_all(probes).await { let (peer, version, epoch) = result?; if version < CROSS_POOL_FENCE_SUPPORTED_VERSION { return Err(Error::other("cross-pool fence capability version is unsupported")); } + minimum_version = minimum_version.min(version); insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?; } - Ok(peer_epochs) + // A single-node deployment has no remote member to lower the local + // policy version advertised by this binary. + if minimum_version == u32::MAX { + minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION; + } + Ok((peer_epochs, minimum_version)) } } @@ -1827,12 +2162,13 @@ impl NotificationSys { join_all(futures).await } - pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec { + pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter().cloned() { + let payload = canonical_prepare_payload.clone(); futures.push(async move { if let Some(client) = client { - notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await) + notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await) } else { unreachable_notification_peer_err() } @@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results( mod tests { use super::*; + #[test] + fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() { + let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]); + let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2); + assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing"); + assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion"); + + let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3); + assert!(generic_v3.is_ok()); + assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion"); + } + #[test] fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() { let now = Instant::now(); let mut peer_epochs = BTreeMap::new(); peer_epochs.insert("peer-a".to_string(), Uuid::new_v4()); - let proof = FleetCapabilityProof { - topology_fingerprint: "topology-a".to_string(), - peer_epochs: Arc::new(peer_epochs), - expires_at: now + Duration::from_secs(1), - }; + let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1)); assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now)); @@ -2495,11 +2839,7 @@ mod tests { #[test] fn remote_version_state_fleet_proof_accepts_single_node_membership() { let now = Instant::now(); - let proof = FleetCapabilityProof { - topology_fingerprint: "topology-a".to_string(), - peer_epochs: Arc::new(BTreeMap::new()), - expires_at: now + Duration::from_secs(1), - }; + let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1)); assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); } @@ -2507,21 +2847,97 @@ mod tests { #[test] fn remote_version_state_fleet_proof_token_changes_with_process_epoch() { let now = Instant::now(); - let proof = FleetCapabilityProof { - topology_fingerprint: "topology-a".to_string(), - peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), - expires_at: now + Duration::from_secs(1), - }; + let proof = FleetCapabilityProof::new( + "topology-a".to_string(), + Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), + now + Duration::from_secs(1), + ); let captured = proof.token(); - let restarted = FleetCapabilityProof { - topology_fingerprint: proof.topology_fingerprint.clone(), - peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), - expires_at: proof.expires_at, - }; + let restarted = FleetCapabilityProof::new( + proof.topology_fingerprint.clone(), + Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), + proof.expires_at, + ); assert!(captured != restarted.token()); } + #[test] + fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() { + let topology = "topology-a"; + let now = Instant::now(); + let node_a_view = FleetCapabilityProof::new( + topology.to_string(), + Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])), + now + Duration::from_secs(1), + ); + let node_b_view = FleetCapabilityProof::new( + topology.to_string(), + Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])), + now + Duration::from_secs(1), + ); + let restarted_node_a_view = FleetCapabilityProof::new( + topology.to_string(), + Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])), + now + Duration::from_secs(1), + ); + + let generations = [&node_a_view, &node_b_view, &restarted_node_a_view] + .map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint)); + assert_eq!(generations[0], generations[1]); + assert_eq!(generations[0], generations[2]); + assert_ne!( + generations[0], + stable_tier_delete_journal_topology_generation("topology-b"), + "a real topology change must produce a different durable generation" + ); + } + + #[test] + fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() { + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let now = Instant::now(); + let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none()); + let original = slot + .read() + .expect("proof slot should not poison") + .proof + .as_ref() + .expect("successful probe should publish proof") + .token(); + let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint); + + let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]); + assert!( + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1)) + .is_none() + ); + let state = slot.read().expect("proof slot should not poison"); + let fresh = state + .proof + .as_ref() + .expect("restart probe should publish a fresh proof") + .token(); + + assert!(!fleet_capability_proof_matches_at( + &state, + &original, + "topology-a", + now + Duration::from_millis(2) + )); + assert!(fleet_capability_proof_matches_at( + &state, + &fresh, + "topology-a", + now + Duration::from_millis(2) + )); + assert_eq!( + original_generation, + stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint) + ); + } + #[test] fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() { let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); @@ -2561,15 +2977,106 @@ mod tests { assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs)); } + #[test] + fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() { + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let now = Instant::now(); + let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none()); + + let admitted = { + let state = slot.read().expect("proof slot should not poison"); + acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now) + .expect("a fresh proof should admit one journal operation") + }; + { + let state = slot.read().expect("proof slot should not poison"); + assert!( + tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now), + "a freshly admitted journal proof must remain current" + ); + assert!( + acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,) + .is_none(), + "TTL expiry must stop new admission" + ); + assert!( + !tier_delete_journal_fleet_proof_matches_at( + &state, + &admitted, + "topology-a", + now + REMOTE_VERSION_STATE_PROOF_TTL, + ), + "TTL expiry must also stop an admitted proof at its next durable fence" + ); + assert!(!admitted._permit.generation.is_drained()); + } + + let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); + let blocked = publish_fleet_capability_probe_result( + &slot, + "topology-a", + Ok(restarted_peers.clone()), + now + Duration::from_millis(1), + ) + .expect("a successor proof must wait for the admitted generation"); + assert!(blocked.to_string().contains("previous generation to drain")); + { + let state = slot.read().expect("proof slot should not poison"); + assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains"); + assert!(state.draining_generation.is_some()); + assert!( + !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),), + "a restarted peer must revoke an admitted proof before its next durable fence" + ); + } + + drop(admitted); + assert!( + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),) + .is_none(), + "the successor may publish after the in-flight operation releases its permit" + ); + let state = slot.read().expect("proof slot should not poison"); + assert!(state.proof.is_some()); + assert!(state.draining_generation.is_none()); + } + + #[test] + fn tier_delete_journal_topology_conflict_revokes_admitted_generation() { + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let now = Instant::now(); + let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none()); + let admitted = { + let state = slot.read().expect("proof slot should not poison"); + acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now) + .expect("a fresh proof should admit one journal operation") + }; + + mark_fleet_capability_topology_conflict(&slot); + + let state = slot.read().expect("proof slot should not poison"); + assert!(state.topology_conflict); + assert!(state.proof.is_none()); + assert!(state.draining_generation.is_some()); + assert!(!admitted._permit.generation.is_accepting()); + assert!( + !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now), + "topology conflict must revoke an already admitted journal proof" + ); + } + #[test] fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() { let now = Instant::now(); let mut state = FleetCapabilityProofState { - proof: Some(FleetCapabilityProof { - topology_fingerprint: "topology-a".to_string(), - peer_epochs: Arc::new(BTreeMap::new()), - expires_at: now + Duration::from_secs(1), - }), + proof: Some(FleetCapabilityProof::new( + "topology-a".to_string(), + Arc::new(BTreeMap::new()), + now + Duration::from_secs(1), + )), + draining_generation: None, topology_conflict: false, }; assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some()); @@ -3279,7 +3786,7 @@ mod tests { assert_eq!(commit.len(), 1); assert!(commit[0].err.is_some()); - let abort = sys.abort_tier_mutation(mutation_id).await; + let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await; assert_eq!(abort.len(), 1); assert!(abort[0].err.is_some()); } diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index d6c8b917a..882abde01 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -356,7 +356,7 @@ impl ECStore { }; run_guard.ensure_held("rebalance version migration")?; let result = migrate_entry_version( - &RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()), + &RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()), bucket.clone(), pool_index, version, @@ -1478,6 +1478,7 @@ mod tests { let (_temp_dirs, store, _unused_store) = crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await; prepare_rebalance_test_volumes(store.as_ref()).await; + crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await; let source_set = store.pools[0].get_disks_by_key(object); let target_set = store.pools[1].get_disks_by_key(object); let version_id = uuid::Uuid::new_v4(); @@ -1520,14 +1521,17 @@ mod tests { let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await; let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID); let barrier = TieredMetadataCommitBarrier::install(bucket, object); - let task = spawn_real_rebalance_entry( + let mut task = spawn_real_rebalance_entry( Arc::clone(&store), Arc::clone(&source_set), entry, REBALANCE_ID, Arc::new(RebalanceBucketConfigs::default()), ); - barrier.wait_until_paused().await; + tokio::select! { + _ = barrier.wait_until_paused() => {} + result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"), + } run_signal_fence.mark_lost(); barrier.release(); drop(barrier); diff --git a/crates/ecstore/src/services/rebalance/migration.rs b/crates/ecstore/src/services/rebalance/migration.rs index cf2dabd8f..26d8ca0e8 100644 --- a/crates/ecstore/src/services/rebalance/migration.rs +++ b/crates/ecstore/src/services/rebalance/migration.rs @@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync { pub(crate) struct RebalanceMigrationBackend<'a> { source: &'a SetDisks, - store: &'a ECStore, + store: std::sync::Arc, lock_lost_signal: Option>, } impl<'a> RebalanceMigrationBackend<'a> { pub(crate) fn new( source: &'a SetDisks, - store: &'a ECStore, + store: std::sync::Arc, lock_lost_signal: Option>, ) -> Self { Self { diff --git a/crates/ecstore/src/services/rebalance/mod.rs b/crates/ecstore/src/services/rebalance/mod.rs index 96d02eedc..dab89bac7 100644 --- a/crates/ecstore/src/services/rebalance/mod.rs +++ b/crates/ecstore/src/services/rebalance/mod.rs @@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts( test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts( rebalance_meta: Option, ) -> ( diff --git a/crates/ecstore/src/services/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs index 801c1e3c9..67f27d76e 100644 --- a/crates/ecstore/src/services/tier/mod.rs +++ b/crates/ecstore/src/services/tier/mod.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] pub mod test_util; pub mod tier; pub mod tier_admin; diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index a8da9c7e7..687e1f3b1 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -47,7 +47,7 @@ use tokio::{ }; use tracing::{debug, error, info, warn}; -use crate::error::{Error, Result, StorageError}; +use crate::error::{Error, Result, StorageError, stable_io_error}; use crate::services::tier::{ tier_admin::TierCreds, tier_config::{TierConfig, TierType, TierWasabi}, @@ -69,7 +69,7 @@ use crate::{ tier_delete_journal::{TIER_DELETE_JOURNAL_PREFIX, decode_tier_delete_journal_entry}, transition_transaction::{TRANSITION_TRANSACTION_RECORD_PREFIX, decode_transition_transaction_record}, }, - cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState}, + cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState, tier_mutation_error_is_definitely_rejected}, config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, layout::endpoints::EndpointServerPools, @@ -88,10 +88,11 @@ use super::{ tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR}, tier_mutation_intent::{ TierMutationDigest, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState, TierMutationIntentTarget, - advance_tier_coordinator_mutation_intent_record_idempotent, advance_tier_mutation_intent_record_idempotent, - delete_tier_coordinator_mutation_intent_record, delete_tier_mutation_intent_record, - list_tier_coordinator_mutation_intent_records, list_tier_mutation_intent_records, - save_tier_coordinator_mutation_intent_record_if_absent, + acquire_tier_mutation_mutex, advance_tier_coordinator_mutation_intent_record_idempotent, + advance_tier_mutation_intent_record_idempotent, delete_tier_coordinator_mutation_intent_record_if_current, + delete_tier_mutation_intent_record_if_current, list_tier_coordinator_mutation_intent_records, + list_tier_mutation_intent_records, load_tier_coordinator_mutation_intent_record_with_etag, + load_tier_mutation_intent_record_with_etag, save_tier_coordinator_mutation_intent_record_if_absent, }, warm_backend::WarmBackendImpl, }; @@ -100,8 +101,10 @@ const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30); const TIER_REFERENCE_PROOF_LIST_LIMIT: i32 = 1000; +const TIER_REFERENCE_PROOF_PHYSICAL_WALK_CONCURRENCY: usize = 4; const TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT: usize = 1000; const TIER_MUTATION_INTENT_TTL: Duration = Duration::from_secs(15 * 60); +const TIER_MUTATION_TOMBSTONE_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60); const TIER_MUTATION_BLOCK_MESSAGE: &str = "Remote tier configuration is being replaced"; const TIER_MUTATION_REFRESH_RETRY_BASE: Duration = Duration::from_secs(1); const TIER_MUTATION_REFRESH_RETRY_CAP: Duration = Duration::from_secs(30); @@ -268,6 +271,12 @@ struct RecoveredTierMutationIntent { coordinator_state: Option, } +impl RecoveredTierMutationIntent { + fn is_peer_only_terminal(&self) -> bool { + self.has_peer_record && !self.has_coordinator_record && self.intent.state != TierMutationIntentState::Prepared + } +} + struct TierMutationRecoverySnapshot { coordinator_intents: Vec, intents: Vec, @@ -284,11 +293,7 @@ struct CommittedMutationRecoveryPlan { impl TierMutationRecoverySnapshot { fn requires_config_lock(&self) -> bool { - self.has_committed_mutation_blocks - || self - .intents - .iter() - .any(|recovered| recovered.has_coordinator_record || recovered.intent.state != TierMutationIntentState::Prepared) + self.has_committed_mutation_blocks || self.intents.iter().any(|recovered| recovered.has_coordinator_record) } fn allowance(&self) -> MutationBlockAllowance { @@ -296,13 +301,14 @@ impl TierMutationRecoverySnapshot { } fn committed_intents(&self) -> impl Iterator { - self.intents - .iter() - .filter(|recovered| recovered.intent.state == TierMutationIntentState::Committed) + self.intents.iter().filter(|recovered| { + recovered.intent.state == TierMutationIntentState::Committed + && (!recovered.is_peer_only_terminal() || self.allowance.mutation_ids.contains(&recovered.intent.mutation_id)) + }) } fn is_quiescent(&self) -> bool { - self.intents.is_empty() && self.allowance.mutation_ids.is_empty() + self.allowance.mutation_ids.is_empty() && self.intents.iter().all(RecoveredTierMutationIntent::is_peer_only_terminal) } } @@ -720,11 +726,55 @@ trait TierReferenceProofStore: > + Send + Sync { + async fn authoritative_buckets_for_reference_proof(&self) -> Result> { + self.list_bucket(&crate::storage_api_contracts::bucket::BucketOptions { + deleted: true, + no_metadata: true, + ..Default::default() + }) + .await + } + async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result>; + + async fn authoritative_bucket_tier_reference( + self: Arc, + bucket: &str, + targets: &[TierMutationIntentTarget], + ) -> std::result::Result, AdminError> + where + Self: Sized, + { + find_authoritative_bucket_tier_reference_by_walk(self, bucket, targets).await + } } #[async_trait::async_trait] impl TierReferenceProofStore for ECStore { + async fn authoritative_buckets_for_reference_proof(&self) -> Result> { + let opts = crate::storage_api_contracts::bucket::BucketOptions { + deleted: true, + no_metadata: true, + ..Default::default() + }; + let (logical, physical) = tokio::join!( + ::list_bucket(self, &opts), + self.list_bucket_for_scanner(&opts), + ); + let logical = logical?; + let physical = physical?; + if !physical.topology_complete { + return Err(Error::other( + "tier reference proof cannot enumerate every physical bucket with complete set quorum", + )); + } + let mut buckets = BTreeMap::new(); + for bucket in logical.into_iter().chain(physical.buckets) { + buckets.entry(bucket.name.clone()).or_insert(bucket); + } + Ok(buckets.into_values().collect()) + } + async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result> { match get_lifecycle_config(bucket).await { Ok((config, _updated_at)) => Ok(Some(config)), @@ -732,6 +782,14 @@ impl TierReferenceProofStore for ECStore { Err(err) => Err(err), } } + + async fn authoritative_bucket_tier_reference( + self: Arc, + bucket: &str, + targets: &[TierMutationIntentTarget], + ) -> std::result::Result, AdminError> { + find_authoritative_bucket_tier_reference_on_physical_sets(self, bucket, targets).await + } } async fn ensure_no_authoritative_tier_object_references( @@ -762,7 +820,7 @@ where S: TierReferenceProofStore, { let buckets = api - .list_bucket(&Default::default()) + .authoritative_buckets_for_reference_proof() .await .map_err(tier_reference_proof_admin_error)?; for bucket in buckets { @@ -773,41 +831,171 @@ where if !force { ensure_no_authoritative_lifecycle_references(api.as_ref(), &bucket.name, targets).await?; } - let mut marker = None; - let mut version_marker = None; - loop { - let page = api - .clone() - .list_object_versions( - &bucket.name, - "", - marker.take(), - version_marker.take(), - None, - TIER_REFERENCE_PROOF_LIST_LIMIT, - ) - .await - .map_err(tier_reference_proof_admin_error)?; - for object in &page.objects { - if tier_object_blocks_any_target_rebind(object, targets).map_err(tier_reference_proof_admin_error)? { - return Err(tier_reference_proof_in_use_error(&object.transitioned_object.tier, object)); - } - } - if !page.is_truncated { - break; - } - marker = page.next_marker; - version_marker = page.next_version_idmarker; - if marker.is_none() { - return Err(tier_reference_proof_admin_error(io::Error::other( - "tier reference proof listing is truncated without a next marker", - ))); - } + if let Some(object) = api.clone().authoritative_bucket_tier_reference(&bucket.name, targets).await? { + return Err(tier_reference_proof_in_use_error(&object.transitioned_object.tier, &object)); } } ensure_no_authoritative_persisted_references(api, targets).await } +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn ensure_no_authoritative_tier_references_for_test( + api: Arc, + tier_name: &str, + old_backend_identity: TierDestinationId, + force: bool, +) -> std::result::Result<(), AdminError> { + ensure_no_authoritative_tier_object_references( + api, + &[TierMutationIntentTarget { + tier_name: tier_name.to_string(), + old_backend_identity: Some(old_backend_identity), + new_backend_identity: None, + }], + force, + ) + .await +} + +async fn find_authoritative_bucket_tier_reference_by_walk( + api: Arc, + bucket: &str, + targets: &[TierMutationIntentTarget], +) -> std::result::Result, AdminError> +where + S: TierReferenceProofStore, +{ + let (tx, mut rx) = tokio::sync::mpsc::channel::>(100); + let cancellation = tokio_util::sync::CancellationToken::new(); + let walk = api.clone().walk( + cancellation.clone(), + bucket, + "", + tx, + TierReferenceProofWalkOptions { + include_free_versions: true, + // Namespace size determines total proof time. Drive-level progress + // stalls remain bounded by the walk implementation. + walkdir_timeout: Some(Duration::ZERO), + ..Default::default() + }, + ); + let collect = async { + while let Some(result) = rx.recv().await { + if let Some(err) = result.err { + cancellation.cancel(); + return Err(err); + } + let Some(object) = result.item else { + continue; + }; + match tier_object_blocks_any_target_rebind(&object, targets) { + Ok(true) => { + cancellation.cancel(); + return Ok(Some(object)); + } + Ok(false) => {} + Err(err) => { + cancellation.cancel(); + return Err(Error::other(err)); + } + } + } + Ok(None) + }; + let (walk_result, collect_result) = tokio::join!(walk, collect); + match collect_result.map_err(tier_reference_proof_admin_error)? { + Some(blocker) => Ok(Some(blocker)), + None => { + walk_result.map_err(tier_reference_proof_admin_error)?; + Ok(None) + } + } +} + +async fn find_authoritative_bucket_tier_reference_on_physical_sets( + api: Arc, + bucket: &str, + targets: &[TierMutationIntentTarget], +) -> std::result::Result, AdminError> { + use futures::StreamExt as _; + + let physical_sets = api + .pools + .iter() + .flat_map(|pool| pool.disk_set.iter().cloned()) + .collect::>(); + let (tx, mut rx) = tokio::sync::mpsc::channel::>(100); + let cancellation = tokio_util::sync::CancellationToken::new(); + let walk_cancel = cancellation.clone(); + let bucket_owned = bucket.to_string(); + let walk = async move { + let results = futures::stream::iter(physical_sets.into_iter().map(|set| { + let tx = tx.clone(); + let cancellation = walk_cancel.clone(); + let bucket = bucket_owned.clone(); + async move { + let result = set + .walk( + cancellation.clone(), + &bucket, + "", + tx, + TierReferenceProofWalkOptions { + include_free_versions: true, + // Total time scales with namespace size; every + // underlying drive walk still carries its bounded + // progress-stall timeout. + walkdir_timeout: Some(Duration::ZERO), + ..Default::default() + }, + ) + .await; + if result.is_err() { + cancellation.cancel(); + } + result + } + })) + .buffer_unordered(TIER_REFERENCE_PROOF_PHYSICAL_WALK_CONCURRENCY) + .collect::>() + .await; + drop(tx); + results.into_iter().collect::>>().map(|_| ()) + }; + let collect = async { + while let Some(result) = rx.recv().await { + if let Some(err) = result.err { + cancellation.cancel(); + return Err(err); + } + let Some(object) = result.item else { + continue; + }; + match tier_object_blocks_any_target_rebind(&object, targets) { + Ok(true) => { + cancellation.cancel(); + return Ok(Some(object)); + } + Ok(false) => {} + Err(err) => { + cancellation.cancel(); + return Err(Error::other(err)); + } + } + } + Ok(None) + }; + let (walk_result, collect_result) = tokio::join!(walk, collect); + match collect_result.map_err(tier_reference_proof_admin_error)? { + Some(blocker) => Ok(Some(blocker)), + None => { + walk_result.map_err(tier_reference_proof_admin_error)?; + Ok(None) + } + } +} + async fn ensure_no_authoritative_lifecycle_references( api: &impl TierReferenceProofStore, bucket: &str, @@ -1038,7 +1226,11 @@ trait TierMutationPeer: Send + Sync { fn peer_label(&self) -> String; async fn prepare_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result; async fn commit_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result; - async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result; + async fn abort_tier_mutation( + &self, + mutation_id: uuid::Uuid, + canonical_prepare_payload: Bytes, + ) -> Result; } #[async_trait::async_trait] @@ -1059,8 +1251,14 @@ impl TierMutationPeer for PeerRestClient { .state) } - async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result { - Ok(PeerRestClient::abort_tier_mutation(self, mutation_id).await?.state) + async fn abort_tier_mutation( + &self, + mutation_id: uuid::Uuid, + canonical_prepare_payload: Bytes, + ) -> Result { + Ok(PeerRestClient::abort_tier_mutation(self, mutation_id, canonical_prepare_payload) + .await? + .state) } } @@ -1121,6 +1319,18 @@ async fn prepare_tier_mutation_peers( }); } Err(err) => { + if tier_mutation_prepare_was_definitely_rejected(&err) { + return Err(TierMutationPrepareFailure { + error: tier_mutation_fanout_admin_error("prepare", format!("peer {label}: {err}")), + prepared_peers: prepared, + }); + } + // The request may have reached the peer and installed its + // durable Prepared block before the response failed (for + // example while draining an in-flight lease). Include it in + // abort fanout so ambiguous prepare outcomes fail closed + // without stranding a runtime block. + prepared.push(peer); return Err(TierMutationPrepareFailure { error: tier_mutation_fanout_admin_error("prepare", format!("peer {label}: {err}")), prepared_peers: prepared, @@ -1131,16 +1341,22 @@ async fn prepare_tier_mutation_peers( Ok(prepared) } +fn tier_mutation_prepare_was_definitely_rejected(err: &Error) -> bool { + tier_mutation_error_is_definitely_rejected(err) +} + struct TierMutationPrepareFailure { error: AdminError, prepared_peers: Vec>, } -async fn abort_tier_mutation_peers(mutation_id: uuid::Uuid, peers: Vec>) -> io::Result<()> { +async fn abort_tier_mutation_peers(intent: &TierMutationIntent, peers: Vec>) -> io::Result<()> { + let prepared = intent.original_prepared().map_err(io::Error::other)?; + let payload = Bytes::from(prepared.encode().map_err(io::Error::other)?); let mut failures = Vec::new(); for peer in peers { let label = peer.peer_label(); - let result = peer.abort_tier_mutation(mutation_id).await; + let result = peer.abort_tier_mutation(intent.mutation_id, payload.clone()).await; match result { Ok(PeerTierMutationState::Aborted) => {} Ok(state) => { @@ -1218,6 +1434,7 @@ where S: TierReferenceProofStore, { if let Some(intent) = intent { + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; save_tier_coordinator_mutation_intent_record_if_absent(api, intent) .await .map_err(io::Error::other)?; @@ -1234,6 +1451,7 @@ where S: TierReferenceProofStore, { if let Some(intent) = intent { + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; advance_tier_coordinator_mutation_intent_record_idempotent( api, intent.mutation_id, @@ -1253,6 +1471,7 @@ where let Some(intent) = intent else { return true; }; + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; match advance_tier_coordinator_mutation_intent_record_idempotent( api, intent.mutation_id, @@ -1289,7 +1508,7 @@ where let Some(intent) = intent else { return true; }; - if let Err(err) = abort_tier_mutation_peers(intent.mutation_id, prepared_peers).await { + if let Err(err) = abort_tier_mutation_peers(intent, prepared_peers).await { warn!( event = "tier_mutation_abort", component = LOG_COMPONENT_ECSTORE, @@ -1299,12 +1518,25 @@ where error = ?err, "tier mutation abort incomplete" ); + // The coordinator record remains Prepared so recovery can retry the + // genuinely ambiguous peer cleanup. The local data path must not stay + // fenced merely because a remote Abort response was lost. + if let Err(clear_err) = TierConfigMgr::clear_prepared_mutation_intent_block(handle, intent.mutation_id).await { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "runtime_fence_clear_failed", + mutation_id = %intent.mutation_id, + error = ?clear_err, + "tier mutation abort incomplete" + ); + } return false; } - if !abort_coordinator_tier_mutation_intent(api, Some(intent)).await { - return false; - } - if let Err(err) = TierConfigMgr::clear_prepared_mutation_intent_block(handle, intent.mutation_id).await { + let coordinator_aborted = abort_coordinator_tier_mutation_intent(api, Some(intent)).await; + let runtime_cleared = if let Err(err) = TierConfigMgr::clear_prepared_mutation_intent_block(handle, intent.mutation_id).await + { warn!( event = "tier_mutation_abort", component = LOG_COMPONENT_ECSTORE, @@ -1314,9 +1546,11 @@ where error = ?err, "tier mutation abort incomplete" ); - return false; - } - true + false + } else { + true + }; + coordinator_aborted && runtime_cleared } fn committed_tier_mutation_intent( @@ -2981,6 +3215,33 @@ impl TierConfigMgr { .is_some_and(|runtime| !lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty()) } + async fn has_retryable_terminal_mutation_cleanup(api: Arc) -> io::Result + where + S: EcstoreObjectIO + ListOperations>, + { + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX); + let peer_cleanup = Self::load_tier_mutation_intents(api.clone()) + .await? + .into_iter() + .any(|intent| { + intent.state != TierMutationIntentState::Prepared && Self::tier_mutation_tombstone_retention_elapsed(&intent, now) + }); + if peer_cleanup { + return Ok(true); + } + Ok(Self::load_coordinator_mutation_intents(api) + .await? + .into_iter() + .any(|intent| intent.state != TierMutationIntentState::Prepared)) + } + + fn terminal_mutation_cleanup_requires_retry(result: io::Result) -> bool { + // A failed durable-state read is Unknown, not proof that no terminal + // cleanup remains. Keep the bounded-backoff retry loop alive until a + // later authoritative read can classify the state. + result.unwrap_or(true) + } + async fn acquire_tier_config_write_lock( api: Arc, ) -> std::result::Result @@ -3376,12 +3637,13 @@ impl TierConfigMgr { let affected_targets = build_tier_mutation_affected_targets(mutation_kind, proof_targets, ¤t_for_targets, &candidate) .map_err(TierConfigUpdateError::Publish)?; - ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets, mutation_force) - .await - .map_err(TierConfigUpdateError::Publish)?; - let coordinator_intent = - build_coordinator_tier_mutation_intent(mutation_kind, version.clone(), &candidate, affected_targets) - .map_err(TierConfigUpdateError::Save)?; + let coordinator_intent = build_coordinator_tier_mutation_intent( + mutation_kind, + version.clone(), + &candidate, + affected_targets.clone(), + ) + .map_err(TierConfigUpdateError::Save)?; save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref()) .await .map_err(TierConfigUpdateError::Save)?; @@ -3394,7 +3656,18 @@ impl TierConfigMgr { let peers = match remote_tier_mutation_peers().await { Ok(peers) => peers, Err(err) => { - TierConfigMgr::request_committed_mutation_refresh(&handle).await; + if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()) + .await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + mutation_id = %intent.mutation_id, + "tier mutation peer discovery failed and local abort was incomplete" + ); + } return Err(TierConfigUpdateError::Publish(tier_mutation_fanout_admin_error("prepare", err))); } }; @@ -3428,6 +3701,22 @@ impl TierConfigMgr { } else { Vec::new() }; + if let Err(proof_error) = + ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets, mutation_force).await + { + if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation reference proof failed and abort was incomplete" + ); + } + return Err(TierConfigUpdateError::Publish(proof_error)); + } let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?; let saved = match candidate .save_tiering_config_if_current_with_info(api.clone(), version.as_deref()) @@ -3628,7 +3917,8 @@ impl TierConfigMgr { .await?; Self::delete_finished_peer_mutation_intents(api.clone(), &snapshot, &recovery_plan.peer_cleanup_order).await?; if !snapshot.is_quiescent() { - Self::load_and_reconcile_mutation_recovery_snapshot(handle, api, false).await?; + let settled_snapshot = Self::load_and_reconcile_mutation_recovery_snapshot(handle, api.clone(), false).await?; + Self::delete_finished_peer_mutation_intents(api, &settled_snapshot, &[]).await?; } drop(config_lock); Ok(()) @@ -3755,6 +4045,87 @@ impl TierConfigMgr { ))) } + fn tier_mutation_tombstone_retention_elapsed(intent: &TierMutationIntent, now_unix_nanos: i64) -> bool { + let clock_skew_nanos = i64::try_from(TIER_MUTATION_TOMBSTONE_CLOCK_SKEW.as_nanos()).unwrap_or(i64::MAX); + intent.expires_at_unix_nanos.saturating_add(clock_skew_nanos) <= now_unix_nanos + } + + async fn delete_exact_peer_terminal_mutation_intent(api: Arc, expected: &TierMutationIntent) -> io::Result + where + S: EcstoreObjectIO + EcstoreObjectOperations, + { + if expected.state == TierMutationIntentState::Prepared { + return Err(io::Error::other("refusing to delete a non-terminal peer tier mutation intent")); + } + let (current, etag) = match load_tier_mutation_intent_record_with_etag(api.clone(), expected.mutation_id).await { + Ok(current) => current, + Err(Error::ConfigNotFound) => return Ok(true), + Err(err) => return Err(tier_mutation_replay_error(err)), + }; + if current != *expected { + return Err(stable_io_error( + "peer tier mutation intent changed before terminal cleanup", + expected.mutation_id.to_string(), + )); + } + match delete_tier_mutation_intent_record_if_current(api, expected.mutation_id, &etag).await { + Ok(()) | Err(Error::ConfigNotFound) => Ok(true), + Err(Error::PreconditionFailed) => Ok(false), + Err(err) => Err(tier_mutation_replay_error(err)), + } + } + + async fn delete_exact_coordinator_terminal_mutation_intent(api: Arc, expected: &TierMutationIntent) -> io::Result + where + S: EcstoreObjectIO + EcstoreObjectOperations, + { + if expected.state == TierMutationIntentState::Prepared { + return Err(io::Error::other("refusing to delete a non-terminal coordinator tier mutation intent")); + } + let (current, etag) = + match load_tier_coordinator_mutation_intent_record_with_etag(api.clone(), expected.mutation_id).await { + Ok(current) => current, + Err(Error::ConfigNotFound) => return Ok(true), + Err(err) => return Err(tier_mutation_replay_error(err)), + }; + if current != *expected { + return Err(stable_io_error( + "coordinator tier mutation intent changed before terminal cleanup", + expected.mutation_id.to_string(), + )); + } + match delete_tier_coordinator_mutation_intent_record_if_current(api, expected.mutation_id, &etag).await { + Ok(()) | Err(Error::ConfigNotFound) => Ok(true), + Err(Error::PreconditionFailed) => Ok(false), + Err(err) => Err(tier_mutation_replay_error(err)), + } + } + + async fn gc_peer_terminal_mutation_intent(api: Arc, mutation_id: uuid::Uuid, now_unix_nanos: i64) -> io::Result<()> + where + S: EcstoreObjectIO + EcstoreObjectOperations, + { + let (current, etag) = match load_tier_mutation_intent_record_with_etag(api.clone(), mutation_id).await { + Ok(current) => current, + Err(Error::ConfigNotFound) => return Ok(()), + Err(err) => return Err(tier_mutation_replay_error(err)), + }; + if current.state == TierMutationIntentState::Prepared + || !Self::tier_mutation_tombstone_retention_elapsed(¤t, now_unix_nanos) + { + return Ok(()); + } + match load_tier_coordinator_mutation_intent_record_with_etag(api.clone(), mutation_id).await { + Ok(_) => return Ok(()), + Err(Error::ConfigNotFound) => {} + Err(err) => return Err(tier_mutation_replay_error(err)), + } + match delete_tier_mutation_intent_record_if_current(api, mutation_id, &etag).await { + Ok(()) | Err(Error::ConfigNotFound | Error::PreconditionFailed) => Ok(()), + Err(err) => Err(tier_mutation_replay_error(err)), + } + } + async fn reconcile_terminal_dual_prefix_mutation_intents( api: Arc, snapshot: &mut TierMutationRecoverySnapshot, @@ -3779,22 +4150,28 @@ impl TierConfigMgr { .iter() .find(|recovered| recovered.intent.mutation_id == intent.mutation_id) .ok_or_else(|| io::Error::other("terminal tier mutation recovery lost its source record"))?; - if recovered.peer_state == Some(TierMutationIntentState::Aborted) - && recovered.coordinator_state == Some(TierMutationIntentState::Committed) + let stale_aborted_peer = recovered.peer_state == Some(TierMutationIntentState::Aborted) + && recovered.coordinator_state == Some(TierMutationIntentState::Committed); { - delete_tier_mutation_intent_record(api.clone(), intent.mutation_id) - .await - .map_err(tier_mutation_replay_error)?; - continue; + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; + if stale_aborted_peer { + let mut peer = intent.original_prepared().map_err(io::Error::other)?; + peer.advance(TierMutationIntentState::Aborted, None) + .map_err(io::Error::other)?; + if !Self::delete_exact_peer_terminal_mutation_intent(api.clone(), &peer).await? { + return Err(io::Error::other("stale aborted peer intent changed during conditional cleanup")); + } + continue; + } + advance_tier_mutation_intent_record_idempotent( + api.clone(), + intent.mutation_id, + intent.state, + intent.committed_config_etag.clone(), + ) + .await + .map_err(io::Error::other)?; } - advance_tier_mutation_intent_record_idempotent( - api.clone(), - intent.mutation_id, - intent.state, - intent.committed_config_etag.clone(), - ) - .await - .map_err(io::Error::other)?; if intent.state == TierMutationIntentState::Aborted { let resolved = match &peers { @@ -3805,13 +4182,14 @@ impl TierConfigMgr { resolved } }; - abort_tier_mutation_peers(intent.mutation_id, resolved).await?; + abort_tier_mutation_peers(&intent, resolved).await?; if let Some(coordinator) = snapshot .coordinator_intents .iter_mut() .find(|coordinator| coordinator.mutation_id == intent.mutation_id) && coordinator.state == TierMutationIntentState::Prepared { + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent( api.clone(), intent.mutation_id, @@ -3920,6 +4298,7 @@ impl TierConfigMgr { .etag .as_ref() .ok_or_else(|| io::Error::other("matching coordinator intent has no tier config ETag"))?; + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent( api.clone(), intent.mutation_id, @@ -3932,6 +4311,17 @@ impl TierConfigMgr { continue; } + let abort_proof_matches = match intent.old_config_etag.as_deref() { + Some(old_config_etag) => loaded.etag.as_deref() == Some(old_config_etag), + None => intent.kind == TierMutationIntentKind::Add, + }; + if !abort_proof_matches { + // A third configuration is neither the candidate nor the old + // configuration. Its history is uncertain, so retain the + // Prepared record and runtime block for a later proof. + continue; + } + let peers = match &peers { Some(peers) => peers.clone(), None => { @@ -3943,11 +4333,12 @@ impl TierConfigMgr { let recovery = if intent.expires_at_unix_nanos <= now { "expired" } else { - "unmatched" + "proven-old" }; - abort_tier_mutation_peers(intent.mutation_id, peers) + abort_tier_mutation_peers(intent, peers) .await .map_err(|err| io::Error::other(format!("{recovery} coordinator tier mutation recovery abort failed: {err}")))?; + let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent( api.clone(), intent.mutation_id, @@ -4059,32 +4450,27 @@ impl TierConfigMgr { async fn delete_finished_peer_mutation_intents( api: Arc, snapshot: &TierMutationRecoverySnapshot, - peer_cleanup_order: &[uuid::Uuid], + _peer_cleanup_order: &[uuid::Uuid], ) -> io::Result<()> where - S: EcstoreObjectOperations, + S: EcstoreObjectIO + EcstoreObjectOperations, { - let mut mutation_ids = peer_cleanup_order.to_vec(); - let scheduled = mutation_ids.iter().copied().collect::>(); - let mut remaining = snapshot + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX); + let mut mutation_ids = snapshot .intents .iter() .filter(|recovered| { recovered.has_peer_record - && matches!( - recovered.intent.state, - TierMutationIntentState::Committed | TierMutationIntentState::Aborted - ) - && !scheduled.contains(&recovered.intent.mutation_id) + && recovered.intent.state != TierMutationIntentState::Prepared + && !snapshot.allowance.mutation_ids.contains(&recovered.intent.mutation_id) }) .map(|recovered| recovered.intent.mutation_id) .collect::>(); - remaining.sort_unstable(); - mutation_ids.extend(remaining); + mutation_ids.sort_unstable(); + mutation_ids.dedup(); for mutation_id in mutation_ids { - delete_tier_mutation_intent_record(api.clone(), mutation_id) - .await - .map_err(tier_mutation_replay_error)?; + let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await; + Self::gc_peer_terminal_mutation_intent(api.clone(), mutation_id, now).await?; } Ok(()) } @@ -4095,24 +4481,42 @@ impl TierConfigMgr { committed_cleanup_order: &[uuid::Uuid], ) -> io::Result<()> where - S: EcstoreObjectOperations, + S: EcstoreObjectIO + EcstoreObjectOperations, { for mutation_id in committed_cleanup_order { - delete_tier_coordinator_mutation_intent_record(api.clone(), *mutation_id) - .await - .map_err(tier_mutation_replay_error)?; + let _mutation_guard = acquire_tier_mutation_mutex(*mutation_id).await; + let expected = snapshot + .coordinator_intents + .iter() + .find(|intent| intent.mutation_id == *mutation_id) + .ok_or_else(|| io::Error::other("coordinator cleanup plan references a missing intent"))?; + if !Self::delete_exact_coordinator_terminal_mutation_intent(api.clone(), expected).await? { + return Err(stable_io_error( + "coordinator tier mutation intent changed during conditional cleanup", + mutation_id.to_string(), + )); + } } let mut aborted = snapshot - .intents + .coordinator_intents .iter() - .filter(|recovered| recovered.has_coordinator_record && recovered.intent.state == TierMutationIntentState::Aborted) - .map(|recovered| recovered.intent.mutation_id) + .filter(|intent| intent.state == TierMutationIntentState::Aborted) + .map(|intent| intent.mutation_id) .collect::>(); aborted.sort_unstable(); for mutation_id in aborted { - delete_tier_coordinator_mutation_intent_record(api.clone(), mutation_id) - .await - .map_err(tier_mutation_replay_error)?; + let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await; + let expected = snapshot + .coordinator_intents + .iter() + .find(|intent| intent.mutation_id == mutation_id) + .ok_or_else(|| io::Error::other("aborted coordinator cleanup lost its source intent"))?; + if !Self::delete_exact_coordinator_terminal_mutation_intent(api.clone(), expected).await? { + return Err(stable_io_error( + "aborted coordinator tier mutation intent changed during conditional cleanup", + mutation_id.to_string(), + )); + } } Ok(()) } @@ -4192,13 +4596,36 @@ impl TierConfigMgr { expected_revision: u64, retain_missing_mutation_blocks: bool, ) -> std::result::Result, AdminError> { + let manager = handle.read().await; + let published_digest = if intents + .iter() + .any(|recovered| recovered.is_peer_only_terminal() && recovered.intent.state == TierMutationIntentState::Committed) + { + Some(tier_config_candidate_digest(&manager).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = format!("Failed to classify retained tier mutation tombstones: {err}"); + admin_err + })?) + } else { + None + }; let mut prepared_mutation_blocks = HashMap::new(); let mut committed_mutation_blocks: HashMap> = HashMap::new(); for recovered in intents { + let settled_tombstone = recovered.is_peer_only_terminal() + && match recovered.intent.state { + TierMutationIntentState::Aborted => true, + TierMutationIntentState::Committed => { + !retain_missing_mutation_blocks || published_digest == Some(recovered.intent.candidate_digest) + } + TierMutationIntentState::Prepared => false, + }; + if settled_tombstone { + continue; + } Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, &recovered.intent)?; Self::collect_committed_mutation_intent_blocks(&mut committed_mutation_blocks, &recovered.intent); } - let manager = handle.read().await; let runtime = tier_driver_runtime(handle, &manager); let mut runtime = lock_unpoisoned(&runtime); if runtime.mutation_blocks_revision != expected_revision { @@ -4278,6 +4705,35 @@ impl TierConfigMgr { Ok(()) } + pub(crate) async fn wait_for_blocked_tier_operation_leases( + handle: &Arc>, + intent: &TierMutationIntent, + ) -> std::result::Result<(), AdminError> { + let generations = { + let manager = handle.read().await; + let Some(runtime) = registered_tier_driver_runtime(&manager) else { + return Ok(()); + }; + let runtime = lock_unpoisoned(&runtime); + intent + .affected_targets + .iter() + .filter_map(|target| runtime.generations.get(&target.tier_name).cloned()) + .collect::>() + }; + let drain = async { + for generation in generations { + generation.wait_for_no_active_leases().await; + } + }; + if tokio::time::timeout(TIER_OPERATION_DRAIN_TIMEOUT, drain).await.is_err() { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out waiting for in-flight remote tier operations before reference proof".to_string(); + return Err(err); + } + Ok(()) + } + pub(crate) async fn clear_prepared_mutation_intent_block( handle: &Arc>, mutation_id: uuid::Uuid, @@ -4622,13 +5078,25 @@ impl TierConfigMgr { Ok(()) } - #[cfg(feature = "test-util")] + #[cfg(any(test, feature = "test-util"))] pub(crate) fn install_test_driver( &mut self, tier_name: &str, driver: WarmBackendImpl, ) -> std::result::Result<(), AdminError> { - self.replace_driver(tier_name, driver) + self.replace_driver(tier_name, driver)?; + if let Some(runtime) = registered_tier_driver_runtime(self) + && let Some(generation) = lock_unpoisoned(&runtime).generations.get(tier_name).cloned() + { + // Test drivers own their candidate inventory; do not reconstruct a + // real network reconciler from the placeholder test configuration. + generation.reconciler.set(None).map_err(|_| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Test tier candidate reconciler was already initialized".to_string(); + err + })?; + } + Ok(()) } fn revoke_driver(&mut self, tier_name: &str) -> Option> { @@ -5043,13 +5511,17 @@ impl TierConfigMgr { match Self::reload_handle_with(handle, api.clone()).await { Ok(()) => return, Err(err) => { - if !Self::has_committed_mutation_block(handle).await { + let has_runtime_fence = Self::has_committed_mutation_block(handle).await; + let has_cleanup = Self::terminal_mutation_cleanup_requires_retry( + Self::has_retryable_terminal_mutation_cleanup(api.clone()).await, + ); + if !has_runtime_fence && !has_cleanup { warn!( event = EVENT_TIER_CONFIG_REFRESH, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_TIER, trigger = "committed_mutation", - result = "failed_without_fence", + result = "failed_without_retryable_state", error = ?err, "tier configuration refresh" ); @@ -5084,7 +5556,11 @@ impl TierConfigMgr { ); } tokio::time::sleep(delay).await; - if !Self::has_committed_mutation_block(handle).await { + let has_runtime_fence = Self::has_committed_mutation_block(handle).await; + let has_cleanup = Self::terminal_mutation_cleanup_requires_retry( + Self::has_retryable_terminal_mutation_cleanup(api.clone()).await, + ); + if !has_runtime_fence && !has_cleanup { return; } retry_attempt = retry_attempt.saturating_add(1); @@ -7188,6 +7664,7 @@ mod tests { label: &'static str, calls: Arc>>, prepare: std::result::Result, + prepare_definitely_rejected: bool, commit: std::result::Result, abort: std::result::Result, } @@ -7211,6 +7688,7 @@ mod tests { label, calls, prepare, + prepare_definitely_rejected: false, commit, abort: Ok(PeerTierMutationState::Aborted), }) @@ -7233,7 +7711,13 @@ mod tests { canonical_payload: Bytes, ) -> Result { self.record(format!("{}:prepare:{}:{}", self.label, mutation_id, canonical_payload.len())); - self.prepare.map_err(Error::other) + match self.prepare { + Ok(state) => Ok(state), + Err(message) if self.prepare_definitely_rejected => Err( + crate::cluster::rpc::peer_rest_client::test_tier_mutation_definitely_rejected_error(message), + ), + Err(message) => Err(Error::other(message)), + } } async fn commit_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result { @@ -7246,7 +7730,15 @@ mod tests { self.commit.map_err(Error::other) } - async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result { + async fn abort_tier_mutation( + &self, + mutation_id: uuid::Uuid, + canonical_prepare_payload: Bytes, + ) -> Result { + let prepared = TierMutationIntent::decode(mutation_id, &canonical_prepare_payload).map_err(Error::other)?; + if prepared.state != TierMutationIntentState::Prepared || prepared.revision != 1 { + return Err(Error::other("test peer requires the original revision-1 Prepared abort payload")); + } self.record(format!("{}:abort:{}", self.label, mutation_id)); self.abort.map_err(Error::other) } @@ -7292,7 +7784,11 @@ mod tests { Ok(PeerTierMutationState::Committed) } - async fn abort_tier_mutation(&self, _mutation_id: uuid::Uuid) -> Result { + async fn abort_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_prepare_payload: Bytes, + ) -> Result { Ok(PeerTierMutationState::Aborted) } } @@ -7345,7 +7841,11 @@ mod tests { Ok(PeerTierMutationState::Committed) } - async fn abort_tier_mutation(&self, _mutation_id: uuid::Uuid) -> Result { + async fn abort_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_prepare_payload: Bytes, + ) -> Result { self.track("abort").await; Ok(PeerTierMutationState::Aborted) } @@ -7448,12 +7948,13 @@ mod tests { #[tokio::test] async fn abort_tier_mutation_peers_serializes_shared_record_writes() { let mutation_id = uuid::Uuid::from_u128(36); + let intent = prepared_remove_intent("COLD-A", mutation_id); let calls = Arc::new(Mutex::new(Vec::new())); let active = Arc::new(AtomicUsize::new(0)); let max_active = Arc::new(AtomicUsize::new(0)); abort_tier_mutation_peers( - mutation_id, + &intent, vec![ ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()), @@ -7785,7 +8286,7 @@ mod tests { FakeTierMutationPeer::boxed_with_prepare_commit( "peer-b", calls.clone(), - Err("prepare unavailable"), + Err("unsupported tier mutation peer protocol version: 4"), Ok(PeerTierMutationState::Committed), ), ], @@ -7813,6 +8314,10 @@ mod tests { assert!(calls.iter().any(|call| call.starts_with("peer-a:prepare:")), "{calls:?}"); assert!(calls.iter().any(|call| call.starts_with("peer-b:prepare:")), "{calls:?}"); assert!(calls.iter().any(|call| call.starts_with("peer-a:abort:")), "{calls:?}"); + assert!( + calls.iter().any(|call| call.starts_with("peer-b:abort:")), + "plain error text must remain ambiguous and receive Abort: {calls:?}" + ); assert!(!calls.iter().any(|call| call.contains(":commit:")), "{calls:?}"); let (persisted, current_version) = load_tier_config_for_update(store.clone()) .await @@ -7826,9 +8331,80 @@ mod tests { assert_eq!(intents[0].state, TierMutationIntentState::Aborted); } + #[tokio::test] + async fn coordinator_old_protocol_rejection_does_not_leave_local_prepared_block() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("tier config fixture should persist"); + let (mut candidate, version) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config fixture should reload"); + let original_version = version.clone(); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + let calls = Arc::new(Mutex::new(Vec::new())); + let old_peer = Arc::new(FakeTierMutationPeer { + label: "peer-v3", + calls: calls.clone(), + prepare: Err("unsupported tier mutation peer protocol version: 4"), + prepare_definitely_rejected: true, + commit: Ok(PeerTierMutationState::Committed), + abort: Err("unsupported tier mutation peer protocol version: 4"), + }) as Arc; + + let err = TIER_MUTATION_TEST_PEERS + .scope(vec![old_peer], async { + TierConfigMgr::update_candidate_owned( + &manager, + store.clone(), + candidate, + version, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + update, + None, + ) + .await + }) + .await + .expect_err("an old v3 peer must reject the v4 mutation before config CAS"); + assert!(matches!(err, TierConfigUpdateError::Publish(_))); + let calls = lock_unpoisoned(&calls).clone(); + assert_eq!(calls.len(), 1, "an explicit pre-dispatch rejection must not receive Abort: {calls:?}"); + assert!(calls[0].starts_with("peer-v3:prepare:"), "{calls:?}"); + + let (unchanged, current_version) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config should remain readable"); + assert_eq!(current_version, original_version); + assert!(unchanged.tiers.contains_key("COLD-A")); + let intents = TierConfigMgr::load_coordinator_mutation_intents(store) + .await + .expect("coordinator intent should be readable"); + assert_eq!(intents.len(), 1); + assert_eq!(intents[0].state, TierMutationIntentState::Aborted); + TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old-peer rejection must restore the local data path immediately"); + } + #[tokio::test] async fn coordinator_prepare_abort_failure_retains_intent_until_reload_converges() { let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } let store = Arc::new(CasConfigStore::default()); let mut persisted = empty_mgr(); persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); @@ -7852,6 +8428,7 @@ mod tests { label: "peer-a", calls: calls.clone(), prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), abort: Err("abort unavailable"), }) as Arc, @@ -7885,6 +8462,9 @@ mod tests { assert_eq!(intents.len(), 1); assert_eq!(intents[0].state, TierMutationIntentState::Prepared); let mutation_id = intents[0].mutation_id; + TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("ambiguous peer cleanup must not leave the coordinator data path fenced"); TIER_MUTATION_TEST_PEERS .scope( @@ -8497,6 +9077,130 @@ mod tests { ); } + #[test] + fn tier_mutation_tombstone_retention_includes_clock_skew_boundary() { + let mut intent = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(0x151)); + intent.expires_at_unix_nanos = 10_000; + intent + .advance(TierMutationIntentState::Aborted, None) + .expect("fixture should advance to an aborted tombstone"); + let skew = i64::try_from(TIER_MUTATION_TOMBSTONE_CLOCK_SKEW.as_nanos()).expect("clock skew should fit i64"); + assert!(!TierConfigMgr::tier_mutation_tombstone_retention_elapsed( + &intent, + intent.expires_at_unix_nanos.saturating_add(skew).saturating_sub(1), + )); + assert!(TierConfigMgr::tier_mutation_tombstone_retention_elapsed( + &intent, + intent.expires_at_unix_nanos.saturating_add(skew), + )); + } + + #[tokio::test] + async fn published_peer_terminal_tombstone_is_retained_without_runtime_fence() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("tier config fixture should persist"); + let (_, current_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config fixture should load with metadata"); + let current_etag = current_etag.expect("tier config fixture should have an ETag"); + let handle = TierConfigMgr::new(); + TierConfigMgr::reload_handle_with(&handle, store.clone()) + .await + .expect("initial reload should publish the authoritative config"); + + let mutation_id = uuid::Uuid::from_u128(0x152); + let mut intent = committed_remove_intent("COLD-A", mutation_id, ¤t_etag); + intent.candidate_digest = tier_config_candidate_digest(&persisted).expect("fixture digest should build"); + intent.expires_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()) + .unwrap_or(i64::MAX) + .saturating_add(i64::try_from(TIER_MUTATION_INTENT_TTL.as_nanos()).expect("intent TTL should fit i64")); + crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent) + .await + .expect("fresh terminal fixture should persist"); + let calls = Arc::new(Mutex::new(Vec::new())); + + TIER_MUTATION_TEST_PEERS + .scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + async { + TierConfigMgr::reload_handle_with(&handle, store.clone()) + .await + .expect("published terminal tombstone should reload"); + TierConfigMgr::reload_handle_with(&handle, store.clone()) + .await + .expect("retained tombstone reload should be idempotent"); + }, + ) + .await; + + assert!(lock_unpoisoned(&calls).is_empty(), "a settled tombstone must not replay peer Commit"); + let retained = TierConfigMgr::load_tier_mutation_intents(store) + .await + .expect("retained tombstone should remain readable"); + assert_eq!(retained, vec![intent]); + let guard = handle.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!( + lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(), + "a retained settled tombstone must not block the published runtime" + ); + } + + #[tokio::test] + async fn peer_terminal_tombstone_gc_uses_etag_and_retains_racing_replacement() { + use crate::services::tier::tier_mutation_intent::{ + save_tier_mutation_intent_record, tier_mutation_intent_record_object_name, + }; + + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("tier config fixture should persist"); + let (_, current_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config fixture should load with metadata"); + let current_etag = current_etag.expect("tier config fixture should have an ETag"); + let handle = TierConfigMgr::new(); + TierConfigMgr::reload_handle_with(&handle, store.clone()) + .await + .expect("initial reload should publish the authoritative config"); + + let mutation_id = uuid::Uuid::from_u128(0x153); + let mut original = committed_remove_intent("COLD-A", mutation_id, ¤t_etag); + original.candidate_digest = tier_config_candidate_digest(&persisted).expect("fixture digest should build"); + original.expires_at_unix_nanos = 1; + save_tier_mutation_intent_record(store.clone(), &original) + .await + .expect("expired terminal fixture should persist"); + let mut replacement = original.clone(); + replacement.expires_at_unix_nanos = 2; + let object = tier_mutation_intent_record_object_name(mutation_id).expect("peer record path should build"); + store + .rewrite_on_next_if_match(object, replacement.encode().expect("replacement should encode")) + .await; + + TierConfigMgr::reload_handle_with(&handle, store.clone()) + .await + .expect("a tombstone ETag race should retain the replacement and retry later"); + + let retained = TierConfigMgr::load_tier_mutation_intents(store) + .await + .expect("racing replacement should remain readable"); + assert_eq!(retained, vec![replacement]); + } + #[tokio::test] async fn quiescent_reload_scans_each_mutation_prefix_at_entry_and_final_race_check() { let store = Arc::new(CasConfigStore::default()); @@ -9098,9 +9802,12 @@ mod tests { .insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); let first = committed_remove_intent("COLD-A", uuid::Uuid::from_u128(0x60), "etag-one"); let second = committed_remove_intent("COLD-A", uuid::Uuid::from_u128(0x61), "etag-two"); - TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[first.clone(), second.clone()]) + TierConfigMgr::apply_committed_mutation_intent_block(&manager, &first) .await - .expect("multiple committed blocks for one tier should reconcile"); + .expect("first committed block should apply"); + TierConfigMgr::apply_committed_mutation_intent_block(&manager, &second) + .await + .expect("second committed block should apply"); let revision = TierConfigMgr::mutation_blocks_revision(&manager).await; let update = TierConfigMgr::admin_update_lock(&manager).await; let err = TierConfigMgr::publish_candidate_owned_with_allowed_mutation_blocks( @@ -9229,10 +9936,8 @@ mod tests { let guard = second_handle.read().await; let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); assert!( - lock_unpoisoned(&runtime) - .committed_mutation_blocks - .get("COLD-A") - .is_some_and(|ids| ids.contains(&second_id)) + lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(), + "a failed tombstone GC must not restore a fence after the authoritative config was published" ); } TIER_MUTATION_TEST_PEERS @@ -9263,6 +9968,71 @@ mod tests { ); } + #[tokio::test] + async fn coordinator_terminal_cleanup_uses_etag_and_retains_racing_replacement() { + use crate::services::tier::tier_mutation_intent::{ + TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, TIER_MUTATION_INTENT_RECORD_PREFIX, save_tier_mutation_intent_record, + tier_mutation_intent_record_object_name, + }; + + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("tier config fixture should persist"); + let (_, current_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config fixture should load with metadata"); + let current_etag = current_etag.expect("tier config fixture should have an ETag"); + let mutation_id = uuid::Uuid::from_u128(0x154); + let mut intent = committed_remove_intent("COLD-A", mutation_id, ¤t_etag); + intent.old_config_etag = Some("previous-etag".to_string()); + intent.candidate_digest = tier_config_candidate_digest(&persisted).expect("fixture digest should build"); + save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent) + .await + .expect("coordinator fixture should persist"); + save_tier_mutation_intent_record(store.clone(), &intent) + .await + .expect("peer fixture should persist"); + + let peer_object = tier_mutation_intent_record_object_name(mutation_id).expect("peer record path should build"); + let coordinator_object = + peer_object.replacen(TIER_MUTATION_INTENT_RECORD_PREFIX, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, 1); + let mut replacement = intent.clone(); + replacement.expires_at_unix_nanos = replacement.expires_at_unix_nanos.saturating_add(1); + store + .rewrite_on_next_if_match(coordinator_object, replacement.encode().expect("coordinator replacement should encode")) + .await; + let calls = Arc::new(Mutex::new(Vec::new())); + let handle = TierConfigMgr::new(); + let err = TIER_MUTATION_TEST_PEERS + .scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + calls, + Ok(PeerTierMutationState::Committed), + )], + async { TierConfigMgr::reload_handle_with(&handle, store.clone()).await }, + ) + .await + .expect_err("a coordinator ETag race must fail closed"); + assert!(err.to_string().contains("changed during conditional cleanup"), "{err}"); + assert_eq!( + TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("coordinator replacement should remain readable"), + vec![replacement] + ); + assert_eq!( + TierConfigMgr::load_tier_mutation_intents(store) + .await + .expect("peer terminal should remain while coordinator cleanup is unresolved"), + vec![intent] + ); + } + #[tokio::test] async fn peer_only_committed_aba_retries_partial_cleanup() { use crate::services::tier::tier_mutation_intent::{ @@ -9356,14 +10126,13 @@ mod tests { .into_iter() .filter(|object| object.starts_with(TIER_MUTATION_INTENT_RECORD_PREFIX)) .collect::>(); - assert_eq!(peer_deletes, vec![first_object.clone(), first_object, second_object]); + assert_eq!(peer_deletes, vec![first_object.clone(), second_object, first_object]); assert_eq!( lock_unpoisoned(&calls).as_slice(), &[ format!("peer-a:commit:{first_id}:intermediate-etag"), format!("peer-a:commit:{second_id}:{current_etag}"), format!("peer-a:commit:{first_id}:intermediate-etag"), - format!("peer-a:commit:{second_id}:{current_etag}"), ] ); } @@ -9847,8 +10616,13 @@ mod tests { .save_tiering_config_if_current(store.clone(), None) .await .expect("tier config fixture should persist"); + let (_, old_config_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("old tier config should load with metadata"); let mutation_id = uuid::Uuid::from_u128(27); - save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &prepared_remove_intent("COLD-A", mutation_id)) + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.old_config_etag = old_config_etag; + save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent) .await .expect("expired unmatched coordinator intent should persist"); let calls = Arc::new(Mutex::new(Vec::new())); @@ -9860,6 +10634,7 @@ mod tests { label: "peer-a", calls: calls.clone(), prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), abort: Err("abort unavailable"), }) as Arc], @@ -9910,6 +10685,50 @@ mod tests { ); } + #[tokio::test] + async fn coordinator_mutation_recovery_retains_prepared_on_third_config() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("third tier config fixture should persist"); + + let mutation_id = uuid::Uuid::from_u128(0x94); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.old_config_etag = Some("different-old-config-etag".to_string()); + intent.candidate_digest = [0x94; 32]; + save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent) + .await + .expect("prepared coordinator intent should persist"); + + let calls = Arc::new(Mutex::new(Vec::new())); + let mut intents = vec![intent]; + TIER_MUTATION_TEST_PEERS + .scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + calls.clone(), + Ok(PeerTierMutationState::Aborted), + )], + async { + TierConfigMgr::recover_prepared_coordinator_mutation_intents(store.clone(), &mut intents) + .await + .expect("uncertain third config should be retained without fanout"); + }, + ) + .await; + + assert_eq!(intents[0].state, TierMutationIntentState::Prepared); + assert!(lock_unpoisoned(&calls).is_empty(), "uncertain recovery must not contact peers"); + let persisted = TierConfigMgr::load_coordinator_mutation_intents(store) + .await + .expect("coordinator scan should retain uncertain evidence"); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].state, TierMutationIntentState::Prepared); + } + #[tokio::test] async fn prepared_mutation_recovery_retries_when_runtime_blocks_change() { let manager = TierConfigMgr::new(); @@ -10017,10 +10836,17 @@ mod tests { .save_tiering_config_if_current(store.clone(), None) .await .expect("tier config fixture should persist"); + let (_, old_config_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("tier config fixture should expose its real ETag"); + let remove_candidate = empty_mgr(); + let mut late_intent = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(0x82)); + late_intent.old_config_etag = old_config_etag; + late_intent.candidate_digest = + tier_config_candidate_digest(&remove_candidate).expect("remove candidate digest should be canonical"); let barrier = store .pause_list_with_prefix_after(TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, 1) .await; - let mutation_id = uuid::Uuid::from_u128(0x82); let handle = TierConfigMgr::new(); TIER_MUTATION_TEST_PEERS @@ -10030,12 +10856,9 @@ mod tests { tokio::time::timeout(Duration::from_secs(1), barrier.arrived.notified()) .await .expect("reload should reach its final coordinator scan"); - save_tier_coordinator_mutation_intent_record_if_absent( - store.clone(), - &prepared_remove_intent("COLD-A", mutation_id), - ) - .await - .expect("racing coordinator intent should persist"); + save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &late_intent) + .await + .expect("racing coordinator intent should persist"); barrier.release.add_permits(1); }; let (reload_result, ()) = tokio::join!(reload, inject); @@ -11461,7 +12284,8 @@ mod tests { delete_log: tokio::sync::Mutex>, list_barrier: tokio::sync::Mutex>>, intent_list_calls: AtomicUsize, - truncate_reference_page_without_marker: AtomicBool, + fail_reference_walk: AtomicBool, + reference_walk_send_count: AtomicUsize, lock_manager: Arc, lock_requests: Mutex>, listed_versions: Mutex>, @@ -11482,7 +12306,8 @@ mod tests { delete_log: tokio::sync::Mutex::new(Vec::new()), list_barrier: tokio::sync::Mutex::new(None), intent_list_calls: AtomicUsize::new(0), - truncate_reference_page_without_marker: AtomicBool::new(false), + fail_reference_walk: AtomicBool::new(false), + reference_walk_send_count: AtomicUsize::new(0), lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()), lock_requests: Mutex::new(Vec::new()), listed_versions: Mutex::new(Vec::new()), @@ -11531,8 +12356,12 @@ mod tests { self.if_match_race_rewrite.lock().await.push_back((object, data)); } - fn omit_truncated_reference_marker(&self) { - self.truncate_reference_page_without_marker.store(true, Ordering::SeqCst); + fn fail_reference_walk(&self) { + self.fail_reference_walk.store(true, Ordering::SeqCst); + } + + fn reference_walk_send_count(&self) -> usize { + self.reference_walk_send_count.load(Ordering::SeqCst) } async fn fail_next_delete_with_prefix(&self, prefix: &str) { @@ -11750,7 +12579,7 @@ mod tests { Err(Error::NotImplemented) } - async fn delete_object(&self, bucket: &str, object: &str, _opts: Self::ObjectOptions) -> Result { + async fn delete_object(&self, bucket: &str, object: &str, opts: Self::ObjectOptions) -> Result { self.delete_log.lock().await.push(object.to_string()); let mut fail_delete_prefix = self.fail_delete_prefix.lock().await; let fail_now = match fail_delete_prefix.as_mut() { @@ -11769,10 +12598,32 @@ mod tests { return Err(Error::other("injected tier mutation intent delete failure")); } drop(fail_delete_prefix); + let race_rewrite = if opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_match_value) + .is_some() + { + self.if_match_race_rewrite.lock().await.pop_front() + } else { + None + }; let mut objects = self.objects.lock().await; + if let Some((target, data)) = race_rewrite + && target == object + { + let etag = self.next_object_etag(&data); + objects.insert(object.to_string(), (data, etag)); + } let (data, etag) = objects - .remove(object) + .get(object) + .cloned() .ok_or_else(|| Error::ObjectNotFound(bucket.to_string(), object.to_string()))?; + opts.precondition_check(&ObjectInfo { + etag: Some(etag.clone()), + ..Default::default() + })?; + objects.remove(object); Ok(ObjectInfo { bucket: bucket.to_string(), name: object.to_string(), @@ -12008,7 +12859,7 @@ mod tests { if is_truncated { objects.truncate(limit); } - let (mut next_marker, next_version_idmarker) = if is_truncated { + let (next_marker, next_version_idmarker) = if is_truncated { objects .last() .map(|object| (Some(object.name.clone()), object.version_id.map(|version| version.to_string()))) @@ -12016,9 +12867,6 @@ mod tests { } else { (None, None) }; - if is_truncated && self.truncate_reference_page_without_marker.load(Ordering::SeqCst) { - next_marker = None; - } Ok(StorageListObjectVersionsInfo { is_truncated, next_marker, @@ -12030,13 +12878,57 @@ mod tests { async fn walk( self: Arc, - _rx: Self::WalkCancellation, - _bucket: &str, - _prefix: &str, - _result: Self::WalkResultSender, - _opts: Self::WalkOptions, + rx: Self::WalkCancellation, + bucket: &str, + prefix: &str, + result: Self::WalkResultSender, + opts: Self::WalkOptions, ) -> Result<()> { - Err(Error::NotImplemented) + if self.fail_reference_walk.load(Ordering::SeqCst) { + if result + .send(StorageObjectInfoOrErr { + item: None, + err: Some(Error::other("injected tier reference walk failure")), + }) + .await + .is_err() + { + return Ok(()); + } + } + let mut objects = self + .listed_versions + .lock() + .expect("tier reference fixture should not poison") + .iter() + .filter(|object| object.bucket == bucket && object.name.starts_with(prefix)) + .filter(|object| opts.include_free_versions || !object.transitioned_object.free_version) + .cloned() + .collect::>(); + objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right))); + if let Some(marker) = opts.marker.as_deref() { + objects.retain(|object| object.name.as_str() > marker); + } + if opts.limit > 0 { + objects.truncate(opts.limit); + } + for object in objects { + if rx.is_cancelled() { + return Err(Error::other("tier reference fixture walk cancelled")); + } + self.reference_walk_send_count.fetch_add(1, Ordering::SeqCst); + if result + .send(StorageObjectInfoOrErr { + item: Some(object), + err: None, + }) + .await + .is_err() + { + return Ok(()); + } + } + Ok(()) } } @@ -12172,6 +13064,13 @@ mod tests { .await .expect("reference proof fixture should persist"); store.add_listed_version(transitioned_tier_object("photos", "2026/a.jpg", "COLD-A", Some(identity))); + for index in 0..500 { + store.add_listed_version(ObjectInfo { + bucket: "photos".to_string(), + name: format!("zz-safe/{index:04}.jpg"), + ..Default::default() + }); + } let manager = TierConfigMgr::new(); manager.write().await.tiers.insert("COLD-A".to_string(), tier); @@ -12187,6 +13086,10 @@ mod tests { other => panic!("reference proof failures must be surfaced as publish errors: {other:?}"), } assert!(manager.read().await.tiers.contains_key("COLD-A")); + assert!( + store.reference_walk_send_count() < 501, + "the proof should cancel its internal walk after the first authoritative blocker" + ); assert!( load_tier_config_for_update(store) .await @@ -12337,7 +13240,7 @@ mod tests { "COLD-A", Some(current_identity), )); - let err = ensure_no_authoritative_tier_object_references(store, &[target], false) + let err = ensure_no_authoritative_tier_object_references(store.clone(), &[target], false) .await .expect_err("references to the old destination identity must block rebind"); assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); @@ -12497,6 +13400,7 @@ mod tests { let journal_store = Arc::new(CasConfigStore::default()); let journal = crate::bucket::lifecycle::tier_sweeper::Jentry { + persisted_version: 0, obj_name: "remote/journal-object".to_string(), version_id: "v1".to_string(), tier_name: "COLD-A".to_string(), @@ -12505,6 +13409,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; journal_store .insert_config_object( @@ -12573,7 +13478,7 @@ mod tests { } #[tokio::test] - async fn zero_reference_proof_fails_closed_when_version_listing_marker_is_missing() { + async fn zero_reference_proof_fails_closed_when_internal_walk_fails() { let current = build_rustfs_tier("COLD-A"); let current_identity = tier_backend_identity(¤t).expect("current identity should encode"); let target = TierMutationIntentTarget { @@ -12582,24 +13487,33 @@ mod tests { new_backend_identity: None, }; let store = Arc::new(CasConfigStore::default()); - for index in 0..=TIER_REFERENCE_PROOF_LIST_LIMIT { + store.add_listed_version(ObjectInfo { + bucket: "photos".to_string(), + name: "safe/object.jpg".to_string(), + ..Default::default() + }); + for index in 0..500 { store.add_listed_version(ObjectInfo { bucket: "photos".to_string(), - name: format!("safe/{index:04}.jpg"), + name: format!("safe/trailing-{index:04}.jpg"), ..Default::default() }); } - store.omit_truncated_reference_marker(); + store.fail_reference_walk(); - let err = ensure_no_authoritative_tier_object_references(store, &[target], false) + let err = ensure_no_authoritative_tier_object_references(store.clone(), &[target], false) .await - .expect_err("truncated authoritative reference scan without a marker must fail closed"); + .expect_err("authoritative internal walk failure must fail closed"); assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); assert!( - err.message.contains("truncated without a next marker"), + err.message.contains("injected tier reference walk failure"), "unexpected reference proof error: {}", err.message ); + assert!( + store.reference_walk_send_count() <= 100, + "the first terminal walk error may overshoot only by the bounded channel capacity" + ); } #[test] @@ -13270,7 +14184,7 @@ mod tests { } #[tokio::test] - async fn committed_mutation_refresh_keeps_fence_and_retries_after_cleanup_failure() { + async fn committed_mutation_refresh_clears_fence_and_retries_after_cleanup_failure() { let (manager, store, mutation_id) = committed_refresh_fixture(true).await; TIER_MUTATION_TEST_PEERS .scope(Vec::new(), async { @@ -13300,21 +14214,18 @@ mod tests { let guard = manager.read().await; let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); assert!( - lock_unpoisoned(&runtime) - .committed_mutation_blocks - .get("COLD-A") - .is_some_and(|ids| ids.contains(&mutation_id)), - "cleanup failure must retain the committed runtime fence" + lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(), + "a published configuration must not restore its committed runtime fence after cleanup failure" ); } tokio::time::timeout(Duration::from_secs(10), async { loop { - let converged = { - let guard = manager.read().await; - let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); - lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty() - }; + let converged = TierConfigMgr::load_tier_mutation_intents(store.clone()) + .await + .expect("committed cleanup retry scan should succeed") + .iter() + .all(|intent| intent.mutation_id != mutation_id); if converged { break; } @@ -13348,6 +14259,15 @@ mod tests { assert_eq!(tier_mutation_refresh_retry_delay(u32::MAX), TIER_MUTATION_REFRESH_RETRY_CAP); } + #[test] + fn terminal_mutation_cleanup_read_failure_keeps_retry_armed() { + assert!(TierConfigMgr::terminal_mutation_cleanup_requires_retry(Ok(true))); + assert!(!TierConfigMgr::terminal_mutation_cleanup_requires_retry(Ok(false))); + assert!(TierConfigMgr::terminal_mutation_cleanup_requires_retry(Err(io::Error::other( + "injected durable cleanup read failure", + )))); + } + #[test] fn legacy_refresh_method_signature_remains_callable() { async fn call_legacy_refresh(manager: &mut TierConfigMgr, api: Arc) { diff --git a/crates/ecstore/src/services/tier/tier_mutation_intent.rs b/crates/ecstore/src/services/tier/tier_mutation_intent.rs index 4b9ea7e61..288b02969 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_intent.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_intent.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use serde::{Deserialize, Serialize}; @@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE; pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records"; pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators"; +pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64; const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3; pub(crate) type TierMutationDigest = [u8; 32]; +static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> = + LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(()))); + +/// Serializes every local phase and recovery action for one mutation id while +/// retaining bounded parallelism for unrelated mutations. +pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> { + TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)] + .lock() + .await +} + +fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize { + let raw = mutation_id.as_u128(); + let mut mixed = (raw as u64) ^ ((raw >> 64) as u64); + // MurmurHash3's 64-bit finalizer gives stable diffusion without allocating + // or relying on RandomState, whose seed differs between processes. + mixed ^= mixed >> 33; + mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd); + mixed ^= mixed >> 33; + mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + mixed ^= mixed >> 33; + (mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1) +} + pub(crate) type Result = std::result::Result; #[derive(Debug, thiserror::Error)] @@ -265,6 +290,30 @@ impl TierMutationIntent { && self.expires_at_unix_nanos == other.expires_at_unix_nanos } + /// Reconstruct the exact Prepared record that originally produced this + /// intent. Abort RPCs are identity-bound to that payload; serializing an + /// Aborted terminal record would both violate the wire contract and use a + /// different revision if a missing peer has to persist a tombstone. + pub(crate) fn original_prepared(&self) -> Result { + if self.state == TierMutationIntentState::Prepared { + self.validate()?; + return Ok(self.clone()); + } + let mut prepared = self.clone(); + prepared.revision = + prepared + .revision + .checked_sub(1) + .filter(|revision| *revision != 0) + .ok_or(TierMutationIntentError::Corrupt( + "terminal intent cannot reconstruct its prepared revision", + ))?; + prepared.state = TierMutationIntentState::Prepared; + prepared.committed_config_etag = None; + prepared.validate()?; + Ok(prepared) + } + pub(crate) fn encode(&self) -> Result> { self.validate()?; let intent_bytes = serde_json::to_vec(self)?; @@ -439,6 +488,16 @@ where load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await } +pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag( + api: Arc, + mutation_id: Uuid, +) -> EcstoreResult<(TierMutationIntent, String)> +where + S: EcstoreObjectIO, +{ + load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await +} + async fn load_tier_mutation_intent_record_with_etag_at_prefix( api: Arc, prefix: &str, @@ -507,6 +566,7 @@ where .await } +#[cfg(test)] pub(crate) async fn delete_tier_mutation_intent_record(api: Arc, mutation_id: Uuid) -> EcstoreResult<()> where S: EcstoreObjectOperations, @@ -514,13 +574,36 @@ where delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await } -pub(crate) async fn delete_tier_coordinator_mutation_intent_record(api: Arc, mutation_id: Uuid) -> EcstoreResult<()> +pub(crate) async fn delete_tier_mutation_intent_record_if_current( + api: Arc, + mutation_id: Uuid, + current_etag: &str, +) -> EcstoreResult<()> where S: EcstoreObjectOperations, { - delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await + delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag) + .await } +pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current( + api: Arc, + mutation_id: Uuid, + current_etag: &str, +) -> EcstoreResult<()> +where + S: EcstoreObjectOperations, +{ + delete_tier_mutation_intent_record_if_current_with_prefix( + api, + TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, + mutation_id, + current_etag, + ) + .await +} + +#[cfg(test)] async fn delete_tier_mutation_intent_record_with_prefix(api: Arc, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()> where S: EcstoreObjectOperations, @@ -533,6 +616,40 @@ where } } +async fn delete_tier_mutation_intent_record_if_current_with_prefix( + api: Arc, + prefix: &str, + mutation_id: Uuid, + current_etag: &str, +) -> EcstoreResult<()> +where + S: EcstoreObjectOperations, +{ + if current_etag.trim().is_empty() { + return Err(Error::other("tier mutation intent current ETag is empty")); + } + let object = + tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?; + match api + .delete_object( + RUSTFS_META_BUCKET, + &object, + ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some(current_etag.to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(_) => Ok(()), + Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound), + Err(err) => Err(err), + } +} + pub(crate) async fn advance_tier_mutation_intent_record_idempotent( api: Arc, mutation_id: Uuid, @@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool { #[cfg(test)] mod tests { use super::*; + use std::time::Duration; const OLD_IDENTITY: TierDestinationId = [1; 32]; const NEW_IDENTITY: TierDestinationId = [2; 32]; @@ -736,6 +854,61 @@ mod tests { } } + #[test] + fn mutation_mutex_uses_exactly_64_stable_shards() { + assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64); + assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS); + + let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse"); + let shard = tier_mutation_mutex_shard_index(mutation_id); + assert!(shard < TIER_MUTATION_MUTEX_SHARDS); + assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id)); + } + + #[tokio::test] + async fn mutation_mutex_serializes_the_same_id() { + let mutation_id = Uuid::new_v4(); + let first = acquire_tier_mutation_mutex(mutation_id).await; + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + started_tx.send(()).expect("test receiver should remain alive"); + let _second = acquire_tier_mutation_mutex(mutation_id).await; + acquired_tx.send(()).expect("test receiver should remain alive"); + }); + started_rx.await.expect("waiter should start"); + assert!( + tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx) + .await + .is_err(), + "the same mutation id must not enter concurrently" + ); + + drop(first); + tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx) + .await + .expect("waiter should acquire after release") + .expect("waiter should report acquisition"); + waiter.await.expect("waiter task should finish"); + } + + #[tokio::test] + async fn mutation_mutex_allows_different_shards_to_progress() { + let first_id = Uuid::new_v4(); + let first_shard = tier_mutation_mutex_shard_index(first_id); + let second_id = (0..1024) + .map(|_| Uuid::new_v4()) + .find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard) + .expect("a distinct shard should be easy to find"); + let first = acquire_tier_mutation_mutex(first_id).await; + + let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id)) + .await + .expect("a different shard must not wait for the first mutation"); + drop(first); + } + #[test] fn intent_round_trip_preserves_committed_state() { let mut intent = prepared_intent(); @@ -873,6 +1046,37 @@ mod tests { )); } + #[test] + fn terminal_intent_reconstructs_original_prepared_abort_payload() { + for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] { + let original = prepared_intent(); + let mut intent = original.clone(); + let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string()); + intent + .advance(terminal, committed_etag) + .expect("terminal transition should succeed"); + + let reconstructed = intent + .original_prepared() + .expect("terminal record should recover prepared payload"); + assert_eq!(reconstructed, original); + assert!(intent.same_identity_as(&reconstructed)); + } + } + + #[test] + fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() { + let mut corrupt = prepared_intent(); + corrupt.state = TierMutationIntentState::Aborted; + + assert!(matches!( + corrupt.original_prepared(), + Err(TierMutationIntentError::Corrupt( + "terminal intent cannot reconstruct its prepared revision" + )) + )); + } + #[test] fn intent_validation_rejects_placeholder_identity() { let mut intent = prepared_intent(); diff --git a/crates/ecstore/src/services/tier/tier_mutation_peer.rs b/crates/ecstore/src/services/tier/tier_mutation_peer.rs index a7f261e2f..c81c1ed5f 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_peer.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_peer.rs @@ -15,12 +15,13 @@ use std::sync::Arc; use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; +use time::OffsetDateTime; use uuid::Uuid; use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches}; use super::tier_mutation_intent::{ - MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent, - load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent, + MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex, + advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent, }; use crate::error::{Error, StorageError}; use crate::store::ECStore; @@ -57,6 +58,8 @@ pub enum TierMutationPeerError { CommitProofMismatch, #[error("tier mutation peer abort proof does not match the persisted tier configuration")] AbortProofMismatch, + #[error("tier mutation peer prepared intent has expired")] + ExpiredIntent, #[error("tier mutation peer runtime error: {0}")] Runtime(#[source] AdminError), #[error("tier mutation peer store error: {0}")] @@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request( canonical_payload: &[u8], ) -> TierMutationPeerResult { validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?; + let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await; match phase { TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await, TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await, @@ -103,43 +107,57 @@ async fn handle_prepare( } let tier_config_mgr = api.tier_config_mgr(); - match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await { - Ok(()) => { - TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent) - .await - .map_err(TierMutationPeerError::Runtime)?; - Ok(TierMutationPeerOutcome { - state: TierMutationPeerState::Prepared, - applied: true, - }) - } - Err(Error::PreconditionFailed) => { - let existing = load_tier_mutation_intent_record(api, mutation_id).await?; - if !existing.same_identity_as(&intent) { - return Err(TierMutationPeerError::ConflictingIntent); + for _ in 0..3 { + let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await { + Ok(existing) => { + if !existing.same_identity_as(&intent) { + return Err(TierMutationPeerError::ConflictingIntent); + } + (existing, false) } - match existing.state { - TierMutationIntentState::Prepared => { - TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing) - .await - .map_err(TierMutationPeerError::Runtime)?; + Err(Error::ConfigNotFound) => { + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX); + if intent.expires_at_unix_nanos <= now { + return Err(TierMutationPeerError::ExpiredIntent); } - TierMutationIntentState::Committed => { - TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing) - .await - .map_err(TierMutationPeerError::Runtime)?; - } - TierMutationIntentState::Aborted => { - TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await; + match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await { + Ok(()) => (intent.clone(), true), + Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err.into()), } } - Ok(TierMutationPeerOutcome { - state: peer_state_from_intent(existing.state), - applied: false, - }) + Err(err) => return Err(err.into()), + }; + + match stored.state { + TierMutationIntentState::Prepared => { + TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored) + .await + .map_err(TierMutationPeerError::Runtime)?; + TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored) + .await + .map_err(TierMutationPeerError::Runtime)?; + } + TierMutationIntentState::Committed => { + TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored) + .await + .map_err(TierMutationPeerError::Runtime)?; + } + TierMutationIntentState::Aborted => { + TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id) + .await + .map_err(TierMutationPeerError::Runtime)?; + TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await; + } } - Err(err) => Err(err.into()), + return Ok(TierMutationPeerOutcome { + state: peer_state_from_intent(stored.state), + applied, + }); } + Err(TierMutationPeerError::Store(Error::other( + "tier mutation prepare raced repeatedly with another decision", + ))) } async fn handle_commit( @@ -201,26 +219,106 @@ async fn handle_abort( mutation_id: Uuid, canonical_payload: &[u8], ) -> TierMutationPeerResult { - if !canonical_payload.is_empty() { - return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string())); + let prepared = TierMutationIntent::decode(mutation_id, canonical_payload) + .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?; + if prepared.state != TierMutationIntentState::Prepared { + return Err(TierMutationPeerError::InvalidPayload( + "abort payload must carry the original prepared intent".to_string(), + )); } - let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?; - if existing.state == TierMutationIntentState::Prepared - && !tier_config_abort_matches(api.clone(), &existing) - .await - .map_err(Error::other)? - { - return Err(TierMutationPeerError::AbortProofMismatch); + let mut tombstone = prepared.clone(); + tombstone + .advance(TierMutationIntentState::Aborted, None) + .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?; + + for _ in 0..3 { + match load_tier_mutation_intent_record(api.clone(), mutation_id).await { + Ok(existing) => { + if !existing.same_identity_as(&prepared) { + return Err(TierMutationPeerError::ConflictingIntent); + } + match existing.state { + TierMutationIntentState::Committed => { + return Ok(TierMutationPeerOutcome { + state: TierMutationPeerState::Committed, + applied: false, + }); + } + TierMutationIntentState::Aborted => { + TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id) + .await + .map_err(TierMutationPeerError::Runtime)?; + TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await; + return Ok(TierMutationPeerOutcome { + state: TierMutationPeerState::Aborted, + applied: false, + }); + } + TierMutationIntentState::Prepared => {} + } + if !tier_config_abort_matches(api.clone(), &prepared) + .await + .map_err(Error::other)? + { + return Err(TierMutationPeerError::AbortProofMismatch); + } + let advanced = advance_tier_mutation_intent_record_idempotent( + api.clone(), + mutation_id, + TierMutationIntentState::Aborted, + None, + ) + .await; + let (intent, applied) = match advanced { + Ok(result) => result, + Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await { + Ok(current) + if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared => + { + (current, false) + } + _ => return Err(err.into()), + }, + }; + if intent.state == TierMutationIntentState::Aborted { + TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await; + TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id) + .await + .map_err(TierMutationPeerError::Runtime)?; + } + return Ok(TierMutationPeerOutcome { + state: peer_state_from_intent(intent.state), + applied, + }); + } + Err(Error::ConfigNotFound) => { + if !tier_config_abort_matches(api.clone(), &prepared) + .await + .map_err(Error::other)? + { + return Err(TierMutationPeerError::AbortProofMismatch); + } + match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await { + Ok(()) => { + TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id) + .await + .map_err(TierMutationPeerError::Runtime)?; + TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await; + return Ok(TierMutationPeerOutcome { + state: TierMutationPeerState::Aborted, + applied: true, + }); + } + Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err.into()), + } + } + Err(err) => return Err(err.into()), + } } - let (intent, applied) = - advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?; - if intent.state == TierMutationIntentState::Aborted { - TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await; - } - Ok(TierMutationPeerOutcome { - state: peer_state_from_intent(intent.state), - applied, - }) + Err(TierMutationPeerError::Store(Error::other( + "tier mutation abort raced repeatedly with prepare", + ))) } fn validate_peer_request_envelope( @@ -228,7 +326,10 @@ fn validate_peer_request_envelope( mutation_id: Uuid, canonical_payload: &[u8], ) -> TierMutationPeerResult<()> { - if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION { + if !matches!( + protocol_version, + rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION + ) { return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version)); } if mutation_id.is_nil() { @@ -276,6 +377,8 @@ mod tests { #[test] fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() { let mutation_id = Uuid::new_v4(); + validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload") + .expect("v3 must remain accepted during the v4 rollout"); assert!(matches!( validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"), Err(TierMutationPeerError::UnsupportedProtocolVersion(_)) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 12a171643..34bd6ccb2 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -859,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock = OnceLock::new(); mod core; #[cfg(test)] -pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier}; +pub(crate) use core::io_primitives::disk_call_counters; +#[cfg(all(test, feature = "test-util"))] +pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; mod ctx; mod metadata; mod ops; @@ -872,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation; pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; #[cfg(test)] pub(crate) use ops::object::DeleteObjectCommitBarrier; -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; #[cfg(all(test, feature = "test-util"))] diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 20a1911f2..5ff71ad11 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard { } #[cfg(any(test, feature = "test-util"))] -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MultipartCommitPause { NewUploadBeforeLockLost, PutPartBeforeLockAcquire, diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 6a680431b..ffa18783e 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -94,6 +94,8 @@ const GET_MID_SIZE_STREAMING_MIN_SIZE: usize = 128 * 1024 + 1; // high-concurrency performance envelope; existing codec/legacy gates decide // which established reader handles the object. const GET_MID_SIZE_STREAMING_MAX_SIZE: usize = 512 * 1024; +const EVENT_LIFECYCLE_TRANSITION_CLEANUP: &str = "lifecycle_transition_cleanup"; +const EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER: &str = "lifecycle_transitioned_delete_cleanup_owner"; fn is_get_mid_size_streaming_enabled() -> bool { #[cfg(test)] @@ -200,14 +202,11 @@ use super::super::capacity_scope_from_disks; #[cfg(all(test, feature = "test-util"))] use super::super::get_lock_acquire_timeout; use crate::bucket::lifecycle::{ - tier_delete_journal::{ - enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry, - record_tier_delete_journal_backend_identity, remove_tier_delete_journal_entry, tier_delete_journal_object_name, - }, + tier_delete_journal::{TierDeleteDispatchAuthorization, record_tier_delete_journal_backend_identity}, tier_sweeper::{ - Jentry, RemoteTierDeleteOutcome, TierDeleteJournalState, attach_tier_delete_source, + Jentry, RemoteTierDeleteOutcome, attach_tier_delete_source, delete_confirmed_transition_candidate_exact_with_lease_idempotent, delete_object_from_remote_tier_with_lease_idempotent, - transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry, + transitioned_force_delete_journal_entry, }, transition_transaction::{ TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, @@ -241,7 +240,7 @@ use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY}; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; -use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; +use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; @@ -249,6 +248,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking; use crate::storage_api_contracts::object::HTTPPreconditions; use crate::store::ECStore; use crate::store::utils::clean_metadata; +use crate::store::{RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence}; use futures::FutureExt as _; use http::HeaderValue; #[cfg(test)] @@ -269,6 +269,49 @@ use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; +fn record_transitioned_delete_cleanup_owner(bucket: &str, object: &str, batch: bool) { + metrics::counter!("rustfs_ilm_transitioned_delete_cleanup_owners_total", "owner" => "tier_free_version").increment(1); + debug!( + event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + cleanup_owner = "tier_free_version", + batch, + "Selected the durable cleanup owner for a transitioned source delete" + ); +} + +async fn acquire_single_tier_delete_lease( + bucket: &str, + object: &str, + opts: &ObjectOptions, + source: &ObjectInfo, +) -> Result> { + let Some(api) = opts.tier_delete_journal_api.as_ref() else { + return Ok(None); + }; + if source.transitioned_object.status != TRANSITION_COMPLETE || set_disk_delete_creates_delete_marker(opts) { + return Ok(None); + } + let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?; + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity( + &api.tier_config_mgr(), + &source.transitioned_object.tier, + backend_identity, + ) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &source.transitioned_object.tier).await, + } + .map_err(Error::other)?; + record_transitioned_delete_cleanup_owner(bucket, object, false); + Ok(Some(lease)) +} + const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256; @@ -447,23 +490,47 @@ fn lifecycle_delete_all_tier_journal_entry( object: &str, version: &FileInfo, opts: &ObjectOptions, -) -> Result> { +) -> Result> { if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE { return Ok(None); } - if version.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown { - return Err(StorageError::PreconditionFailed); - } let logical_object = decode_dir_object(object); let mut source = ObjectInfo::from_file_info(version, bucket, object, true); source.version_id = source.version_id.filter(|version_id| !version_id.is_nil()); - let mut entry = transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state) - .ok_or(StorageError::PreconditionFailed)?; + let Some(mut entry) = transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state) + else { + return Ok(None); + }; attach_tier_delete_source(&mut entry, bucket, &logical_object, &source, opts.versioned, opts.version_suspended); record_tier_delete_journal_backend_identity(&mut entry, &source.user_defined).map_err(Error::other)?; - let name = tier_delete_journal_object_name(&entry); - Ok(Some((name, entry))) + if !entry.can_replace_tier_free_version() { + return Ok(None); + } + Ok(Some(entry)) +} + +async fn acquire_lifecycle_delete_all_tier_lease(version: &FileInfo, opts: &ObjectOptions) -> Result> { + if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE { + return Ok(None); + } + let Some(api) = opts.tier_delete_journal_api.as_ref() else { + return Ok(None); + }; + let backend_identity = tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?; + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity( + &api.tier_config_mgr(), + &version.transition_tier, + backend_identity, + ) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &version.transition_tier).await, + } + .map_err(Error::other)?; + Ok(Some(lease)) } fn lifecycle_delete_all_replication_delete( @@ -525,31 +592,6 @@ fn lifecycle_delete_all_replication_delete( Ok(Some((replication_state, deleted_object))) } -async fn prepare_lifecycle_delete_all_tier_journals( - bucket: &str, - object: &str, - plan: &LifecycleDeleteAllPlan<'_>, - opts: &ObjectOptions, -) -> Result<()> { - let Some(api) = opts.tier_delete_journal_api.as_ref() else { - return Ok(()); - }; - let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?; - for version in plan.history.iter().copied().chain(plan.trigger) { - let Some((name, entry)) = lifecycle_delete_all_tier_journal_entry(bucket, object, version, opts)? else { - continue; - }; - if journal.lock().contains(&name) { - continue; - } - persist_tier_delete_journal_entry(Arc::clone(api), &entry) - .await - .map_err(Error::other)?; - journal.lock().insert(name, entry); - } - Ok(()) -} - #[cfg(test)] mod lifecycle_delete_all_plan_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; @@ -787,7 +829,7 @@ mod lifecycle_delete_all_plan_tests { } #[test] - fn tier_journal_coverage_is_source_exact_and_rejects_legacy_unknown_state() { + fn tier_journal_coverage_is_source_exact_and_falls_back_for_legacy_metadata() { let identity = [7_u8; 32]; let version_id = Uuid::from_u128(1); let mut metadata = HashMap::new(); @@ -812,22 +854,29 @@ mod lifecycle_delete_all_plan_tests { ..Default::default() }; - let (first_name, _) = - lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(2)), &opts) - .expect("exact transitioned source should be journalable") - .expect("completed transition should require a journal"); - let (second_name, _) = - lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(3)), &opts) - .expect("second pool source should be journalable") - .expect("completed transition should require a journal"); - assert_ne!(first_name, second_name, "each pool-local source needs independent coverage"); + let first = lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(2)), &opts) + .expect("exact transitioned source should be journalable") + .expect("completed transition should require a journal"); + let second = lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(3)), &opts) + .expect("second pool source should be journalable") + .expect("completed transition should require a journal"); + assert_ne!(first.source, second.source, "each pool-local source needs independent coverage"); let mut unknown = transitioned(Uuid::from_u128(4)); unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; - assert!(matches!( - lifecycle_delete_all_tier_journal_entry("bucket", "object", &unknown, &opts), - Err(StorageError::PreconditionFailed) - )); + assert!( + lifecycle_delete_all_tier_journal_entry("bucket", "object", &unknown, &opts) + .expect("legacy Unknown metadata should use the free-version fallback") + .is_none() + ); + + let mut missing_identity = transitioned(Uuid::from_u128(5)); + missing_identity.metadata.clear(); + assert!( + lifecycle_delete_all_tier_journal_entry("bucket", "object", &missing_identity, &opts) + .expect("legacy metadata without destination identity should use the free-version fallback") + .is_none() + ); } #[cfg(not(feature = "rio-v2"))] @@ -2729,7 +2778,27 @@ impl SetDisks { data: &mut PutObjReader, opts: &ObjectOptions, ) -> Result<(ObjectInfo, Option)> { - self.put_object_with_old_current_size_boxed(bucket, object, data, opts).await + self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None) + .await + } + + pub(crate) async fn put_object_with_old_current_size_for_data_movement( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + publication_fence: RemoteTuplePublicationFence, + ) -> Result<(ObjectInfo, Option)> { + if !opts.data_movement { + return Err(Error::other("publication-fenced PUT requires data_movement options")); + } + let mut opts = opts.clone(); + // The owned publication capability supplies the exact target write + // domain at commit time; the generic PUT path must not reacquire it. + opts.no_lock = true; + self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence)) + .await } fn put_object_with_old_current_size_boxed<'a>( @@ -2738,8 +2807,9 @@ impl SetDisks { object: &'a str, data: &'a mut PutObjReader, opts: &'a ObjectOptions, + publication_fence: Option, ) -> impl Future)>> + Send + 'a { - Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts)) + Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence)) } async fn put_object_with_old_current_size_inner( @@ -2748,7 +2818,19 @@ impl SetDisks { object: &str, data: &mut PutObjReader, opts: &ObjectOptions, + mut publication_fence: Option, ) -> Result<(ObjectInfo, Option)> { + if publication_fence.is_none() + && opts.data_movement + && rustfs_utils::http::metadata_compat::contains_key_str( + &opts.user_defined, + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + ) + { + return Err(Error::other( + "data movement cannot publish transition ownership without a publication capability", + )); + } crate::hp_guard!("SetDisks::put_object"); let mut scope_outcome_guard = opts .scanner_publication_commit_scope @@ -2764,6 +2846,7 @@ impl SetDisks { let mut decommission_target_lock_covered = false; let mut decommission_capacity_guard = None; let mut bucket_lifecycle_guard = None; + let mut publication_commit_guard: Option = None; // This pre-body check is advisory fast-fail only: the authoritative // precondition evaluation happens under the commit namespace lock @@ -3233,7 +3316,8 @@ impl SetDisks { decommission_target_lock_covered = target_lock_covered; decommission_capacity_guard = capacity_guard; } - if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered { + if publication_fence.is_some() || (!opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered) + { #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id @@ -3246,19 +3330,23 @@ impl SetDisks { .await?, ); } - #[cfg(any(test, feature = "test-util"))] - { - object_lock_guard = Some( - self.acquire_write_lock_diag_with_pending_hook("put_object_commit", bucket, object, || { - notify_put_object_commit_namespace_pending(bucket, object); - }) - .await?, - ); - notify_put_object_commit_namespace_acquired(bucket, object); - } - #[cfg(not(any(test, feature = "test-util")))] - { - object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?); + if let Some(fence) = publication_fence.take() { + publication_commit_guard = Some(fence.into_commit_guard(self.pool_index, bucket, object).await?); + } else { + #[cfg(any(test, feature = "test-util"))] + { + object_lock_guard = Some( + self.acquire_write_lock_diag_with_pending_hook("put_object_commit", bucket, object, || { + notify_put_object_commit_namespace_pending(bucket, object); + }) + .await?, + ); + notify_put_object_commit_namespace_acquired(bucket, object); + } + #[cfg(not(any(test, feature = "test-util")))] + { + object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?); + } } } #[cfg(any(test, feature = "test-util"))] @@ -3371,7 +3459,11 @@ impl SetDisks { // Fence every commit-time read before entering rename_data. Once // rename_data returns Ok the write is durable and must not be aborted. - if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || publication_commit_guard + .as_ref() + .is_some_and(RemoteTuplePublicationCommitGuard::is_lock_lost) + { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "put_object_commit", bucket: bucket.to_string(), @@ -3572,6 +3664,7 @@ impl SetDisks { let commit_tmp_dir = tmp_dir.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); + let commit_publication_guard = publication_commit_guard.take(); let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); let commit_decommission_capacity_guard = decommission_capacity_guard.take(); let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone(); @@ -3580,7 +3673,9 @@ impl SetDisks { // its terminal state is known before the coordinator releases // remote leases. let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) - && (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some()) + && (commit_object_lock_guard.is_some() + || commit_decommission_object_lock_guard.is_some() + || commit_publication_guard.is_some()) && commit_scanner_publication_scope.is_none(); let detach_commit_owner = commit_scanner_publication_scope.is_some() || commit_allows_early_ack @@ -3602,6 +3697,7 @@ impl SetDisks { let commit = move |cancellation: Option| async move { let mut _object_lock_guard = commit_object_lock_guard; let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; + let mut _publication_guard = commit_publication_guard; let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard; let mut _decommission_capacity_guard = commit_decommission_capacity_guard; let mut quota_reservation = quota_reservation; @@ -3618,6 +3714,9 @@ impl SetDisks { || _decommission_object_lock_guard .as_ref() .is_some_and(|guard| guard.is_lock_lost()) + || _publication_guard + .as_ref() + .is_some_and(RemoteTuplePublicationCommitGuard::is_lock_lost) || commit_namespace_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) @@ -3674,6 +3773,9 @@ impl SetDisks { || _decommission_object_lock_guard .as_ref() .is_some_and(|guard| guard.is_lock_lost()) + || _publication_guard + .as_ref() + .is_some_and(RemoteTuplePublicationCommitGuard::is_lock_lost) || commit_namespace_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) @@ -3792,6 +3894,7 @@ impl SetDisks { .or_else(|| commit_version_suspended.then(Uuid::nil)) .map(|version_id| version_id.to_string()); let object_lock_guard = _object_lock_guard.take(); + let publication_guard = _publication_guard.take(); let bucket_lifecycle_guard = _bucket_lifecycle_guard.take(); let decommission_object_lock_guard = _decommission_object_lock_guard.take(); let decommission_capacity_guard = _decommission_capacity_guard.take(); @@ -3811,6 +3914,7 @@ impl SetDisks { guard_release_rx, ( object_lock_guard, + publication_guard, bucket_lifecycle_guard, decommission_object_lock_guard, decommission_capacity_guard, @@ -3830,12 +3934,14 @@ impl SetDisks { }, move |( object_lock_guard, + publication_guard, bucket_lifecycle_guard, decommission_object_lock_guard, decommission_capacity_guard, ), targets| async move { drop(object_lock_guard); + drop(publication_guard); drop(bucket_lifecycle_guard); cleanup_set .cleanup_rename_tail( @@ -3948,6 +4054,7 @@ impl SetDisks { // The exact old-data-dir reclamation below is best-effort space // cleanup; it must not serialize the next operation on this object. drop(_object_lock_guard.take()); + drop(_publication_guard.take()); drop(_bucket_lifecycle_guard.take()); rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed)); @@ -4839,12 +4946,15 @@ pub(crate) async fn cleanup_uncommitted_transition_upload( fn log_transition_upload_cleanup_failure(lease: &TierOperationLease, object: &str, cleanup_version: &str, err: &std::io::Error) { warn!( + event = EVENT_LIFECYCLE_TRANSITION_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, tier = lease.tier_name(), tier_generation = lease.generation(), object, remote_version = cleanup_version, error = ?err, - "failed to clean uncommitted transition upload" + "Transition upload cleanup failed" ); } @@ -4852,22 +4962,16 @@ pub(crate) struct TransitionUploadCleanup { lease: TierOperationLease, object: String, candidate: Option, - cleanup_ctx: Arc, cleanup_api: Option>, armed: bool, } impl TransitionUploadCleanup { - pub(crate) fn new( - lease: TierOperationLease, - object: &str, - cleanup_ctx: Arc, - ) -> Self { + pub(crate) fn new(lease: TierOperationLease, object: &str) -> Self { Self { lease, object: object.to_string(), candidate: None, - cleanup_ctx, cleanup_api: None, armed: true, } @@ -4900,10 +5004,17 @@ impl TransitionUploadCleanup { } } - async fn cleanup_rejected_upload(&mut self, api: Option>) -> std::io::Result<()> { + async fn cleanup_rejected_upload( + &mut self, + api: Option>, + transaction: &mut TransitionTransaction, + ) -> std::io::Result<()> { self.cleanup_api = api.clone(); - let candidate = self.cleanup_candidate()?; - let result = cleanup_rejected_transition_upload_durably( + let candidate = self.cleanup_candidate()?.clone(); + let owner_error = persist_rejected_transition_cleanup_owner(api.as_ref(), transaction, &candidate) + .await + .err(); + let cleanup = cleanup_rejected_transition_upload_durably( &self.lease, &self.object, candidate.cleanup_version(), @@ -4911,10 +5022,17 @@ impl TransitionUploadCleanup { api, ) .await; - if result.is_ok() { - self.armed = false; + match (owner_error, cleanup) { + (_, Ok(())) => { + self.armed = false; + Ok(()) + } + (None, Err(cleanup_error)) => Err(cleanup_error), + (Some(owner_error), Err(cleanup_error)) => Err(crate::error::stable_io_error( + "rejected transition upload cleanup error followed a transaction owner update failure", + format!("owner error: {owner_error}; cleanup error: {cleanup_error}"), + )), } - result } pub(crate) fn disarm(&mut self) { @@ -4934,11 +5052,14 @@ impl Drop for TransitionUploadCleanup { Ok(lease) => lease, Err(err) => { warn!( + event = EVENT_LIFECYCLE_TRANSITION_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, tier = self.lease.tier_name(), tier_generation = self.lease.generation(), object = self.object, error = ?err, - "unable to retain tier lease for cancelled transition cleanup" + "Cancelled transition cleanup could not retain its tier lease" ); return; } @@ -4947,23 +5068,22 @@ impl Drop for TransitionUploadCleanup { let cleanup_version = candidate.cleanup_version().to_string(); let version_id_exact = candidate.cleanup_version_is_exact(); let cleanup_api = self.cleanup_api.clone(); - let cleanup_ctx = self.cleanup_ctx.clone(); if let Ok(handle) = tokio::runtime::Handle::try_current() { handle.spawn(async move { - let api = match cleanup_api { - Some(api) => Some(api), - None => transition_cleanup_store(&cleanup_ctx).await, - }; if let Err(err) = - cleanup_rejected_transition_upload_durably(&lease, &object, &cleanup_version, version_id_exact, api).await + cleanup_rejected_transition_upload_durably(&lease, &object, &cleanup_version, version_id_exact, cleanup_api) + .await { warn!( + event = EVENT_LIFECYCLE_TRANSITION_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, tier = lease.tier_name(), tier_generation = lease.generation(), object, remote_version = cleanup_version, error = ?err, - "cancelled transition upload was neither deleted nor journaled" + "Cancelled transition upload remains transaction-owned for recovery" ); } }); @@ -4971,84 +5091,47 @@ impl Drop for TransitionUploadCleanup { } } +async fn persist_rejected_transition_cleanup_owner( + api: Option<&Arc>, + transaction: &mut TransitionTransaction, + candidate: &TransitionUploadCandidate, +) -> Result<()> { + match transaction.state { + TransitionTransactionState::UploadOutcomeUnknown => { + transaction + .advance( + transaction.fence(), + TransitionTransactionState::Uploaded, + Some(TransitionRemoteVersion::known_from_put_response(candidate.remote_version().to_string())), + ) + .map_err(Error::other)?; + } + TransitionTransactionState::Uploaded => {} + state => { + return Err(Error::other_with_context( + "transition transaction state cannot own a rejected upload", + format!("state {state:?}"), + )); + } + } + save_transition_transaction_if_available(api, transaction).await +} + pub(crate) async fn cleanup_rejected_transition_upload_durably( lease: &TierOperationLease, object: &str, cleanup_version: &str, version_id_exact: bool, - api: Option>, + _api: Option>, ) -> std::io::Result<()> { - let journal_entry = Jentry { - obj_name: object.to_string(), - version_id: cleanup_version.to_string(), - tier_name: lease.tier_name().to_string(), - backend_identity: Some(lease.backend_identity()), - version_id_exact, - version_state: if !version_id_exact { - rustfs_filemeta::TransitionVersionState::KnownDisabled - } else if cleanup_version == "null" { - rustfs_filemeta::TransitionVersionState::SuspendedNull - } else { - rustfs_filemeta::TransitionVersionState::Exact - }, - state: TierDeleteJournalState::Committed, - source: None, - }; - - let journal_error = if let Some(api) = api.as_ref() { - match persist_tier_delete_journal_entry(api.clone(), &journal_entry).await { - Ok(()) => { - match cleanup_uncommitted_transition_upload(lease, object, cleanup_version, version_id_exact).await { - Ok(_) => { - if let Err(err) = remove_tier_delete_journal_entry(api.clone(), &journal_entry).await { - warn!( - tier = lease.tier_name(), - object, - error = ?err, - "rejected transition upload was deleted but its cleanup journal was retained" - ); - } - } - Err(err) => log_transition_upload_cleanup_failure(lease, object, cleanup_version, &err), - } - return Ok(()); - } - Err(err) => err, - } - } else { - std::io::Error::other("object store unavailable for rejected transition cleanup journal") - }; - warn!( - tier = lease.tier_name(), - object, - error = ?journal_error, - "failed to persist rejected transition upload cleanup journal" - ); - - let cleanup_error = match cleanup_uncommitted_transition_upload(lease, object, cleanup_version, version_id_exact).await { - Ok(_) => return Ok(()), - Err(err) => { - log_transition_upload_cleanup_failure(lease, object, cleanup_version, &err); - err - } - }; - if let Some(api) = api { - match persist_tier_delete_journal_entry(api, &journal_entry).await { - Ok(()) => return Ok(()), - Err(retry_error) => { - return Err(std::io::Error::other(format!( - "rejected transition upload was neither deleted nor journaled: initial journal error: {journal_error}; cleanup error: {cleanup_error}; journal retry error: {retry_error}" - ))); - } - } - } - Err(std::io::Error::other(format!( - "rejected transition upload was neither deleted nor journaled: journal error: {journal_error}; cleanup error: {cleanup_error}" - ))) + cleanup_uncommitted_transition_upload(lease, object, cleanup_version, version_id_exact) + .await + .map(|_| ()) + .inspect_err(|err| log_transition_upload_cleanup_failure(lease, object, cleanup_version, err)) } async fn transition_cleanup_store(ctx: &Arc) -> Option> { - #[cfg(feature = "test-util")] + #[cfg(any(test, feature = "test-util"))] pause_transition_cleanup_store().await; transition_object_store(ctx).await @@ -5270,24 +5353,24 @@ async fn delete_transition_transaction_after_remote_cleanup( } } -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] #[derive(Default)] struct TransitionCleanupStoreBarrierState { arrived: tokio::sync::Notify, release: tokio::sync::Notify, } -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] /// One-shot test barrier placed before transition cleanup resolves its ECStore. pub(crate) struct TransitionCleanupStoreBarrier { state: Arc, } -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] impl TransitionCleanupStoreBarrier { /// Install the process-local barrier for the next cleanup-store resolution. pub(crate) fn install() -> Self { @@ -5310,7 +5393,7 @@ impl TransitionCleanupStoreBarrier { } } -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] impl Drop for TransitionCleanupStoreBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -5324,7 +5407,7 @@ impl Drop for TransitionCleanupStoreBarrier { } } -#[cfg(feature = "test-util")] +#[cfg(any(test, feature = "test-util"))] async fn pause_transition_cleanup_store() { let barrier = TRANSITION_CLEANUP_STORE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -5791,6 +5874,7 @@ impl DeleteObjectCommitBarrier { Self::install_with_mode(bucket, object, false) } + #[cfg(feature = "test-util")] pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self { Self::install_with_mode(bucket, object, true) } @@ -6247,6 +6331,86 @@ mod transition_version_id_tests { } impl SetDisks { + pub(crate) async fn replay_authorized_tier_delete_sources( + &self, + bucket: &str, + object: &str, + authorization: &TierDeleteDispatchAuthorization, + opts: &ObjectOptions, + ) -> Result { + let versions = match self.load_file_info_versions_exact(bucket, object).await { + Ok(Some(versions)) => versions, + Ok(None) => return Ok(0), + Err(err) if crate::error::is_err_strict_volume_not_found(&err) => return Ok(0), + Err(err) => return Err(err), + }; + let lock_state = opts + .object_lock_config_snapshot + .as_deref() + .ok_or_else(|| Error::other("authorized tier delete replay is missing its Object Lock snapshot"))? + .state(); + let bypass_governance = opts + .object_lock_delete + .as_ref() + .is_some_and(|delete_opts| delete_opts.bypass_governance); + let encoded_object = rustfs_utils::path::encode_dir_object(object); + let mut deleted = 0; + + for version in versions.versions { + let Some(candidate) = lifecycle_delete_all_tier_journal_entry(bucket, &encoded_object, &version, opts)? else { + continue; + }; + if !authorization.authorizes_journal_entry(&candidate)? { + continue; + } + let object_info = + ObjectInfo::from_file_info(&version, bucket, &encoded_object, opts.versioned || opts.version_suspended); + if check_object_lock_for_deletion_with_state(lock_state, &object_info, bypass_governance)?.is_some() { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); + } + let replication_delete = if opts.lifecycle_delete_all.is_some() { + lifecycle_delete_all_replication_delete(bucket, &encoded_object, &version, opts)? + } else { + None + }; + let mut delete_request = FileInfo { + name: encoded_object.clone(), + version_id: version.version_id, + replication_state_internal: replication_delete + .as_ref() + .map(|(state, _)| replication_state_to_filemeta(state)), + ..Default::default() + }; + delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string()); + delete_request.set_skip_tier_free_version(); + ensure_delete_commit_locks_held(None, bucket, &encoded_object, opts)?; + authorization.authorized_journal_name(&candidate)?; + begin_scanner_publication_delete_mutation(opts.scanner_publication_commit_scope.as_ref())?; + self.delete_object_version(bucket, &encoded_object, &delete_request, false) + .await?; + if let Some((_, deleted_object)) = replication_delete { + ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await; + } + deleted += 1; + } + + ensure_delete_commit_locks_held(None, bucket, &encoded_object, opts)?; + if let Some(remaining) = self.load_file_info_versions_exact(bucket, object).await? { + for version in remaining.versions { + let Some(candidate) = lifecycle_delete_all_tier_journal_entry(bucket, &encoded_object, &version, opts)? else { + continue; + }; + if authorization.authorizes_journal_entry(&candidate)? { + return Err(Error::other("authorized tier delete source remained after exact replay")); + } + } + } + if deleted > 0 { + self.invalidate_get_object_metadata_cache(bucket, &encoded_object).await; + } + Ok(deleted) + } + async fn update_object_tags_locked( &self, operation: &'static str, @@ -6812,7 +6976,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } }; let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new(); - let mut journal_entries: Vec<(usize, Jentry)> = Vec::new(); + let mut tier_reference_leases: Vec<(usize, String, Option)> = Vec::new(); + let mut transitioned_cleanup_items = vec![false; objects.len()]; for (i, dobj) in objects.iter().enumerate() { if del_errs[i].is_some() { @@ -6898,20 +7063,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } if opts.tier_delete_journal_api.is_some() - && let Some(mut je) = transitioned_delete_journal_entry_for_source( - version_id, - versioned, - version_suspended, - bucket, - &replication_object_name, - &goi, - ) + && !source_missing + && goi.transitioned_object.status == TRANSITION_COMPLETE + && !set_disk_delete_creates_delete_marker(&check_opts) { - if let Err(err) = record_tier_delete_journal_backend_identity(&mut je, &goi.user_defined) { - del_errs[i] = Some(Error::other(err)); - continue; + match tier_destination_id_from_metadata(&goi.user_defined) { + Ok(identity) => { + transitioned_cleanup_items[i] = true; + tier_reference_leases.push((i, goi.transitioned_object.tier.clone(), identity)); + } + Err(err) => { + del_errs[i] = Some(Error::other(err)); + continue; + } } - journal_entries.push((i, je)); } let mut admitted = dobj.clone(); @@ -7076,18 +7241,47 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return (del_objects, del_errs, accounting); } - let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len()); + let mut tier_leases: Vec = Vec::new(); if let Some(api) = opts.tier_delete_journal_api.as_ref() { - for (idx, mut je) in journal_entries { - if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &je).await { - del_errs[idx] = Some(Error::other(err)); - continue; + let mut reference_groups: HashMap<(String, Option), Vec> = HashMap::new(); + for (idx, tier_name, backend_identity) in tier_reference_leases { + reference_groups.entry((tier_name, backend_identity)).or_default().push(idx); + } + for ((tier_name, backend_identity), indices) in reference_groups { + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity( + &api.tier_config_mgr(), + &tier_name, + backend_identity, + ) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, + }; + match lease { + Ok(lease) => tier_leases.push(lease), + Err(err) => { + let message = err.to_string(); + for idx in indices { + del_errs[idx] = Some(Error::other(message.clone())); + } + } } - je.state = TierDeleteJournalState::Prepared; - persisted_journal_entries.push((idx, je)); } } + for (idx, transitioned) in transitioned_cleanup_items.into_iter().enumerate() { + if transitioned && del_errs[idx].is_none() { + record_transitioned_delete_cleanup_owner(bucket, &decode_dir_object(&objects[idx].object_name), true); + } + } + + // Keep backend generations pinned through the source mutation, its + // free-version write quorum, and any local rollback. Ordinary + // single/batch deletes never transfer cleanup ownership to a journal. + let _tier_leases = tier_leases; + for fi_vers in &mut vers { fi_vers.versions.retain(|fi| del_errs[fi.idx].is_none()); } @@ -7270,37 +7464,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // TODO(backlog): support partial object deletion for multi-part objects - if let Some(api) = opts.tier_delete_journal_api.as_ref() { - for (idx, je) in persisted_journal_entries { - if del_errs[idx].is_none() { - let mut committed = je; - committed.state = TierDeleteJournalState::Committed; - if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &committed).await { - warn!( - object = %committed.obj_name, - tier = %committed.tier_name, - error = ?err, - "batch tier delete committed locally but journal commit failed; recovery will retry" - ); - } else if let Err(err) = enqueue_committed_tier_delete_journal_entry(&committed).await { - warn!( - object = %committed.obj_name, - tier = %committed.tier_name, - error = ?err, - "batch tier delete journal committed but could not be queued; recovery will retry" - ); - } - } else if let Err(err) = remove_tier_delete_journal_entry(Arc::clone(api), &je).await { - warn!( - object = %je.obj_name, - tier = %je.tier_name, - error = ?err, - "failed to remove aborted batch tier delete journal" - ); - } - } - } - if dist_erasure { self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await; } @@ -7353,6 +7516,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; if opts.delete_prefix { + let destructive_dispatch_phase = opts.lifecycle_delete_all.as_ref().is_none_or(|request| { + matches!( + request.phase, + crate::object_api::LifecycleDeleteAllPhase::History | crate::object_api::LifecycleDeleteAllPhase::Trigger + ) + }); + if destructive_dispatch_phase && opts.tier_delete_journal_api.is_some() { + let authorization = opts + .tier_delete_dispatch_authorization + .as_ref() + .ok_or(StorageError::PreconditionFailed)?; + let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + authorization.ensure_current(bucket, incarnation, object)?; + if !authorization.mutation_started() { + return Err(StorageError::PreconditionFailed); + } + } if opts.delete_prefix_object && !is_meta_bucketname(bucket) { let object_lock_config = if opts.data_movement { None @@ -7384,7 +7564,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if let Some(trigger) = opts.lifecycle_delete_all.as_ref() { let plan = lifecycle_delete_all_plan(&versions, trigger)?; if trigger.phase == crate::object_api::LifecycleDeleteAllPhase::Preflight { - prepare_lifecycle_delete_all_tier_journals(bucket, object, &plan, &opts).await?; return Ok(ObjectInfo::default()); } if trigger.phase == crate::object_api::LifecycleDeleteAllPhase::FinalPreflight { @@ -7403,6 +7582,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { }; for version in plan { ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + // The lease closes the gap between the durable + // reference proof and publishing either a committed + // v6 journal or the legacy free-version fallback. + let _tier_lease = acquire_lifecycle_delete_all_tier_lease(version, &opts).await?; let replication_delete = lifecycle_delete_all_replication_delete(bucket, object, version, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), @@ -7413,15 +7596,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { ..Default::default() }; delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string()); - if opts.tier_delete_journal_api.is_some() - && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE + if version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE + && let Some(entry) = lifecycle_delete_all_tier_journal_entry(bucket, object, version, &opts)? { - let (name, _entry) = lifecycle_delete_all_tier_journal_entry(bucket, object, version, &opts)? + let authorization = opts + .tier_delete_dispatch_authorization + .as_ref() .ok_or(StorageError::PreconditionFailed)?; - let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?; - if !journal.lock().contains(&name) { - return Err(StorageError::PreconditionFailed); - } + let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + authorization.ensure_current(bucket, incarnation, object)?; + authorization.authorized_journal_name(&entry)?; delete_request.set_skip_tier_free_version(); } begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; @@ -7616,10 +7800,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { fi.set_tier_free_version_id(&find_vid.to_string()); - if opts.skip_free_version { - fi.set_skip_tier_free_version(); - } - fi.version_id = if let Some(vid) = opts.version_id.as_ref() { let vid = Uuid::parse_str(vid.as_str())?; (!opts.version_suspended || !vid.is_nil()).then_some(vid) @@ -7633,6 +7813,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; + let _tier_delete_lease = acquire_single_tier_delete_lease(bucket, object, &opts, &goi).await?; + if opts.skip_free_version { + fi.set_skip_tier_free_version(); + } self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts)) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -7669,12 +7853,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { dfi.set_tier_free_version_id(&find_vid.to_string()); + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; + let _tier_delete_lease = acquire_single_tier_delete_lease(bucket, object, &opts, &goi).await?; if opts.skip_free_version { dfi.set_skip_tier_free_version(); } - - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; - begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; self.delete_object_version(bucket, object, &dfi, opts.delete_marker) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -8044,7 +8228,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { Ok(writer.produced()) }; - let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, self.ctx.clone()); + let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj); advance_and_save_transition_transaction( transaction_api.as_ref(), &mut transaction, @@ -8070,7 +8254,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { Err(failure) => { if failure.candidate.is_some() { let cleanup_api = transition_cleanup_store(&self.ctx).await; - if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { + if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api, &mut transaction).await { return Err(StorageError::Io(std::io::Error::other(format!( "{}; rejected remote upload cleanup failed: {cleanup_err}", failure.error @@ -8085,7 +8269,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if let Err(err) = upload_cleanup.lease.validate_remote_version_id(candidate.remote_version()) { let cleanup_api = transition_cleanup_store(&self.ctx).await; - if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { + if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api, &mut transaction).await { return Err(StorageError::Io(std::io::Error::other(format!( "{err}; rejected remote upload cleanup failed: {cleanup_err}" )))); @@ -8101,7 +8285,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { Ok(version) => version, Err(err) => { let cleanup_api = transition_cleanup_store(&self.ctx).await; - if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { + if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api, &mut transaction).await { return Err(StorageError::Io(std::io::Error::other(format!( "{err}; rejected remote upload cleanup failed: {cleanup_err}" )))); @@ -8120,7 +8304,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { .await { let cleanup_api = transition_cleanup_store(&self.ctx).await; - if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { + if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api, &mut transaction).await { return Err(StorageError::Io(std::io::Error::other(format!( "{err}; uploaded transition transaction persist failed and cleanup failed: {cleanup_err}" )))); @@ -8399,6 +8583,25 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let actual_fi = actual.fi(); oi = ObjectInfo::from_file_info(actual_fi, bucket, object, opts.versioned || opts.version_suspended); + // The tier reader releases its own lease at EOF, before PUT or + // CompleteMultipartUpload necessarily reaches metadata quorum. Keep a + // second exact-generation lease for the whole restore so the final + // same-key write lock and this lease together fence remote-tuple + // publication through commit. + let expected_backend_identity = tier_destination_id_from_metadata(&oi.user_defined).map_err(Error::other)?; + let tier_config_mgr = self.ctx.tier_config_mgr(); + let _remote_tuple_publication_lease = match expected_backend_identity { + Some(identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity( + &tier_config_mgr, + &oi.transitioned_object.tier, + identity, + ) + .await + } + None => TierConfigMgr::acquire_operation_lease(&tier_config_mgr, &oi.transitioned_object.tier).await, + } + .map_err(Error::other)?; let expected_operation_id = restore_operation_id_from_metadata(&opts.user_defined)?; if let Some(expected_operation_id) = expected_operation_id { require_restore_operation_id(oi.user_defined.as_ref(), expected_operation_id)?; @@ -8434,8 +8637,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } // Keep the public ECStore capacity admission attached to each local // commit. The tier reads below must remain outside the object write - // lock so HEAD/GET do not wait for a slow remote copy-back. - ropts.no_lock = opts.no_lock; + // lock so HEAD/GET do not wait for a slow remote copy-back. Restore + // does not hold an outer object write lock while copying bytes, so a + // caller-supplied boolean is not transferable lock authority: PUT and + // Complete must acquire their own commit-late write locks. + ropts.no_lock = false; ropts.expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id; ropts.bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); ropts.namespace_lock_fence = opts.namespace_lock_fence.clone(); @@ -8577,7 +8783,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id, bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(), user_defined: restore_commit_metadata, - no_lock: opts.no_lock, + no_lock: false, decommission_capacity_admission: opts.decommission_capacity_admission.clone(), ..Default::default() }; @@ -11872,6 +12078,78 @@ mod transition_commit_failure_tests { assert_eq!(old_backend.object_count().await, 0); } + #[tokio::test] + #[serial_test::serial] + async fn restore_keeps_remote_tuple_lease_after_reader_eof_until_local_commit() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "restore-remote-tuple-publication-lease-bucket"; + let object = "object.bin"; + let payload = b"restore must retain its tier generation through metadata quorum".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(payload); + let original = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let manager = runtime_sources::global_tier_config_mgr(); + let _backend = register_mock_tier(&manager, &tier_name).await; + set_disks + .transition_object( + bucket, + object, + &ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("source object should transition before restore"); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier_name).await, + 0, + "transition should release its operation lease before restore" + ); + + // BeforeNamespace is reached after the restore reader has drained but + // before the destination metadata write lock and quorum rename. + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let restore_set = Arc::clone(&set_disks); + let restore = tokio::spawn(async move { + let mut opts = ObjectOptions::default(); + opts.transition.restore_request.days = Some(1); + restore_set.restore_transitioned_object(bucket, object, &opts).await + }); + barrier.wait_until_paused().await; + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier_name).await, + 1, + "the publication lease must outlive the EOF-scoped tier reader lease" + ); + + barrier.release(); + restore + .await + .expect("restore task should join") + .expect("restore should commit after the barrier releases"); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier_name).await, + 0, + "restore completion should release the publication lease" + ); + } + #[tokio::test] #[serial_test::serial] async fn failed_restore_cleanup_does_not_overwrite_concurrent_unversioned_put() { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index fcda7ffbc..cc67c4964 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -29,6 +29,25 @@ use std::future::Future; const DELETED_BUCKETS_PREFIX: &str = ".deleted"; const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4; +const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked"; + +fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) { + metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1); + debug!( + event = EVENT_BUCKET_DELETE_BLOCKED, + component = "ecstore", + subsystem = "bucket", + bucket, + blocker = kind.as_str(), + files = residue.files, + uuid_data_dirs = residue.uuid_data_dirs, + entries_scanned = residue.entries_scanned, + diagnostic_bytes_read = residue.diagnostic_bytes_read, + diagnostic_truncated = residue.diagnostic_truncated, + sample = residue.sample.as_deref().unwrap_or(""), + "Bucket deletion was blocked by durable local state" + ); +} fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize { set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY) @@ -156,6 +175,7 @@ where async fn bucket_delete_local_blocker( ctx: &crate::runtime::instance::InstanceContext, bucket: &str, + budget: &mut BucketDeleteDiagnosticBudget, ) -> Result> { let local_disks = runtime_sources::local_disks_in(ctx).await; let mut residue = BucketMetadataLessResidue::default(); @@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker( let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else { continue; }; - let scan = scan_metadata_less_residue(&bucket_path?).await?; + let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?; if scan.xlmeta_found { + record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan); return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string()))); } residue.files = residue.files.saturating_add(scan.files); residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs); + residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned); + residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read); if residue.sample.is_none() { - residue.sample = scan.sample; + residue.sample = scan.sample.clone(); + } + if scan.diagnostic_truncated { + residue.diagnostic_truncated = true; + record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue); + return Ok(Some(StorageError::BucketNotEmptyWithDetails { + bucket: bucket.to_string(), + details: residue.describe(), + })); } } if residue.has_residue_without_xlmeta() { + record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue); return Ok(Some(StorageError::BucketNotEmptyWithDetails { bucket: bucket.to_string(), details: residue.describe(), @@ -783,12 +815,14 @@ impl ECStore { } } }; + let mut diagnostic_budget = None; if bucket_exists { validate_table_bucket_delete_guard(&self.ctx, bucket).await?; if !opts.force { - if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? { + let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new); + if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? { return Err(blocker); } delete_opts.force_if_empty = true; @@ -827,7 +861,12 @@ impl ECStore { { if delete_opts.force_if_empty && matches!(&err, StorageError::BucketNotEmpty(_)) - && let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? + && let Some(blocker) = bucket_delete_local_blocker( + &self.ctx, + bucket, + diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new), + ) + .await? { return Err(blocker); } @@ -856,15 +895,17 @@ impl ECStore { #[cfg(test)] mod tests { use super::{ - SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, - bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion, - scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata, + BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind, + BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, + bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume, + run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue, + scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata, validate_table_bucket_delete_allowed, }; use crate::bucket::metadata::table_bucket_catalog_metadata_prefix; use crate::bucket::metadata_sys; use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier; - use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; + use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}; use crate::error::StorageError; use crate::object_api::{ObjectOptions, PutObjReader}; use crate::runtime::instance::InstanceContext; @@ -879,6 +920,7 @@ mod tests { layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, }; use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo}; + use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE}; use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; use serial_test::serial; use std::path::{Path, PathBuf}; @@ -892,6 +934,88 @@ mod tests { static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec, Arc)> = OnceCell::const_new(); + #[tokio::test(start_paused = true)] + async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() { + let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100)); + tokio::time::advance(Duration::from_secs(10)).await; + + let first_polled = Arc::new(AtomicBool::new(false)); + let first_polled_for_io = first_polled.clone(); + let first = budget + .run_io(async move { + first_polled_for_io.store(true, Ordering::SeqCst); + Ok::<_, std::io::Error>(7_u8) + }) + .await + .expect("the first diagnostic IO should succeed"); + assert_eq!(first, Some(7)); + assert!(first_polled.load(Ordering::SeqCst)); + + tokio::time::advance(Duration::from_millis(101)).await; + let expired_polled = Arc::new(AtomicBool::new(false)); + let expired_polled_for_io = expired_polled.clone(); + let expired = budget + .run_io(async move { + expired_polled_for_io.store(true, Ordering::SeqCst); + Ok::<_, std::io::Error>(9_u8) + }) + .await + .expect("an expired diagnostic budget should not become an IO error"); + assert_eq!(expired, None); + assert!( + !expired_polled.load(Ordering::SeqCst), + "the deadline must remain latched after the first scan IO" + ); + } + + #[tokio::test(start_paused = true)] + async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() { + let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100)); + let io_polled = Arc::new(AtomicBool::new(false)); + let io_polled_for_future = io_polled.clone(); + + let result = budget + .run_io(std::future::poll_fn(move |_cx| { + io_polled_for_future.store(true, Ordering::SeqCst); + std::task::Poll::>::Pending + })) + .await + .expect("a diagnostic timeout should fail closed without an IO error"); + + assert_eq!(result, None); + assert!( + io_polled.load(Ordering::SeqCst), + "the first diagnostic IO must be polled before its timeout" + ); + } + + #[tokio::test(start_paused = true)] + async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() { + let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created"); + let orphan_root = root.path().join("orphan-root"); + let xlmeta_root = root.path().join("xlmeta-root"); + std::fs::create_dir_all(&orphan_root).expect("orphan root should be created"); + std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created"); + std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written"); + std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written"); + + let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5)); + let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5)); + tokio::time::advance(Duration::from_secs(60)).await; + + let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget) + .await + .expect("delayed orphan scan should complete"); + assert!(orphan.has_residue_without_xlmeta()); + assert!(!orphan.diagnostic_truncated); + + let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget) + .await + .expect("delayed xl.meta scan should complete"); + assert!(xlmeta.xlmeta_found); + assert!(!xlmeta.diagnostic_truncated); + } + #[tokio::test(start_paused = true)] async fn bucket_namespace_operation_fails_closed_after_lease_expiry() { let ttl = Duration::from_millis(20); @@ -1313,6 +1437,175 @@ mod tests { assert!(sample.ends_with("/part.1")); } + #[tokio::test] + async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() { + let root = tempfile::tempdir().expect("temporary diagnostic roots should be created"); + let first_root = root.path().join("disk-a"); + let second_root = root.path().join("disk-b"); + tokio::fs::create_dir_all(&first_root) + .await + .expect("first diagnostic root should be created"); + tokio::fs::create_dir_all(&second_root) + .await + .expect("second diagnostic root should be created"); + for index in 0..3 { + std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written"); + std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written"); + } + + let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5)); + let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget) + .await + .expect("first root should fit the shared budget"); + assert!(!first.diagnostic_truncated); + let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget) + .await + .expect("second root should stop at the remaining shared budget"); + assert!(second.diagnostic_truncated); + assert!(first.entries_scanned + second.entries_scanned <= 4); + } + + #[tokio::test] + async fn metadata_less_residue_scan_honors_an_expired_request_deadline() { + let root = tempfile::tempdir().expect("temporary diagnostic root should be created"); + std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written"); + let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO); + + let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget) + .await + .expect("an expired diagnostic budget should fail closed without an IO error"); + + assert!(scan.diagnostic_truncated); + assert_eq!(scan.entries_scanned, 0); + assert_eq!(scan.diagnostic_bytes_read, 0); + } + + #[tokio::test] + async fn metadata_less_residue_scan_stops_at_diagnostic_budget() { + let root = tempfile::tempdir().expect("temporary bucket root should be created"); + let bucket_path = root.path().join("bucket"); + tokio::fs::create_dir_all(&bucket_path) + .await + .expect("budget fixture directory should be created"); + for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) { + std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created"); + } + + let residue = scan_metadata_less_residue(&bucket_path) + .await + .expect("budgeted residue scan should fail closed without an IO error"); + assert!(residue.diagnostic_truncated); + assert!(residue.has_residue_without_xlmeta()); + assert!(!residue.xlmeta_found); + assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES); + assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES); + assert_eq!(residue.diagnostic_bytes_read, 0); + } + + #[tokio::test] + async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() { + let root = tempfile::tempdir().expect("temporary bucket root should be created"); + let bucket_path = root.path().join("bucket"); + let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent")) + .await + .expect("visible object directory should be created"); + let mut visible = FileMeta::new(); + visible + .add_version(FileInfo { + version_id: Some(Uuid::new_v4()), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }) + .expect("visible version should encode"); + tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal")) + .await + .expect("visible xl.meta should be written"); + + let visible_scan = scan_metadata_less_residue(&bucket_path) + .await + .expect("visible xl.meta scan should succeed"); + assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion)); + + tokio::fs::remove_dir_all(bucket_path.join("visible")) + .await + .expect("visible fixture should be removed"); + let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent")) + .await + .expect("free-version object directory should be created"); + let source_version_id = Uuid::new_v4(); + let mut free = FileMeta::new(); + free.add_version(FileInfo { + version_id: Some(source_version_id), + transition_status: TRANSITION_COMPLETE.to_string(), + transitioned_objname: "remote/object".to_string(), + transition_version_id: Some(Uuid::new_v4()), + transition_tier: "WARM".to_string(), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }) + .expect("transitioned source should encode"); + let mut delete = FileInfo { + version_id: Some(source_version_id), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + delete.set_tier_free_version_id(&Uuid::new_v4().to_string()); + free.delete_version(&delete) + .expect("transitioned source delete should create a free-version"); + tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal")) + .await + .expect("free-version xl.meta should be written"); + + let free_scan = scan_metadata_less_residue(&bucket_path) + .await + .expect("free-version xl.meta scan should succeed"); + assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion)); + + tokio::fs::remove_dir_all(bucket_path.join("free")) + .await + .expect("free-version fixture should be removed"); + + let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent")) + .await + .expect("exact-limit object directory should be created"); + let exact_limit = tokio::fs::File::create(&exact_limit_path) + .await + .expect("exact-limit xl.meta should be created"); + exact_limit + .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES) + .await + .expect("exact-limit xl.meta should be extended without allocating its contents"); + let exact_limit_scan = scan_metadata_less_residue(&bucket_path) + .await + .expect("exact-limit xl.meta scan should remain fail closed"); + assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta)); + assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES); + tokio::fs::remove_dir_all(bucket_path.join("exact-limit")) + .await + .expect("exact-limit fixture should be removed"); + + let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent")) + .await + .expect("oversized object directory should be created"); + let oversized = tokio::fs::File::create(&oversized_path) + .await + .expect("oversized xl.meta should be created"); + oversized + .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1) + .await + .expect("oversized xl.meta should be extended without allocating its contents"); + let oversized_scan = scan_metadata_less_residue(&bucket_path) + .await + .expect("oversized xl.meta scan should remain fail closed"); + assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta)); + assert_eq!(oversized_scan.diagnostic_bytes_read, 0); + assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES); + } + #[tokio::test] #[serial] async fn scanner_bucket_listing_unions_every_erasure_set() { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index fb72ac5dc..274403549 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -379,6 +379,15 @@ impl ECStore { endpoint_pools: EndpointServerPools, ctx: CancellationToken, instance_ctx: Arc, + ) -> Result> { + Box::pin(Self::new_with_instance_ctx_inner(address, endpoint_pools, ctx, instance_ctx)).await + } + + async fn new_with_instance_ctx_inner( + address: SocketAddr, + endpoint_pools: EndpointServerPools, + ctx: CancellationToken, + instance_ctx: Arc, ) -> Result> { instance_ctx.bind_background_cancel_token(ctx.clone()); @@ -788,7 +797,7 @@ mod tests { use crate::{ bucket::lifecycle::{ DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, - bucket_lifecycle_ops::{ManualTransitionRunOptions, recover_manual_transition_jobs_once}, + bucket_lifecycle_ops::{ExpiryState, ManualTransitionRunOptions, recover_manual_transition_jobs_once}, lifecycle::{TRANSITION_PENDING, TransitionOptions}, manual_transition_job::{ ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord, @@ -797,9 +806,18 @@ mod tests { manual_transition_worker_result_object_name, manual_transition_worker_result_task_key, }, tier_delete_journal::{ - TIER_DELETE_JOURNAL_PREFIX, encode_tier_delete_journal_entry, persist_tier_delete_journal_entry, - recover_tier_delete_journal_entries, tier_delete_journal_object_name, + DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, + TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage, + TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry, + install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch, + recover_test_tier_delete_dispatch_manifest, recover_test_tier_delete_dispatch_manifest_with_page_budget, + recover_tier_delete_dispatch_manifests, recover_tier_delete_journal_entries, + test_tier_delete_dispatch_manifest_checkpoint, test_tier_delete_dispatch_manifest_state, + tier_delete_dispatch_manifest_operation_lock_held_for_test, + tier_delete_dispatch_manifest_recovery_count_for_test, tier_delete_dispatch_manifest_recovery_inflight_for_test, + tier_delete_journal_object_name, }, + tier_free_version_recovery::recover_tier_free_versions, tier_sweeper::{ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source, }, @@ -820,9 +838,13 @@ mod tests { data_movement::SourceCleanupDeleteBarrier, disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}, runtime::{global::set_object_store_resolver, sources as runtime_sources}, + services::notification_sys::acquire_tier_delete_journal_fleet_proof, services::tier::{ test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier}, - tier::{TIER_CONFIG_FILE, TierConfigMgr, tier_config_candidate_digest}, + tier::{ + ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, TIER_CONFIG_FILE, TierConfigMgr, + ensure_no_authoritative_tier_references_for_test, tier_config_candidate_digest, + }, tier_config::{TierConfig, TierType, TierWasabi}, tier_mutation_intent::{ TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState, @@ -835,6 +857,8 @@ mod tests { }, storage_api_contracts::list::ListOperations as _, }; + #[cfg(feature = "test-util")] + use crate::{bucket::replication::ReplicationObjectBridge, storage_api_contracts::bucket::DeleteBucketOptions}; use crate::{ bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map}, core::pools::{ @@ -861,7 +885,7 @@ mod tests { use rustfs_config::server_config::KVS; #[cfg(feature = "test-util")] use rustfs_filemeta::{FileInfo, FileMeta}; - use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry, ObjectPartInfo}; + use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry}; #[cfg(feature = "test-util")] use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; use rustfs_rio::{Checksum, ChecksumType}; @@ -886,6 +910,22 @@ mod tests { use tokio_util::sync::CancellationToken; use uuid::Uuid; + #[test] + fn new_with_instance_ctx_future_remains_stack_bounded() { + let future = crate::store::ECStore::new_with_instance_ctx( + std::net::SocketAddr::from(([127, 0, 0, 1], 0)), + EndpointServerPools(Vec::new()), + CancellationToken::new(), + Arc::new(crate::runtime::instance::InstanceContext::new()), + ); + let future_size = std::mem::size_of_val(&future); + + assert!( + future_size <= 4 * 1024, + "ECStore constructor future must remain stack-bounded; measured {future_size} bytes" + ); + } + fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec { meta.encode_config_data_for_test().expect("pool metadata should encode") } @@ -2020,6 +2060,7 @@ mod tests { }); } let endpoint_pools = EndpointServerPools(pools); + crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test(); let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) @@ -2050,6 +2091,375 @@ mod tests { (instance_ctx, store, shutdown) } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn cross_key_materialized_copy_does_not_publish_remote_tuple_ownership() { + let temp_dir = tempfile::tempdir().expect("create materialized-copy store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "copy-materialized", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + + let bucket = format!("copy-materialized-{}", Uuid::new_v4()); + let source_object = "source-restored.bin"; + let target_object = "target-local.bin"; + let payload = b"materialized copy must own only its new local bytes".to_vec(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("copy test bucket should be created"); + + let tier_name = "COPY-MATERIALIZED-TIER"; + register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("mock tier generation should be available"); + let backend_identity = lease.backend_identity(); + drop(lease); + + let mut source_metadata = HashMap::from([ + ("content-type".to_string(), "application/octet-stream".to_string()), + ( + "x-amz-restore".to_string(), + "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(), + ), + ]); + for (suffix, value) in [ + ( + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(), + ), + ( + rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, + "remote/source-restored.bin".to_string(), + ), + (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()), + (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-source-version".to_string()), + ( + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(), + ), + ( + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(backend_identity), + ), + (rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, Uuid::new_v4().to_string()), + ] { + rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value); + } + + let mut source_body = PutObjReader::from_vec(payload.clone()); + store + .put_object( + &bucket, + source_object, + &mut source_body, + &ObjectOptions { + user_defined: source_metadata, + ..Default::default() + }, + ) + .await + .expect("restored transitioned source should be written"); + + let source_opts = ObjectOptions::default(); + let mut source_info = store + .get_object_info(&bucket, source_object, &source_opts) + .await + .expect("source metadata should be readable"); + assert_eq!( + source_info.transitioned_object.status, + crate::bucket::lifecycle::core::TRANSITION_COMPLETE + ); + assert_eq!(source_info.transitioned_object.tier, tier_name); + assert!(source_info.data_dir.is_some(), "restored source must retain local data"); + source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone())); + + let copied = store + .copy_object( + &bucket, + source_object, + &bucket, + target_object, + &mut source_info, + &source_opts, + &ObjectOptions::default(), + ) + .await + .expect("cross-key materialized copy should succeed"); + assert!(copied.transitioned_object.status.is_empty()); + assert!(copied.transitioned_object.name.is_empty()); + assert!(copied.transitioned_object.version_id.is_empty()); + assert!(copied.transitioned_object.tier.is_empty()); + assert!(!copied.transitioned_object.free_version); + for suffix in [ + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_utils::http::SUFFIX_TRANSITION_TIER, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, + ] { + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str(copied.user_defined.as_ref(), suffix), + "target retained protected source suffix {suffix}" + ); + } + assert_eq!( + copied.user_defined.get("content-type").map(String::as_str), + Some("application/octet-stream") + ); + + let mut target_reader = store + .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("target object should be readable"); + let mut target_body = Vec::new(); + target_reader + .stream + .read_to_end(&mut target_body) + .await + .expect("target body should stream"); + assert_eq!(target_body, payload); + + let source_after = store + .get_object_info(&bucket, source_object, &source_opts) + .await + .expect("source metadata should remain readable after copy"); + assert_eq!( + source_after.transitioned_object.status, + crate::bucket::lifecycle::core::TRANSITION_COMPLETE + ); + assert_eq!(source_after.transitioned_object.tier, tier_name); + assert_eq!(source_after.transitioned_object.name, "remote/source-restored.bin"); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() { + let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + + let bucket = format!("mpu-staging-publication-{}", Uuid::new_v4()); + let object = "remote-owned.bin"; + let payload = b"multipart part staging must not publish an object".to_vec(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("multipart staging test bucket should be created"); + + let tier_name = "MPU-STAGING-TIER"; + register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("mock tier generation should be available"); + let backend_identity = lease.backend_identity(); + drop(lease); + + let mut source_metadata = HashMap::from([( + "x-amz-restore".to_string(), + "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(), + )]); + for (suffix, value) in [ + ( + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + crate::bucket::lifecycle::core::TRANSITION_COMPLETE.to_string(), + ), + (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/mpu.bin".to_string()), + (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()), + (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-mpu-version".to_string()), + ( + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(), + ), + ( + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(backend_identity), + ), + ] { + rustfs_utils::http::metadata_compat::insert_str(&mut source_metadata, suffix, value); + } + let mut source_body = PutObjReader::from_vec(payload.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_body, + &ObjectOptions { + user_defined: source_metadata, + ..Default::default() + }, + ) + .await + .expect("remote-owned source should be written to the source pool"); + let source = store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("remote-owned source metadata should be readable"); + let publication = store + .acquire_remote_tuple_publication_fence(&bucket, 0, &source, true) + .await + .expect("multipart migration should capture its publication capability"); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, + 0, + "capturing the commit-late capability must not pin the tier generation" + ); + + let bucket_incarnation_id = store + .bucket_incarnation_id(&bucket) + .await + .expect("bucket incarnation should resolve"); + mark_test_pool_decommissioning(&store, 0).await; + let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0) + .await + .with_mutation_id(Uuid::new_v4()); + let mut upload_metadata = source.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str( + &mut upload_metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "mpu-staging-test".to_string(), + ); + let mut staging_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + user_defined: upload_metadata, + expected_bucket_incarnation_id: Some(bucket_incarnation_id), + ..Default::default() + }; + capacity_owner.apply_to(&mut staging_opts); + let (upload, target_pool_idx, staged_incarnation_id) = store + .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None) + .await + .expect("target multipart upload should be staged"); + assert_eq!(target_pool_idx, 1, "the active pool must receive the staged upload"); + assert_eq!(staged_incarnation_id, Some(bucket_incarnation_id)); + + let barrier = crate::set_disk::MultipartCommitBarrier::install( + &bucket, + object, + crate::set_disk::MultipartCommitPause::PutPartBeforeLockLost, + ); + let part_store = Arc::clone(&store); + let part_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let part_payload = payload.clone(); + let part = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(part_payload); + let mut part_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + part_number: Some(1), + expected_bucket_incarnation_id: Some(bucket_incarnation_id), + ..Default::default() + }; + capacity_owner.apply_to(&mut part_opts); + part_store + .put_object_part_for_data_movement(1, &part_bucket, object, &upload_id, &mut reader, &part_opts) + .await + }); + barrier.wait_until_paused().await; + + assert_eq!( + TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, + 0, + "UploadPart staging must not pin the tier generation" + ); + let encoded = rustfs_utils::path::encode_dir_object(object); + let mut proof_opts = ObjectOptions::default(); + let proof = tokio::time::timeout( + Duration::from_millis(250), + store.acquire_all_physical_object_read_locks("tier_delete_journal_recovery", &bucket, &encoded, &mut proof_opts), + ) + .await + .expect("UploadPart staging must not hold any object publication write lock") + .expect("the all-physical recovery proof should acquire every object namespace"); + let source_head = store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("source HEAD should remain available while the part commit is paused"); + assert_eq!(source_head.etag, source.etag); + assert!( + store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "UploadPart staging must not publish target object metadata" + ); + drop(proof); + + barrier.release(); + drop(barrier); + part.await + .expect("UploadPart staging task should join") + .expect("UploadPart staging should commit after the observation releases"); + let mut abort_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(bucket_incarnation_id), + ..Default::default() + }; + capacity_owner.apply_to(&mut abort_opts); + store + .abort_multipart_upload_for_data_movement(1, &bucket, object, &upload.upload_id, &abort_opts) + .await + .expect("staged target upload should be removable without publishing"); + drop(publication); + assert_eq!(TierConfigMgr::active_operation_lease_count(&ctx.tier_config_mgr(), tier_name).await, 0); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + struct OfflineTestDisks { + disks: Vec, + } + + #[cfg(feature = "test-util")] + impl Drop for OfflineTestDisks { + fn drop(&mut self) { + for disk in &self.disks { + disk.reset_health_for_store_init_retry(); + } + } + } + + #[cfg(feature = "test-util")] + async fn force_set_disks_offline_for_test(set: &Arc) -> OfflineTestDisks { + let disks = set + .disks + .read() + .await + .iter() + .map(|disk| disk.clone().expect("fault-injection set should start fully online")) + .collect::>(); + for disk in &disks { + disk.close().await.expect("fault injection should stop per-disk monitoring"); + disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline); + } + + // Sets has an independent endpoint monitor that renews missing slots. + // Keep each slot populated by the original, deliberately faulty handle + // and prove an explicit reconnect pass cannot heal this test fault. + set.connect_disks().await; + let current = set.disks.read().await.clone(); + assert_eq!(current.len(), disks.len()); + for (slot, expected) in current.iter().zip(&disks) { + let disk = slot.as_ref().expect("offline test slot must remain populated"); + assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle"); + assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline); + assert!( + disk.is_online().await, + "the valid disk identity must prevent endpoint renewal while the IO health gate remains offline" + ); + } + OfflineTestDisks { disks } + } + fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta { let now = OffsetDateTime::now_utc(); let mut pool_stats = vec![RebalanceStats::default(); pool_count]; @@ -2204,6 +2614,52 @@ mod tests { .expect("test decommission capacity reservation should activate"); } + #[cfg(feature = "test-util")] + async fn mark_test_pool_decommissioning_with_split_targets(store: &Arc, pool_idx: usize) { + let layout = DecommissionErasureLayout { data: 2, parity: 2 }; + let source_physical_bytes = 1024 * 1024 * 1024; + let capacity = store + .pools + .iter() + .enumerate() + .map(|(index, _)| { + if index == pool_idx { + DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes) + } else { + // The reservation peak is twice the predicted target size. + // Give each target only half so the real allocator must + // authorize both receipt-bearing pools. + DecommissionPoolCapacityInfo::for_test(index, layout, source_physical_bytes, source_physical_bytes, 0) + } + }) + .collect::>(); + set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect()); + store + .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new()) + .await + .expect("split-target decommission capacity reservation should activate"); + } + + #[cfg(feature = "test-util")] + async fn test_decommission_capacity_owner( + store: &crate::store::ECStore, + source_pool_index: usize, + ) -> crate::core::pools::DecommissionCapacityOwner { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[source_pool_index] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("test decommission capacity reservation should exist"); + crate::core::pools::DecommissionCapacityOwner { + source_pool_index, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + } + } + const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy"; const DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT: &str = "migrate_object"; #[cfg(feature = "test-util")] @@ -2386,6 +2842,84 @@ mod tests { .expect("complete decommission multipart source object"); } + struct DataMovementSourceSeed { + pool_idx: usize, + version_id: Option, + body: Vec, + multipart_split: Option, + mod_time: OffsetDateTime, + user_defined: HashMap, + } + + async fn seed_data_movement_source_reader( + store: &Arc, + bucket: &str, + object: &str, + seed: DataMovementSourceSeed, + ) -> GetObjectReader { + let DataMovementSourceSeed { + pool_idx, + version_id, + body, + multipart_split, + mod_time, + user_defined, + } = seed; + let pool = &store.pools[pool_idx]; + let write_opts = ObjectOptions { + versioned: version_id.is_some(), + version_id: version_id.map(|version_id| version_id.to_string()), + mod_time: Some(mod_time), + user_defined, + ..Default::default() + }; + if let Some(split) = multipart_split { + assert!(split > 0 && split < body.len(), "multipart source split must create two non-empty parts"); + let upload = pool + .new_multipart_upload(bucket, object, &write_opts) + .await + .expect("create physical data-movement source upload"); + let mut completed_parts = Vec::with_capacity(2); + for (part_number, bytes) in [(1, &body[..split]), (2, &body[split..])] { + let mut reader = PutObjReader::from_vec(bytes.to_vec()); + let part = pool + .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default()) + .await + .expect("write physical data-movement source part"); + completed_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + pool.clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &write_opts) + .await + .expect("complete physical data-movement source object"); + } else { + let mut reader = PutObjReader::from_vec(body); + pool.put_object(bucket, object, &mut reader, &write_opts) + .await + .expect("write physical data-movement source object"); + } + + pool.get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: version_id.is_some(), + version_id: version_id.map(|version_id| version_id.to_string()), + raw_data_movement_read: true, + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("read the physical data-movement source snapshot") + } + async fn assert_pool_object_present(pool: &Arc, bucket: &str, object: &str) { pool.get_object_info(bucket, object, &ObjectOptions::default()) .await @@ -2896,7 +3430,7 @@ mod tests { .make_bucket(&bucket, &MakeBucketOptions::default()) .await .expect("create data movement bucket"); - let source_mod_time = OffsetDateTime::UNIX_EPOCH; + let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; let target_mod_time = source_mod_time + time::Duration::SECOND; let object = "single-object"; @@ -2916,24 +3450,25 @@ mod tests { .expect("write newer single-part target"); let source_body = b"stale migration body".to_vec(); + let source_reader = seed_data_movement_source_reader( + &store, + &bucket, + object, + DataMovementSourceSeed { + pool_idx: 0, + version_id: None, + body: source_body, + multipart_split: None, + mod_time: source_mod_time, + user_defined: HashMap::new(), + }, + ) + .await; crate::data_movement::migrate_object( store.clone(), 0, bucket.clone(), - GetObjectReader { - stream: Box::new(Cursor::new(source_body.clone())), - object_info: ObjectInfo { - bucket: bucket.clone(), - name: object.to_string(), - size: i64::try_from(source_body.len()).expect("single source size should fit i64"), - actual_size: i64::try_from(source_body.len()).expect("single source size should fit i64"), - etag: Some("0123456789abcdef0123456789abcdef".to_string()), - mod_time: Some(source_mod_time), - ..Default::default() - }, - buffered_body: None, - body_source: Default::default(), - }, + source_reader, None, "test_data_movement", ) @@ -2967,42 +3502,25 @@ mod tests { let first_part_size = 5 * 1024 * 1024; let mut multipart_source_body = vec![b'a'; first_part_size]; multipart_source_body.push(b'b'); - let multipart_source_size = - i64::try_from(multipart_source_body.len()).expect("multipart source size should fit i64"); + let multipart_source_reader = seed_data_movement_source_reader( + &store, + &bucket, + multipart_object, + DataMovementSourceSeed { + pool_idx: 0, + version_id: None, + body: multipart_source_body, + multipart_split: Some(first_part_size), + mod_time: source_mod_time, + user_defined: HashMap::new(), + }, + ) + .await; crate::data_movement::migrate_object( store.clone(), 0, bucket.clone(), - GetObjectReader { - stream: Box::new(Cursor::new(multipart_source_body)), - object_info: ObjectInfo { - bucket: bucket.clone(), - name: multipart_object.to_string(), - size: multipart_source_size, - actual_size: multipart_source_size, - etag: Some("source-multipart-etag-2".to_string()), - mod_time: Some(source_mod_time), - parts: Arc::new(vec![ - ObjectPartInfo { - number: 1, - size: first_part_size, - actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), - etag: "source-part-1".to_string(), - ..Default::default() - }, - ObjectPartInfo { - number: 2, - size: 1, - actual_size: 1, - etag: "source-part-2".to_string(), - ..Default::default() - }, - ]), - ..Default::default() - }, - buffered_body: None, - body_source: Default::default(), - }, + multipart_source_reader, None, "test_data_movement", ) @@ -3538,40 +4056,20 @@ mod tests { let mut new_body = vec![b'c'; first_part_size]; new_body.push(b'd'); - let source_reader = |name: &str, body: Vec, mod_time, metadata: HashMap| { - let size = i64::try_from(body.len()).expect("source body size should fit i64"); - GetObjectReader { - stream: Box::new(Cursor::new(body)), - object_info: ObjectInfo { - bucket: bucket.clone(), - name: name.to_string(), + let source_reader = |name: &'static str, body: Vec, mod_time, metadata: HashMap| { + seed_data_movement_source_reader( + &store, + &bucket, + name, + DataMovementSourceSeed { + pool_idx: 0, version_id: Some(version_id), - size, - actual_size: size, - etag: Some(format!("{name}-multipart-etag-2")), - mod_time: Some(mod_time), - user_defined: Arc::new(metadata), - parts: Arc::new(vec![ - ObjectPartInfo { - number: 1, - size: first_part_size, - actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), - etag: format!("{name}-part-1"), - ..Default::default() - }, - ObjectPartInfo { - number: 2, - size: 1, - actual_size: 1, - etag: format!("{name}-part-2"), - ..Default::default() - }, - ]), - ..Default::default() + body, + multipart_split: Some(first_part_size), + mod_time, + user_defined: metadata, }, - buffered_body: None, - body_source: Default::default(), - } + ) }; let replaceable = "replaceable.bin"; @@ -3584,7 +4082,8 @@ mod tests { old_body.clone(), old_time, HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), - ), + ) + .await, None, "test_stale_target_seed", ) @@ -3623,7 +4122,8 @@ mod tests { new_body.clone(), new_time, HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), - ), + ) + .await, None, "test_stale_target_replace", ) @@ -3688,7 +4188,7 @@ mod tests { store.clone(), 0, bucket.clone(), - source_reader(client_target, new_body.clone(), new_time, HashMap::new()), + source_reader(client_target, new_body.clone(), new_time, HashMap::new()).await, None, "test_client_target_reject", ) @@ -3727,7 +4227,8 @@ mod tests { old_body.clone(), old_time, HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), - ), + ) + .await, None, "test_acknowledged_target_seed", ) @@ -3765,7 +4266,8 @@ mod tests { new_body.clone(), new_time, HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), - ), + ) + .await, None, "test_acknowledged_target_reject", ) @@ -3800,7 +4302,8 @@ mod tests { old_body.clone(), old_time, HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), - ), + ) + .await, None, "test_metadata_acknowledged_target_seed", ) @@ -3854,7 +4357,8 @@ mod tests { new_body.clone(), new_time, HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), - ), + ) + .await, None, "test_metadata_acknowledged_target_reject", ) @@ -3880,7 +4384,7 @@ mod tests { store.clone(), 0, bucket.clone(), - source_reader(object, old_body.clone(), old_time, retained_metadata.clone()), + source_reader(object, old_body.clone(), old_time, retained_metadata.clone()).await, None, "test_retained_target_seed", ) @@ -3890,7 +4394,7 @@ mod tests { store.clone(), 0, bucket.clone(), - source_reader(object, new_body.clone(), new_time, retained_metadata), + source_reader(object, new_body.clone(), new_time, retained_metadata).await, None, "test_retained_target_replace", ) @@ -4896,26 +5400,30 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(storage_class_env)] async fn decommission_outer_fence_loss_blocks_multipart_commits() { - let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir"); - let (_ctx, store, shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-fence-loss", &[4, 4])) - .await; - crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; - - let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4()); - store - .make_bucket(&bucket, &MakeBucketOptions::default()) - .await - .expect("create decommission multipart fence-loss bucket"); - for object in ["new-upload.bin", "complete.bin"] { - write_decommission_test_multipart_source(&store, 0, &bucket, object).await; - } - mark_test_pool_decommissioning(&store, 0).await; - for (object, pause) in [ - ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost), ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost), + ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost), ] { + // Each injected commit failure intentionally leaves a capacity intent for + // reconciliation. Isolate the scenarios so one failure cannot turn the + // next scenario into a capacity-recovery test before its barrier is hit. + let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "decommission-multipart-fence-loss", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission multipart fence-loss bucket"); + write_decommission_test_multipart_source(&store, 0, &bucket, object).await; + mark_test_pool_decommissioning(&store, 0).await; + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( &bucket, object, @@ -4943,8 +5451,10 @@ mod tests { }); tokio::select! { - () = barrier.wait_until_paused() => {} - result = &mut worker => panic!("decommission multipart worker exited before the commit barrier: {result:?}"), + _ = barrier.wait_until_paused() => {} + result = &mut worker => { + panic!("decommission multipart worker finished before the requested {pause:?} commit barrier: {result:?}"); + } } loss_hook.mark_lost(); barrier.release(); @@ -4957,7 +5467,7 @@ mod tests { if let Some(commit_observation) = commit_observation { assert!( !commit_observation.committed(), - "new multipart upload metadata must not commit after the outer fence is lost" + "NewMultipart must reject a lost outer decommission/capacity fence even though it does not acquire the final remote-tuple publication fence" ); } assert_pool_object_absent(&store.pools[1], &bucket, object).await; @@ -4980,9 +5490,9 @@ mod tests { ) .await .expect("same-mutation retry should recover the durable capacity intent"); - } - shutdown.cancel(); + shutdown.cancel(); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -6351,7 +6861,7 @@ mod tests { #[serial_test::serial(storage_class_env)] async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() { let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir"); - let (_ctx, store, _shutdown) = + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let bucket = "stale-tier-source-bucket"; @@ -6360,6 +6870,7 @@ mod tests { .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("bucket should be created"); + register_mock_tier(&ctx.tier_config_mgr(), "STALE-TIER").await; let incarnation = store .bucket_incarnation_id(bucket) .await @@ -6598,8 +7109,10 @@ mod tests { .make_bucket(&bucket, &MakeBucketOptions::default()) .await .expect("create multipart selected-target bucket"); + let first_part_size = 5 * 1024 * 1024; + let mut source_body = vec![b'a'; first_part_size]; + source_body.push(b'b'); for (pool_idx, version_id, body, mod_time) in [ - (0, source_version, b"source version".to_vec(), source_mod_time), ( 1, other_version, @@ -6624,52 +7137,31 @@ mod tests { .await .expect("seed multipart data movement pool"); } + let source_reader = seed_data_movement_source_reader( + &store, + &bucket, + object, + DataMovementSourceSeed { + pool_idx: 0, + version_id: Some(source_version), + body: source_body, + multipart_split: Some(first_part_size), + mod_time: source_mod_time, + user_defined: HashMap::new(), + }, + ) + .await; *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); - let first_part_size = 5 * 1024 * 1024; - let mut source_body = vec![b'a'; first_part_size]; - source_body.push(b'b'); - let source_size = i64::try_from(source_body.len()).expect("multipart source size should fit i64"); let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket); let migration_store = store.clone(); let migration_bucket = bucket.clone(); let migration = tokio::spawn(async move { - let object_bucket = migration_bucket.clone(); crate::data_movement::migrate_object( migration_store, 0, migration_bucket, - GetObjectReader { - stream: Box::new(Cursor::new(source_body)), - object_info: ObjectInfo { - bucket: object_bucket, - name: object.to_string(), - version_id: Some(source_version), - size: source_size, - actual_size: source_size, - etag: Some("source-multipart-etag-2".to_string()), - mod_time: Some(source_mod_time), - parts: Arc::new(vec![ - ObjectPartInfo { - number: 1, - size: first_part_size, - actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), - etag: "source-part-1".to_string(), - ..Default::default() - }, - ObjectPartInfo { - number: 2, - size: 1, - actual_size: 1, - etag: "source-part-2".to_string(), - ..Default::default() - }, - ]), - ..Default::default() - }, - buffered_body: None, - body_source: Default::default(), - }, + source_reader, None, "test_multipart_selected_target", ) @@ -6789,6 +7281,7 @@ mod tests { .expect("tier lease should resolve") .backend_identity(); let entry = Jentry { + persisted_version: 0, obj_name: "receipt-recovery-object".to_string(), version_id: "receipt-recovery-version".to_string(), tier_name: tier_name.to_string(), @@ -6797,6 +7290,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, }; let path = tier_delete_journal_object_name(&entry); let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); @@ -6990,6 +7484,7 @@ mod tests { .expect("tier lease should resolve") .backend_identity(); let entry = Jentry { + persisted_version: 0, obj_name: "multi-source-recovery-object".to_string(), version_id: "multi-source-recovery-version".to_string(), tier_name: tier_name.to_string(), @@ -6998,6 +7493,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, }; let path = tier_delete_journal_object_name(&entry); let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); @@ -7116,6 +7612,8 @@ mod tests { content_sha256: format!("{:064x}", index + RECEIPT_COUNT), identity_sha256: "f".repeat(64), committed: false, + dispatch_identity_sha256: None, + state: None, }, }; store @@ -7223,6 +7721,7 @@ mod tests { .expect("tier lease should resolve") .backend_identity(); let tier_entry = Jentry { + persisted_version: 0, obj_name: "decommissioned-remote-object".to_string(), version_id: "decommissioned-remote-version".to_string(), tier_name: tier_name.to_string(), @@ -7231,6 +7730,7 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: TierDeleteJournalState::Committed, source: None, + dispatch: None, }; let tier_path = tier_delete_journal_object_name(&tier_entry); let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode"); @@ -7803,6 +8303,2002 @@ mod tests { .len() } + #[cfg(feature = "test-util")] + async fn tier_delete_dispatch_manifest_count(store: Arc) -> usize { + store + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 100, + false, + None, + false, + ) + .await + .expect("tier delete dispatch manifests should be listable") + .objects + .len() + } + + #[cfg(feature = "test-util")] + fn synthetic_v6_dispatch_entry( + bucket: &str, + object: &str, + tier_name: &str, + backend_identity: [u8; 32], + remote_version: &str, + ) -> Jentry { + Jentry { + persisted_version: 0, + obj_name: format!("remote/{object}"), + version_id: remote_version.to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Prepared, + source: Some(TierDeleteSourceIdentity { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: Some(uuid::Uuid::new_v4().to_string()), + versioned: true, + version_suspended: false, + data_dir: Some(uuid::Uuid::new_v4().to_string()), + etag: Some(format!("etag-{object}")), + mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()), + }), + dispatch: None, + } + } + + #[cfg(feature = "test-util")] + async fn install_aborting_dispatch_fixture( + store: Arc, + bucket: &str, + incarnation: uuid::Uuid, + prefix: &str, + tier_name: &str, + backend_identity: [u8; 32], + journal_count: usize, + ) -> (String, Vec) { + let entries = (0..journal_count) + .map(|index| { + ( + synthetic_v6_dispatch_entry( + bucket, + &format!("{prefix}{index:06}.bin"), + tier_name, + backend_identity, + &uuid::Uuid::new_v4().to_string(), + ), + Some(TierDeleteJournalState::Prepared), + ) + }) + .collect(); + install_test_tier_delete_dispatch_fixture( + store, + bucket, + incarnation, + prefix, + entries, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("Aborting rollback fixture should persist") + } + + #[cfg(feature = "test-util")] + async fn drive_tier_delete_dispatch_restart_to_convergence(store: Arc) { + tokio::time::timeout(Duration::from_secs(30), async { + let mut stable_empty_observations = 0; + loop { + recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("restarted manifest recovery pass should finish"); + recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("restarted journal recovery pass should finish"); + if tier_delete_journal_count(store.clone()).await == 0 + && tier_delete_dispatch_manifest_count(store.clone()).await == 0 + && tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0 + { + stable_empty_observations += 1; + if stable_empty_observations == 10 { + break; + } + } else { + stable_empty_observations = 0; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("same-disk restart recovery should converge without retained dispatch state"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_delete_journal_rejects_unbound_new_prepare_without_persisting() { + let temp_dir = tempfile::tempdir().expect("create journal admission store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-journal-unbound-prepare", &[4])).await; + let entry = Jentry { + persisted_version: 0, + obj_name: "remote/object".to_string(), + version_id: uuid::Uuid::new_v4().to_string(), + tier_name: "COLD-A".to_string(), + backend_identity: Some([7; 32]), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Prepared, + source: Some(TierDeleteSourceIdentity { + bucket: "source-bucket".to_string(), + object: "source-object".to_string(), + version_id: Some(uuid::Uuid::new_v4().to_string()), + versioned: true, + version_suspended: false, + data_dir: Some(uuid::Uuid::new_v4().to_string()), + etag: Some("source-etag".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()), + }), + dispatch: None, + }; + let error = persist_tier_delete_journal_entry(store.clone(), &entry) + .await + .expect_err("a new source-owning journal without a v6 manifest binding must fail closed"); + assert!(error.to_string().contains("dispatch manifest binding"), "unexpected error: {error}"); + assert_eq!(tier_delete_journal_count(store).await, 0, "rejected prepare must not write a journal"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_recovery_rolls_back_unapproved_crash_states_without_remote_io() { + let temp_dir = tempfile::tempdir().expect("create dispatch rollback store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-rollback", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-manifest-rollback-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("dispatch rollback bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("bucket incarnation should resolve"); + let tier_name = "DISPATCH-ROLLBACK"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + + let preparing = [ + ("preparing/a.bin", TierDeleteJournalState::Prepared), + ("preparing/b.bin", TierDeleteJournalState::Dispatched), + ] + .into_iter() + .map(|(object, state)| { + ( + synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()), + Some(state), + ) + }) + .collect(); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "preparing/", + preparing, + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("Preparing crash fixture should persist"); + + let aborting = [ + ("aborting/missing.bin", None), + ("aborting/prepared.bin", Some(TierDeleteJournalState::Prepared)), + ("aborting/dispatched.bin", Some(TierDeleteJournalState::Dispatched)), + ] + .into_iter() + .map(|(object, state)| { + ( + synthetic_v6_dispatch_entry(bucket, object, tier_name, identity, &uuid::Uuid::new_v4().to_string()), + state, + ) + }) + .collect(); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "aborting/", + aborting, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("Aborting crash fixture should persist"); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "empty/", + Vec::new(), + TierDeleteDispatchManifestState::Completed, + ) + .await + .expect("empty Completed crash fixture should persist"); + + let deleted = tokio::time::timeout(Duration::from_secs(30), async { + let mut deleted = 0; + loop { + let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("manifest recovery should reconcile negative-seal crash states"); + deleted += stats.deleted; + if tier_delete_dispatch_manifest_count(store.clone()).await == 0 { + break deleted; + } + // A different process-global test can briefly revoke the + // fleet proof. Recovery must fail closed for that pass and + // converge after the proof generation is republished. + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("valid negative-seal crash states should eventually converge"); + assert_eq!(deleted, 3); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0); + assert_eq!( + backend.remove_count().await, + 0, + "rollback recovery must perform zero remote delete operations" + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_recovery_quarantines_impossible_committed_rollback_set() { + let temp_dir = tempfile::tempdir().expect("create dispatch quarantine store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-manifest-quarantine", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-manifest-quarantine-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("dispatch quarantine bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("bucket incarnation should resolve"); + let tier_name = "DISPATCH-QUARANTINE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let entries = [TierDeleteJournalState::Prepared, TierDeleteJournalState::Committed] + .into_iter() + .enumerate() + .map(|(index, state)| { + ( + synthetic_v6_dispatch_entry( + bucket, + &format!("quarantine/{index}.bin"), + tier_name, + identity, + &uuid::Uuid::new_v4().to_string(), + ), + Some(state), + ) + }) + .collect(); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "quarantine/", + entries, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("impossible rollback fixture should persist"); + + let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("quarantined manifest scan should finish"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1)); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 2, + "validation must precede every deletion" + ); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store, &manifest_name) + .await + .expect("quarantined manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Aborting) + ); + assert_eq!(backend.remove_count().await, 0); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member() { + const JOURNAL_COUNT: usize = 65; + + let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "bounded-dispatch-rollback-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bounded rollback bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("bounded rollback bucket incarnation should resolve"); + let tier_name = "BOUNDED-DISPATCH-ROLLBACK"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("bounded rollback tier lease should resolve") + .backend_identity(); + let (manifest_name, entries) = + install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "slow/", tier_name, identity, JOURNAL_COUNT) + .await; + let first_name = tier_delete_journal_object_name(entries.first().expect("slow rollback fixture should not be empty")); + let last_name = tier_delete_journal_object_name(entries.last().expect("slow rollback fixture should not be empty")); + let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&first_name, &last_name); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = + tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await }); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused()) + .await + .expect("the first rollback member should reach the slow hook"); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_observed()) + .await + .expect("a member beyond the first two concurrency windows should remain reachable"); + assert!( + (2..=32).contains(&hook.max_in_flight()), + "rollback delete concurrency must make progress without exceeding its bound; observed {}", + hook.max_in_flight() + ); + hook.release_delete(); + worker + .await + .expect("bounded rollback recovery task should join") + .expect("bounded rollback recovery should converge after the slow member releases"); + drop(hook); + + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(backend.remove_count().await, 0, "rollback must not call the remote tier"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() { + let temp_dir = tempfile::tempdir().expect("create detached rollback store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "detached-dispatch-rollback-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("detached rollback bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("detached rollback bucket incarnation should resolve"); + let tier_name = "DETACHED-DISPATCH-ROLLBACK"; + let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("detached rollback tier lease should resolve") + .backend_identity(); + let (manifest_name, entries) = + install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "detached/", tier_name, identity, 1).await; + let journal_name = tier_delete_journal_object_name(&entries[0]); + let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name); + + assert!( + recover_test_tier_delete_dispatch_manifest_with_page_budget( + store.clone(), + &manifest_name, + Duration::from_millis(10), + ) + .await, + "the synthetic page budget must elapse while the manifest worker continues" + ); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused()) + .await + .expect("the detached manifest worker should continue into its delete phase"); + assert!( + tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name), + "the detached worker must retain its per-store/object deduplication guard" + ); + assert!( + !recover_test_tier_delete_dispatch_manifest_with_page_budget(store.clone(), &manifest_name, Duration::from_secs(30),) + .await, + "a duplicate page observation should be retained without queuing another worker" + ); + + hook.release_delete(); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound)); + if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the detached manifest worker should release its guard after convergence"); + drop(hook); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable() { + const MANIFEST_COUNT: usize = 6; + const WORKER_LIMIT: usize = 4; + + let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK"; + let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("bounded detached rollback tier lease should resolve") + .backend_identity(); + let mut manifest_names = Vec::with_capacity(MANIFEST_COUNT); + for index in 0..MANIFEST_COUNT { + let bucket = format!("bounded-detached-dispatch-rollback-{index}"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bounded detached rollback bucket should be created"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("bounded detached rollback bucket incarnation should resolve"); + let (manifest_name, _) = install_aborting_dispatch_fixture( + store.clone(), + &bucket, + incarnation, + &format!("detached-{index}/"), + tier_name, + identity, + 1, + ) + .await; + manifest_names.push(manifest_name); + } + let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_deletes(); + + let first_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| { + let store = store.clone(); + async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_millis(10)) + .await + } + })) + .buffer_unordered(MANIFEST_COUNT) + .collect::>() + .await; + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(WORKER_LIMIT)) + .await + .expect("every admitted detached worker should pause before its first mutation"); + assert_eq!( + first_pass.into_iter().filter(|timed_out| *timed_out).count(), + WORKER_LIMIT, + "only the per-store worker budget may detach" + ); + assert_eq!( + tier_delete_dispatch_manifest_recovery_count_for_test(&store), + WORKER_LIMIT, + "detached workers must remain bounded per store" + ); + + let repeated_pass = futures::stream::iter(manifest_names.iter().cloned().map(|manifest_name| { + let store = store.clone(); + async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(store, &manifest_name, Duration::from_secs(30)).await + } + })) + .buffer_unordered(MANIFEST_COUNT) + .collect::>() + .await; + assert!( + repeated_pass.into_iter().all(|timed_out| !timed_out), + "duplicates and over-budget manifests must be retained without a waiter queue" + ); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), WORKER_LIMIT); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + tier_delete_dispatch_manifest_recovery_count_for_test(&store), + WORKER_LIMIT, + "shutdown must not drop a bounded batch before its admitted work drains" + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT); + hook.release_all_deletes(); + tokio::time::timeout(Duration::from_secs(30), async { + while tier_delete_dispatch_manifest_recovery_count_for_test(&store) != 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("shutdown should cancel every detached worker and release its registry permit"); + assert_eq!(tier_delete_journal_count(store.clone()).await, MANIFEST_COUNT); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, MANIFEST_COUNT); + drop(hook); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + tier_delete_journal_count(store.clone()).await, + MANIFEST_COUNT, + "releasing the old hook after shutdown must not resume a cancelled mutation" + ); + assert_eq!( + tier_delete_dispatch_manifest_recovery_count_for_test(&store), + 0, + "the shutdown store must leave no registry permits behind" + ); + drop(store); + drop(ctx); + + let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "bounded-detached-dispatch-rollback-restart", + &[4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + assert!(!restarted_shutdown.is_cancelled(), "the restarted store needs a fresh shutdown token"); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let active = tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store); + let manifests = tier_delete_dispatch_manifest_count(restarted_store.clone()).await; + if active > 0 || manifests < MANIFEST_COUNT { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the restarted store should start a fresh manifest recovery"); + tokio::time::timeout(Duration::from_secs(30), async { + while tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) != 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the restarted store recovery should release every worker permit"); + if tier_delete_dispatch_manifest_count(restarted_store.clone()).await != 0 { + let stats = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None) + .await + .expect("the restarted store should retry any conservatively retained manifest"); + assert_eq!(stats.failed, 0, "the restarted recovery must not quarantine a valid manifest"); + } + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0); + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_shutdown_stops_aborting_validation_tail_reads() { + const JOURNAL_COUNT: usize = 65; + const ADMITTED_READS: usize = 32; + + let temp_dir = tempfile::tempdir().expect("create validation-read shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-validation-read-shutdown", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-validation-read-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("validation-read shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("validation-read shutdown bucket incarnation should resolve"); + let tier_name = "VALIDATION-READ-SHUTDOWN"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("validation-read shutdown tier lease should resolve") + .backend_identity(); + let (manifest_name, _) = + install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "archive/", tier_name, identity, JOURNAL_COUNT) + .await; + let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Validation); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS)) + .await + .expect("the first validation read window should be admitted"); + assert_eq!(hook.entry_count(), ADMITTED_READS); + assert_eq!(hook.in_flight(), ADMITTED_READS); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit validation tail reads"); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1); + assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await); + assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test()); + assert_eq!(tier_delete_journal_count(store.clone()).await, JOURNAL_COUNT); + assert_eq!(backend.remove_count().await, 0); + + hook.release_all_reads(); + assert!( + !worker.await.expect("validation-read shutdown worker should join"), + "cooperative shutdown should retain an unvalidated rollback" + ); + assert_eq!(hook.entry_count(), ADMITTED_READS); + assert_eq!(hook.in_flight(), 0); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + JOURNAL_COUNT, + "no rollback DELETE may start" + ); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("retained Aborting manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Aborting) + ); + drop(hook); + drop(store); + drop(ctx); + + let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "dispatch-validation-read-shutdown-restart", + &[4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await; + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_shutdown_stops_authorized_tail_reads_before_receipt_or_cas() { + const JOURNAL_COUNT: usize = 65; + const ADMITTED_READS: usize = 32; + + let temp_dir = tempfile::tempdir().expect("create authorized-read shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-authorized-read-shutdown", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-authorized-read-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("authorized-read shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("authorized-read shutdown bucket incarnation should resolve"); + let tier_name = "AUTHORIZED-READ-SHUTDOWN"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("authorized-read shutdown tier lease should resolve") + .backend_identity(); + let entries = (0..JOURNAL_COUNT) + .map(|index| { + ( + synthetic_v6_dispatch_entry( + bucket, + &format!("archive/{index:06}.bin"), + tier_name, + identity, + &uuid::Uuid::new_v4().to_string(), + ), + Some(TierDeleteJournalState::Committed), + ) + }) + .collect(); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + entries, + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized-read shutdown fixture should persist"); + let tier_config = ctx + .tier_config_mgr() + .read() + .await + .tiers + .get(tier_name) + .cloned() + .expect("authorized-read tier config should remain available for restart"); + let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Authorized); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS)) + .await + .expect("the first authorized read window should be admitted"); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit authorized tail reads"); + assert_eq!(hook.in_flight(), ADMITTED_READS); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1); + assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await); + assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test()); + + hook.release_all_reads(); + assert!( + !worker.await.expect("authorized-read shutdown worker should join"), + "cooperative shutdown should retain an unscanned authorized manifest" + ); + assert_eq!(hook.entry_count(), ADMITTED_READS); + assert_eq!(hook.in_flight(), 0); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("retained Authorized manifest should remain readable"), + Some(TierDeleteDispatchManifestState::DispatchAuthorized), + "cancellation before member reads must prevent the Completed CAS" + ); + assert_eq!( + hook.authorized_progress_count(), + 0, + "cancellation before member reads must not attempt a decommission progress receipt" + ); + assert_eq!(backend.remove_count().await, 0); + drop(hook); + drop(store); + drop(ctx); + + let (restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "dispatch-authorized-read-shutdown-restart", + &[4], + )) + .await; + { + let tier_config_mgr = restarted_ctx.tier_config_mgr(); + let mut manager = tier_config_mgr.write().await; + manager.tiers.insert(tier_name.to_string(), tier_config); + manager + .install_test_driver(tier_name, Box::new(backend.clone())) + .expect("the exact authorized-read tier driver should reinstall after restart"); + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await; + assert_eq!(backend.remove_count().await, JOURNAL_COUNT); + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_completion_cas_is_bounded_and_reaches_the_tail() { + const JOURNAL_COUNT: usize = 65; + const ADMITTED_COMMITS: usize = 32; + + let temp_dir = tempfile::tempdir().expect("create dispatch completion concurrency store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completion-concurrency", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-completion-concurrency-bucket"; + let prefix = "archive/"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("dispatch completion concurrency bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("dispatch completion concurrency bucket incarnation should resolve"); + let tier_name = "DISPATCH-COMPLETION-CONCURRENCY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_remove_failure(true); + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("dispatch completion concurrency tier lease should resolve") + .backend_identity(); + let entries = (0..JOURNAL_COUNT) + .map(|index| { + synthetic_v6_dispatch_entry( + bucket, + &format!("{prefix}{index:06}.bin"), + tier_name, + identity, + &uuid::Uuid::new_v4().to_string(), + ) + }) + .collect::>(); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + prefix, + entries + .iter() + .cloned() + .map(|entry| (entry, Some(TierDeleteJournalState::Dispatched))) + .collect(), + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("dispatch completion concurrency fixture should persist"); + + let lifecycle_guard = store + .acquire_bucket_lifecycle_write_lock(bucket) + .await + .expect("dispatch completion should acquire the bucket lifecycle fence"); + let mut fence_opts = ObjectOptions::default(); + fence_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard); + let bucket_fence = fence_opts + .bucket_lifecycle_lock_fence + .clone() + .expect("dispatch completion should capture the bucket lifecycle fence"); + let fleet_proof = + acquire_tier_delete_journal_fleet_proof().expect("dispatch completion concurrency fixture should have a fleet proof"); + let prepared = + prepare_tier_delete_dispatch(store.clone(), bucket, incarnation, prefix, entries, fleet_proof, &bucket_fence) + .await + .expect("the existing Authorized dispatch should reload"); + let active = prepared + .consume(bucket, incarnation, prefix) + .expect("the existing Authorized dispatch permit should be consumable"); + active + .authorization() + .mark_mutation_started(bucket, incarnation, prefix) + .expect("the completion test should mark its synthetic local mutation"); + + let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completion); + let worker_store = store.clone(); + let worker = tokio::spawn(async move { complete_tier_delete_dispatch(worker_store, &active, &bucket_fence).await }); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_COMMITS)) + .await + .expect("the first completion CAS window should be admitted"); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(hook.entry_count(), ADMITTED_COMMITS, "completion must not admit an unbounded CAS tail"); + assert_eq!(hook.in_flight(), ADMITTED_COMMITS); + + hook.release_all_reads(); + worker + .await + .expect("dispatch completion concurrency worker should join") + .expect("every bounded completion CAS should succeed"); + assert_eq!( + hook.entry_count(), + JOURNAL_COUNT, + "completion must eventually admit the entire journal set" + ); + assert_eq!(hook.in_flight(), 0); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("completed concurrency manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Completed) + ); + drop(hook); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_shutdown_stops_completed_missing_journal_tail_reads() { + const JOURNAL_COUNT: usize = 65; + const ADMITTED_READS: usize = 32; + + let temp_dir = tempfile::tempdir().expect("create completed-read shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-completed-read-shutdown", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-completed-read-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("completed-read shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("completed-read shutdown bucket incarnation should resolve"); + let tier_name = "COMPLETED-READ-SHUTDOWN"; + let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("completed-read shutdown tier lease should resolve") + .backend_identity(); + let entries = (0..JOURNAL_COUNT) + .map(|index| { + ( + synthetic_v6_dispatch_entry( + bucket, + &format!("archive/{index:06}.bin"), + tier_name, + identity, + &uuid::Uuid::new_v4().to_string(), + ), + None, + ) + }) + .collect(); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + entries, + TierDeleteDispatchManifestState::Completed, + ) + .await + .expect("completed-read shutdown fixture should persist"); + let hook = TierDeleteDispatchMemberReadTestHook::install_pause(TierDeleteDispatchMemberReadTestStage::Completed); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_read_pause_count(ADMITTED_READS)) + .await + .expect("the first Completed read window should be admitted"); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(hook.entry_count(), ADMITTED_READS, "shutdown must not admit later Completed chunks"); + assert_eq!(hook.in_flight(), ADMITTED_READS); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1); + assert!(tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await); + assert!(crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test()); + + hook.release_all_reads(); + assert!( + !worker.await.expect("completed-read shutdown worker should join"), + "cooperative shutdown should retain a partially scanned Completed manifest" + ); + assert_eq!(hook.entry_count(), ADMITTED_READS); + assert_eq!(hook.in_flight(), 0); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!( + tier_delete_dispatch_manifest_count(store.clone()).await, + 1, + "manifest DELETE must not run" + ); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("retained Completed manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Completed) + ); + drop(hook); + drop(store); + drop(ctx); + + let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "dispatch-completed-read-shutdown-restart", + &[4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await; + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_shutdown_drains_preparing_cas_before_releasing_fences() { + let temp_dir = tempfile::tempdir().expect("create preparing CAS shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-preparing-cas-shutdown", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-preparing-cas-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("preparing CAS shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("preparing CAS shutdown bucket incarnation should resolve"); + let entry = synthetic_v6_dispatch_entry( + bucket, + "archive/preparing-cas.bin", + "PREPARING-CAS-SHUTDOWN", + [31; 32], + &uuid::Uuid::new_v4().to_string(), + ); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + vec![(entry, Some(TierDeleteJournalState::Prepared))], + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("preparing CAS shutdown fixture should persist"); + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + RUSTFS_META_BUCKET, + &manifest_name, + crate::set_disk::PutObjectCommitPause::BeforeQuotaRename, + ); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + barrier.wait_until_paused().await; + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + tier_delete_dispatch_manifest_recovery_count_for_test(&store), + 1, + "a paused CAS must retain its registry permit after shutdown" + ); + assert!( + tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await, + "a paused CAS must retain its manifest operation lock after shutdown" + ); + assert!( + crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(), + "a paused CAS must retain its fleet-proof generation permit" + ); + + barrier.release(); + assert!( + !worker.await.expect("preparing CAS shutdown worker should join"), + "cooperative shutdown should retain the partially advanced manifest" + ); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert!( + !tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await, + "the operation lock may release only after the admitted CAS drains" + ); + assert!( + com::read_config(store.clone(), &manifest_name).await.is_ok(), + "shutdown must retain the durable manifest for restart replay" + ); + drop(barrier); + drop(store); + drop(ctx); + + let (_restarted_ctx, restarted_store, restarted_shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "dispatch-preparing-cas-shutdown-restart", + &[4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + tokio::time::timeout(Duration::from_secs(30), async { + let mut stable_empty_observations = 0; + loop { + if tier_delete_journal_count(restarted_store.clone()).await == 0 + && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0 + && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0 + { + stable_empty_observations += 1; + if stable_empty_observations == 10 { + break; + } + } else { + stable_empty_observations = 0; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("the restarted background recovery should replay the drained Preparing CAS"); + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0); + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_shutdown_drains_admitted_delete_batch_without_tail_admission() { + const JOURNAL_COUNT: usize = 65; + const ADMITTED_BATCH: usize = 32; + + let temp_dir = tempfile::tempdir().expect("create delete-batch shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-delete-batch-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("delete-batch shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("delete-batch shutdown bucket incarnation should resolve"); + let (manifest_name, entries) = install_aborting_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + "DELETE-BATCH-SHUTDOWN", + [32; 32], + JOURNAL_COUNT, + ) + .await; + let mut journal_names = entries.iter().map(tier_delete_journal_object_name).collect::>(); + journal_names.sort(); + let published_delete_name = journal_names[0].clone(); + let hook = TierDeleteDispatchRollbackTestHook::install_pause_all_except_delete(&published_delete_name); + let barrier = + crate::set_disk::DeleteObjectCommitBarrier::install_after_publish(RUSTFS_META_BUCKET, &published_delete_name); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + barrier.wait_until_paused().await; + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_pause_count(ADMITTED_BATCH - 1)) + .await + .expect("the rest of the bounded delete batch should pause before mutation"); + assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1); + assert!( + tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await, + "the partially published DELETE must retain its operation lock" + ); + assert!( + crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(), + "the partially published DELETE must retain its fleet-proof permit" + ); + assert_eq!( + hook.delete_entry_count(), + ADMITTED_BATCH, + "shutdown must stop admitting the tail behind the bounded batch" + ); + + barrier.release(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + tier_delete_dispatch_manifest_recovery_count_for_test(&store), + 1, + "the worker must retain its guards until every admitted sibling drains" + ); + hook.release_all_deletes(); + assert!( + !worker.await.expect("delete-batch shutdown worker should join"), + "cooperative shutdown should retain the partial rollback for restart" + ); + assert_eq!(hook.delete_entry_count(), ADMITTED_BATCH); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert!( + tier_delete_journal_count(store.clone()).await >= JOURNAL_COUNT - 1, + "only the already-published DELETE may commit before restart" + ); + drop(barrier); + drop(hook); + drop(store); + drop(ctx); + + let (_restarted_ctx, restarted_store, restarted_shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-delete-batch-shutdown-restart", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + tokio::time::timeout(Duration::from_secs(30), async { + let mut stable_empty_observations = 0; + loop { + if tier_delete_journal_count(restarted_store.clone()).await == 0 + && tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0 + && tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0 + { + stable_empty_observations += 1; + if stable_empty_observations == 10 { + break; + } + } else { + stable_empty_observations = 0; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("the restarted background recovery should replay the partially drained delete batch"); + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(restarted_store).await, 0); + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatch_manifest_rollback_retries_delete_and_confirmation_failures() { + const JOURNAL_COUNT: usize = 40; + + let temp_dir = tempfile::tempdir().expect("create rollback retry store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "dispatch-rollback-retry-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("rollback retry bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("rollback retry bucket incarnation should resolve"); + let tier_name = "DISPATCH-ROLLBACK-RETRY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("rollback retry tier lease should resolve") + .backend_identity(); + + let (delete_manifest, delete_entries) = install_aborting_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "delete-failure/", + tier_name, + identity, + JOURNAL_COUNT, + ) + .await; + let delete_failure_name = tier_delete_journal_object_name(&delete_entries[10]); + let delete_hook = TierDeleteDispatchRollbackTestHook::install_delete_failure(&delete_failure_name); + let delete_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest) + .await + .expect_err("an injected member delete failure must retain the manifest") + .to_string(); + assert!(delete_error.contains("injected tier delete dispatch rollback delete failure")); + assert!( + tier_delete_journal_count(store.clone()).await > 0, + "the failed member and all work not admitted after the first error must remain retryable" + ); + drop(delete_hook); + recover_test_tier_delete_dispatch_manifest(store.clone(), &delete_manifest) + .await + .expect("the delete-stage retry should converge"); + + let (confirmation_manifest, confirmation_entries) = install_aborting_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "confirmation-failure/", + tier_name, + identity, + JOURNAL_COUNT, + ) + .await; + let confirmation_failure_name = tier_delete_journal_object_name(&confirmation_entries[10]); + let confirmation_hook = TierDeleteDispatchRollbackTestHook::install_confirmation_failure(&confirmation_failure_name); + let confirmation_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest) + .await + .expect_err("an injected confirmation failure must retain the manifest") + .to_string(); + assert!(confirmation_error.contains("injected tier delete dispatch rollback confirmation failure")); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "confirmation failure occurs only after the bounded delete phase drains" + ); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1); + drop(confirmation_hook); + recover_test_tier_delete_dispatch_manifest(store.clone(), &confirmation_manifest) + .await + .expect("the confirmation-stage retry should converge over missing members"); + + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() { + let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "v6-checkpoint-fence-loss-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("checkpoint fence-loss bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("checkpoint fence-loss bucket incarnation should resolve"); + let entry = synthetic_v6_dispatch_entry( + bucket, + "archive/fence-loss.bin", + "FENCE-LOSS-TIER", + [18; 32], + &uuid::Uuid::new_v4().to_string(), + ); + let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + vec![(entry, Some(TierDeleteJournalState::Prepared))], + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("checkpoint fence-loss fixture should persist"); + let journal_name = tier_delete_journal_object_name(&entries[0]); + let manifest_data = com::read_config(store.clone(), &manifest_name) + .await + .expect("checkpoint fence-loss manifest should be readable"); + let journal_data = com::read_config(store.clone(), &journal_name) + .await + .expect("checkpoint fence-loss journal should be readable"); + for pool in &store.pools { + com::save_config(pool.clone(), &manifest_name, manifest_data.clone()) + .await + .expect("manifest copy should persist in each checkpoint fence-loss pool"); + com::save_config(pool.clone(), &journal_name, journal_data.clone()) + .await + .expect("journal copy should persist in each checkpoint fence-loss pool"); + } + mark_test_pool_decommissioning(&store, 0).await; + for (path, data) in [ + (&manifest_name, manifest_data.as_slice()), + (&journal_name, journal_data.as_slice()), + ] { + let record = validate_durable_ilm_record(path, data).expect("checkpoint fence-loss v6 record should validate"); + store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false) + .await + .expect("checkpoint fence-loss receipt should persist"); + } + + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + RUSTFS_META_BUCKET, + &manifest_name, + crate::store::object::DecommissionMutationFenceTestPhase::Migration, + ); + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + RUSTFS_META_BUCKET, + &manifest_name, + crate::set_disk::PutObjectCommitPause::BeforeQuotaRename, + ); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = + tokio::spawn(async move { recover_test_tier_delete_dispatch_manifest(worker_store, &worker_manifest).await }); + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + let error = worker + .await + .expect("checkpoint fence-loss recovery task should join") + .expect_err("lost publication fence must reject the no-lock checkpoint PUT") + .to_string(); + assert!( + error.contains("lock") || error.contains("fence"), + "unexpected checkpoint fence-loss error: {error}" + ); + assert_eq!( + com::read_config(store.pools[1].clone(), &manifest_name) + .await + .expect("fenced target manifest should remain readable"), + manifest_data, + "the target checkpoint must not commit after its publication fence is lost" + ); + + drop(loss_hook); + recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect("checkpoint recovery should converge after a fresh fence is acquired"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn v6_decommission_checkpoint_partial_target_failure_retries_without_advancing_receipts() { + let temp_dir = tempfile::tempdir().expect("create v6 partial checkpoint store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-partial-checkpoint", &[4, 4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "v6-partial-checkpoint-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("partial checkpoint bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("partial checkpoint bucket incarnation should resolve"); + let entry = synthetic_v6_dispatch_entry( + bucket, + "archive/partial.bin", + "PARTIAL-CHECKPOINT-TIER", + [19; 32], + &uuid::Uuid::new_v4().to_string(), + ); + let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + vec![(entry, Some(TierDeleteJournalState::Prepared))], + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("partial checkpoint fixture should persist"); + let journal_name = tier_delete_journal_object_name(&entries[0]); + let manifest_data = com::read_config(store.clone(), &manifest_name) + .await + .expect("partial checkpoint manifest should be readable"); + let journal_data = com::read_config(store.clone(), &journal_name) + .await + .expect("partial checkpoint journal should be readable"); + for pool in &store.pools { + com::save_config(pool.clone(), &manifest_name, manifest_data.clone()) + .await + .expect("manifest copy should persist in each partial checkpoint pool"); + com::save_config(pool.clone(), &journal_name, journal_data.clone()) + .await + .expect("journal copy should persist in each partial checkpoint pool"); + } + mark_test_pool_decommissioning_with_split_targets(&store, 0).await; + for target_pool_index in [1, 2] { + for (path, data) in [ + (&manifest_name, manifest_data.as_slice()), + (&journal_name, journal_data.as_slice()), + ] { + let record = validate_durable_ilm_record(path, data).expect("partial checkpoint v6 record should validate"); + store + .persist_decommission_durable_ilm_receipt_for_test(0, target_pool_index, path, &record, false) + .await + .expect("partial checkpoint receipt should persist on each target"); + } + } + let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint( + store.clone(), + &manifest_name, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("partial checkpoint Aborting generation should encode"); + let targets = store + .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag) + .await + .expect("partial checkpoint targets should resolve") + .expect("the active decommission should own the partial checkpoint"); + assert_eq!( + targets.iter().map(|target| target.target_pool_index).collect::>(), + vec![1, 2], + "the capacity reservation must exercise two independently confirmed targets" + ); + + let failure_hook = DecommissionCheckpointTargetFailureHook::install(2); + let error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect_err("the injected second-target failure must reject the checkpoint"); + assert!( + error.to_string().contains("injected decommission checkpoint target failure"), + "unexpected partial checkpoint error: {error}" + ); + let mut diagnostic = &error as &(dyn std::error::Error + 'static); + let mut target_context_found = false; + while let Some(source) = diagnostic.source() { + if source.to_string().contains("target pool 2") { + target_context_found = true; + break; + } + diagnostic = source; + } + assert!( + target_context_found, + "the stable checkpoint error must retain the failed target in its diagnostic source chain: {error:?}" + ); + assert_eq!( + com::read_config(store.pools[1].clone(), &manifest_name) + .await + .expect("first target checkpoint should be readable"), + aborting_data, + "the first target should model a committed checkpoint before the later failure" + ); + assert_eq!( + com::read_config(store.pools[2].clone(), &manifest_name) + .await + .expect("second target checkpoint should be readable"), + manifest_data, + "the failed second target must retain the prior generation" + ); + let retry_targets = store + .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag) + .await + .expect("unadvanced receipts must still authorize the exact partial checkpoint retry") + .expect("the partial checkpoint retry should remain decommission-owned"); + assert!(retry_targets[0].already_committed); + assert!(!retry_targets[1].already_committed); + + drop(failure_hook); + recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect("the partial checkpoint should converge after the target recovers"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn v6_decommission_checkpoint_shutdown_drains_target_put_and_replays_capacity() { + let temp_dir = tempfile::tempdir().expect("create checkpoint shutdown store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-shutdown", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "v6-checkpoint-shutdown-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("checkpoint shutdown bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("checkpoint shutdown bucket incarnation should resolve"); + let entry = synthetic_v6_dispatch_entry( + bucket, + "archive/checkpoint-shutdown.bin", + "CHECKPOINT-SHUTDOWN", + [33; 32], + &uuid::Uuid::new_v4().to_string(), + ); + let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + vec![(entry, Some(TierDeleteJournalState::Prepared))], + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("checkpoint shutdown fixture should persist"); + let journal_name = tier_delete_journal_object_name(&entries[0]); + let manifest_data = com::read_config(store.clone(), &manifest_name) + .await + .expect("checkpoint shutdown manifest should be readable"); + let journal_data = com::read_config(store.clone(), &journal_name) + .await + .expect("checkpoint shutdown journal should be readable"); + for pool in &store.pools { + com::save_config(pool.clone(), &manifest_name, manifest_data.clone()) + .await + .expect("manifest copy should persist in each checkpoint shutdown pool"); + com::save_config(pool.clone(), &journal_name, journal_data.clone()) + .await + .expect("journal copy should persist in each checkpoint shutdown pool"); + } + mark_test_pool_decommissioning(&store, 0).await; + for (path, data) in [ + (&manifest_name, manifest_data.as_slice()), + (&journal_name, journal_data.as_slice()), + ] { + let record = validate_durable_ilm_record(path, data).expect("checkpoint shutdown v6 record should validate"); + store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false) + .await + .expect("checkpoint shutdown receipt should persist"); + } + let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint( + store.clone(), + &manifest_name, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("checkpoint shutdown Aborting generation should encode"); + let targets = store + .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag) + .await + .expect("checkpoint shutdown target should resolve") + .expect("the active decommission should own the checkpoint target"); + assert_eq!(targets.len(), 1); + let target = targets[0].clone(); + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + RUSTFS_META_BUCKET, + &manifest_name, + crate::set_disk::PutObjectCommitPause::BeforeQuotaRename, + ); + let worker_store = store.clone(); + let worker_manifest = manifest_name.clone(); + let worker = tokio::spawn(async move { + recover_test_tier_delete_dispatch_manifest_with_page_budget(worker_store, &worker_manifest, Duration::from_secs(30)) + .await + }); + barrier.wait_until_paused().await; + assert!( + store + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await, + "the target PUT must persist its capacity intent before commit" + ); + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 1); + assert!( + tier_delete_dispatch_manifest_operation_lock_held_for_test(store.clone(), &manifest_name).await, + "the target checkpoint PUT must retain its operation lock after shutdown" + ); + assert!( + crate::services::notification_sys::tier_delete_journal_fleet_proof_has_inflight_for_test(), + "the target checkpoint PUT must retain its fleet-proof permit after shutdown" + ); + + barrier.release(); + assert!( + !worker.await.expect("checkpoint shutdown worker should join"), + "cooperative shutdown should retain the partially committed checkpoint" + ); + assert_eq!(tier_delete_dispatch_manifest_recovery_count_for_test(&store), 0); + assert_eq!( + com::read_config(store.pools[target.target_pool_index].clone(), &manifest_name) + .await + .expect("the committed target checkpoint should be readable"), + aborting_data, + "the admitted target PUT must finish before its fences release" + ); + assert!( + !store + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await, + "the admitted target PUT must drain its capacity transaction before releasing the recovery fences" + ); + drop(barrier); + let mut reloaded_pool_meta = PoolMeta::default(); + reloaded_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("checkpoint shutdown pool metadata should reload from disk"); + *store.pool_meta.write().await = reloaded_pool_meta; + recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect("a fresh recovery attempt should replay the committed checkpoint and receipt"); + assert!( + !store + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await, + "durable replay must reconcile the exact capacity transaction" + ); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("checkpoint shutdown receipts should be listable"), + 2, + "durable replay must advance both the manifest and journal receipts" + ); + drop(ctx); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn v6_dispatch_recovery_advances_decommission_receipts_and_cleans_target_copies() { + let temp_dir = tempfile::tempdir().expect("create v6 decommission receipt store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-dispatch-decommission-receipts", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "v6-dispatch-decommission-receipts-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("receipt bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("receipt bucket incarnation should resolve"); + let entry = synthetic_v6_dispatch_entry( + bucket, + "archive/receipt.bin", + "RECEIPT-TIER", + [17; 32], + &uuid::Uuid::new_v4().to_string(), + ); + let (manifest_name, entries) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + vec![(entry, Some(TierDeleteJournalState::Prepared))], + TierDeleteDispatchManifestState::Preparing, + ) + .await + .expect("Preparing v6 fixture should persist"); + let journal_name = tier_delete_journal_object_name(&entries[0]); + let manifest_data = com::read_config(store.clone(), &manifest_name) + .await + .expect("manifest fixture should be readable"); + let journal_data = com::read_config(store.clone(), &journal_name) + .await + .expect("journal fixture should be readable"); + + // Model an in-flight decommission after both records have been copied + // to its target pool but before their state machines advance. + for pool in &store.pools { + com::save_config(pool.clone(), &manifest_name, manifest_data.clone()) + .await + .expect("manifest copy should persist in each pool"); + com::save_config(pool.clone(), &journal_name, journal_data.clone()) + .await + .expect("journal copy should persist in each pool"); + } + // Exercise the same durable capacity reservation and identity that a + // real decommission start installs. A hand-written active flag makes + // every target mutation look external and masks whether recovery is + // correctly charged to the decommission owner. + mark_test_pool_decommissioning(&store, 0).await; + + let missing_receipt_error = recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect_err("an active decommission without receipt coverage must block v6 checkpoint mutation") + .to_string(); + assert!( + missing_receipt_error.contains("missing receipt coverage"), + "unexpected missing-receipt error: {missing_receipt_error}" + ); + assert_eq!( + com::read_config(store.pools[1].clone(), &manifest_name) + .await + .expect("blocked checkpoint must retain the target manifest generation"), + manifest_data, + "missing receipt coverage must fail before mutating a target copy" + ); + + for (path, data) in [ + (&manifest_name, manifest_data.as_slice()), + (&journal_name, journal_data.as_slice()), + ] { + let record = validate_durable_ilm_record(path, data).expect("v6 durable record should validate"); + store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, path, &record, false) + .await + .expect("initial decommission receipt should persist"); + } + + // Model a crash after the target checkpoint PUT committed but before + // its temporary-capacity intent was resolved. Recovery must recognize + // the exact target bytes and finish that same deterministic intent + // instead of treating the checkpoint as an uncharged success. + let (aborting_data, preparing_etag) = test_tier_delete_dispatch_manifest_checkpoint( + store.clone(), + &manifest_name, + TierDeleteDispatchManifestState::Aborting, + ) + .await + .expect("the Aborting checkpoint should encode"); + let checkpoint_targets = store + .decommission_durable_ilm_checkpoint_targets(&manifest_name, &aborting_data, &preparing_etag) + .await + .expect("receipt-bearing checkpoint targets should resolve") + .expect("an active decommission should own the checkpoint"); + assert_eq!(checkpoint_targets.len(), 1); + let checkpoint_target = checkpoint_targets + .into_iter() + .next() + .expect("one checkpoint target should exist"); + let checkpoint_owner = checkpoint_target.capacity_owner; + let target_pool_index = checkpoint_target.target_pool_index; + let injected_write_error = store + .run_decommission_capacity_temporary_mutation_with_capacity_lease( + target_pool_index, + Some(checkpoint_owner), + Some(aborting_data.len()), + |_| { + let target_pool = store.pools[target_pool_index].clone(); + let manifest_name = manifest_name.clone(); + let aborting_data = aborting_data.clone(); + async move { + com::save_config(target_pool, &manifest_name, aborting_data) + .await + .expect("the injected target checkpoint should commit"); + Err::<(), Error>(Error::other("injected crash after checkpoint target commit")) + } + }, + ) + .await + .expect_err("the injected post-commit crash should leave a pending capacity intent") + .to_string(); + assert!(injected_write_error.contains("injected crash after checkpoint target commit")); + assert!( + store + .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner) + .await, + "the committed target must retain its exact unresolved capacity intent" + ); + + recover_test_tier_delete_dispatch_manifest(store.clone(), &manifest_name) + .await + .expect("Preparing rollback should advance receipts and finish terminal cleanup"); + assert!( + !store + .has_decommission_capacity_temporary_mutation_state(target_pool_index, checkpoint_owner) + .await, + "already-committed checkpoint recovery must resolve the pending capacity intent" + ); + let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("the terminal source checkpoint should remain non-actionable"); + assert_eq!((stats.scanned, stats.deleted, stats.retained, stats.failed), (1, 0, 1, 0)); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("v6 terminal receipts should be listable"), + 2 + ); + + { + let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test(); + let proof_error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manifest_name), + &manifest_name, + ) + .await + .expect_err("revoked v6 fleet proof must block final source cleanup") + .to_string(); + assert!( + proof_error.contains("fleet capability is unavailable"), + "unexpected revoked-proof cleanup error: {proof_error}" + ); + assert!( + com::read_config(store.pools[0].clone(), &manifest_name).await.is_ok(), + "revoked fleet proof must retain the last source manifest copy" + ); + } + + for path in [&manifest_name, &journal_name] { + store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(path), path) + .await + .expect("terminal receipt should authorize verified source cleanup"); + } + let stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("verified source cleanup should remove the last manifest copy"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (0, 0, 0)); + for path in [&manifest_name, &journal_name] { + for pool in &store.pools { + assert!( + matches!(com::read_config(pool.clone(), path).await, Err(Error::ConfigNotFound)), + "terminal receipt cleanup must remove `{path}` from every staged pool" + ); + } + } + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn authorized_all_committed_manifest_recovers_after_last_commit_crash() { + let temp_dir = tempfile::tempdir().expect("create authorized manifest recovery store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-manifest-last-commit", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "authorized-manifest-last-commit-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("authorized recovery bucket should be created"); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("bucket incarnation should resolve"); + let tier_name = "AUTHORIZED-LAST-COMMIT"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve"); + let mut entries = Vec::new(); + for index in 0..2 { + let remote_version = uuid::Uuid::new_v4().to_string(); + let entry = synthetic_v6_dispatch_entry( + bucket, + &format!("archive/{index}.bin"), + tier_name, + lease.backend_identity(), + &remote_version, + ); + backend.set_put_remote_version(Some(remote_version)).await; + lease + .put(&entry.obj_name, ReaderImpl::Body(bytes::Bytes::from_static(b"remote candidate")), 16) + .await + .expect("remote candidate should be seeded"); + entries.push((entry, Some(TierDeleteJournalState::Committed))); + } + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + "archive/", + entries, + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("last-commit crash fixture should persist"); + + let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("all-Committed Authorized manifest should advance"); + assert_eq!((manifest_stats.scanned, manifest_stats.advanced, manifest_stats.failed), (1, 1, 0)); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("advanced manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Completed) + ); + let journal_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("Completed journal set should recover remote candidates"); + assert_eq!((journal_stats.scanned, journal_stats.deleted, journal_stats.failed), (2, 2, 0)); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1); + let gc_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("manifest coordinator should remove the empty Completed manifest"); + assert_eq!((gc_stats.scanned, gc_stats.deleted, gc_stats.failed), (1, 1, 0)); + assert_eq!(tier_delete_dispatch_manifest_count(store).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), 2); + } + #[cfg(feature = "test-util")] async fn transition_transaction_record_count(store: Arc) -> usize { store @@ -7873,6 +10369,2500 @@ mod tests { .expect("tier delete journal recovery should complete"); } + #[cfg(feature = "test-util")] + async fn wait_for_tier_free_version_recovery( + store: Arc, + backend: &MockWarmBackend, + expected_removes: usize, + ) { + ExpiryState::resize_workers(1, store.clone()).await; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let stats = recover_tier_free_versions(store.clone(), 100, None, None) + .await + .expect("tier free-version recovery scan should succeed"); + if backend.remove_versions().await.len() >= expected_removes && stats.enqueued == 0 && stats.failed == 0 { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("tier free-version recovery should complete"); + } + + #[cfg(feature = "test-util")] + async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) { + let expiry_state = store.ctx.expiry_state(); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let idle = { + let state = expiry_state.read().await; + state.pending_tasks() == 0 && state.active_tasks() == 0 + }; + if idle { + // Recheck after a scheduler turn so the tiny hand-off + // between dequeue and active-task accounting cannot make + // the test observe a false idle state. + tokio::time::sleep(Duration::from_millis(10)).await; + let state = expiry_state.read().await; + if state.pending_tasks() == 0 && state.active_tasks() == 0 { + return; + } + } else { + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + }) + .await + .expect("lifecycle expiry workers should become idle"); + } + + #[cfg(feature = "test-util")] + async fn run_transitioned_delete_free_version_owner_case( + store_name: &str, + tier_name: &str, + bucket: &str, + object: &str, + restore_before_delete: bool, + ) { + let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), store_name, &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("source bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b'd'; 1024 * 1024]); + let original = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: original.etag.clone().expect("source should have an etag"), + ..Default::default() + }, + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("source transition should commit"); + assert_eq!(backend.object_count().await, 1, "transition should create one remote object"); + if restore_before_delete { + store + .clone() + .restore_transitioned_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + restore_request: s3s::dto::RestoreRequest { + days: Some(1), + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + ) + .await + .expect("restore should complete before transitioned delete"); + let restored = store + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("restored transitioned source should remain readable"); + assert!( + restored.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()), + "restore completion metadata must be present before the delete regression" + ); + } + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default()) + .await + .expect("transitioned source delete should commit"); + + let local_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-delete metadata should remain decodable"); + let local_versions = local_versions.expect("ordinary delete must leave a durable free-version cleanup owner"); + assert_eq!( + local_versions + .versions + .iter() + .chain(local_versions.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "ordinary delete must leave exactly one hidden free-version owner" + ); + assert_eq!( + local_versions + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "ordinary delete must remove the visible transitioned source" + ); + let listed = store + .clone() + .list_object_versions(bucket, "", None, None, None, 100) + .await + .expect("source versions should be listable"); + assert!(listed.objects.is_empty(), "source must have no visible versions after delete"); + + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary delete must not create a journal" + ); + + backend.set_remove_failure(false); + ExpiryState::resize_workers(1, store.clone()).await; + let recovered = recover_tier_free_versions(store.clone(), 100, None, None) + .await + .expect("free-version recovery should scan the hidden owner"); + assert_eq!((recovered.enqueued, recovered.failed), (1, 0)); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let metadata_absent = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("free-version cleanup metadata should remain readable") + .is_none(); + if metadata_absent && backend.remove_versions().await.len() == 1 { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("free-version recovery should delete the remote and exact local owner"); + assert_eq!(backend.object_count().await, 0, "free-version recovery should remove the remote object"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("ordinary bucket delete should succeed after the last transitioned object"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn transitioned_delete_uses_free_version_as_cleanup_owner() { + run_transitioned_delete_free_version_owner_case( + "transitioned-delete-journal-owner", + "DELETEOWNER", + "transitioned-delete-journal-owner-bucket", + "transition/archive.bin", + false, + ) + .await; + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn restored_transitioned_delete_uses_free_version_as_cleanup_owner() { + run_transitioned_delete_free_version_owner_case( + "restored-transitioned-delete-journal-owner", + "RESTOREDELETEOWNER", + "restored-transitioned-delete-journal-owner-bucket", + "transition/archive.bin", + true, + ) + .await; + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool() { + let temp_dir = tempfile::tempdir().expect("create tier reference store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-free-reference-proof", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "FREE-PROOF"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("test tier identity should resolve") + .backend_identity(); + let bucket = "tier-free-reference-proof-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("source bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b'f'; 1024 * 1024]); + let original = store.pools[0] + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + store.pools[0] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: original.etag.clone().expect("source should have an etag"), + ..Default::default() + }, + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("source transition should commit"); + backend.set_remove_failure(true); + store.pools[0] + .delete_object( + bucket, + object, + ObjectOptions { + tier_delete_journal_api: Some(store.clone()), + ..Default::default() + }, + ) + .await + .expect("journal-less compatibility delete should retain a free-version"); + + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("free-version metadata should remain decodable") + .expect("free-version metadata should remain on disk"); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "fixture must contain one real serialized hidden free-version" + ); + + let mut hot_reader = PutObjReader::from_vec(vec![b'h'; 1024 * 1024]); + store.pools[1] + .put_object( + bucket, + object, + &mut hot_reader, + &ObjectOptions { + mod_time: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)), + ..Default::default() + }, + ) + .await + .expect("second pool hot source should be written"); + let merged = store + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("merged lookup should return the second-pool hot source"); + assert!( + merged.transitioned_object.status.is_empty() && !merged.transitioned_object.free_version, + "the ordinary second-pool source must hide the first-pool free-version from merged lookup" + ); + + let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true) + .await + .expect_err("force tier removal must not bypass a physical free-version cleanup obligation"); + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_reference_proof_fails_closed_when_physical_bucket_topology_is_incomplete() { + let temp_dir = tempfile::tempdir().expect("create incomplete tier reference store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-reference-incomplete-topology", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "INCOMPLETE-PROOF"; + register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("test tier identity should resolve") + .backend_identity(); + store.pools[0].disk_set[0].disks.write().await[0] = None; + + let err = ensure_no_authoritative_tier_references_for_test(store, tier_name, backend_identity, true) + .await + .expect_err("an incomplete physical bucket enumeration must block tier mutation"); + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!( + err.message.contains("complete set quorum"), + "unexpected reference proof error: {}", + err.message + ); + } + + #[cfg(feature = "test-util")] + async fn copy_test_xlmeta_between_pools( + root: &std::path::Path, + source_pool: usize, + target_pool: usize, + bucket: &str, + object: &str, + ) { + for disk_index in 0..4 { + let source = root.join(format!("pool{source_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); + let target = root.join(format!("pool{target_pool}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); + tokio::fs::create_dir_all(target.parent().expect("target xl.meta should have a parent")) + .await + .expect("target object directory should be created"); + tokio::fs::copy(&source, &target) + .await + .expect("xl.meta should copy exactly between pools"); + } + } + + #[cfg(feature = "test-util")] + async fn rewrite_transitioned_xlmeta_as_legacy_unknown( + root: &std::path::Path, + pool_index: usize, + bucket: &str, + object: &str, + ) { + for disk_index in 0..4 { + let metadata_path = + root.join(format!("pool{pool_index}/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); + let encoded = tokio::fs::read(&metadata_path) + .await + .expect("transitioned xl.meta should be readable"); + let mut metadata = FileMeta::load(&encoded).expect("transitioned xl.meta should decode"); + let mut found = false; + for shallow in &mut metadata.versions { + let mut version = shallow + .parse_version_meta() + .expect("transitioned version record should decode"); + let Some(object_meta) = version.object.as_mut() else { + continue; + }; + if rustfs_utils::http::metadata_compat::get_bytes( + &object_meta.meta_sys, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS, + ) + .as_deref() + != Some(rustfs_filemeta::TRANSITION_COMPLETE.as_bytes()) + { + continue; + } + found = true; + for suffix in [ + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID, + ] { + rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix); + } + *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version) + .expect("legacy transitioned version should re-encode"); + } + assert!(found, "transitioned source should exist in serialized xl.meta"); + tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("legacy transitioned xl.meta should encode")) + .await + .expect("legacy transitioned xl.meta should persist"); + } + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix() { + let temp_dir = tempfile::tempdir().expect("create legacy transitioned delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-transitioned-delete", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "LEGACY-UNKNOWN-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "legacy-unknown-transitioned-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("legacy source bucket should be created"); + + for (index, object) in ["single.bin", "batch.bin", "prefix/legacy.bin"].into_iter().enumerate() { + let mut reader = PutObjReader::from_vec(vec![b'l' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("legacy source should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("legacy source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("legacy source should transition"); + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await; + let legacy = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("legacy source metadata should remain decodable") + .expect("legacy source should remain on disk") + .versions + .into_iter() + .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE) + .expect("legacy transitioned source should remain visible before delete"); + assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); + assert_eq!( + crate::services::tier::tier::tier_destination_id_from_metadata(&legacy.metadata) + .expect("legacy metadata identity lookup should not fail"), + None + ); + } + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal(bucket, "single.bin", ObjectOptions::default()) + .await + .expect("legacy single delete should retain its free-version fallback"); + let free_version_prefix_error = store + .delete_object_with_tier_delete_journal( + bucket, + "single.bin", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("recursive prefix delete must not discard an existing hidden cleanup owner"); + assert!( + free_version_prefix_error + .to_string() + .contains("tier free-version cleanup obligation"), + "unexpected hidden-owner prefix failure: {free_version_prefix_error}" + ); + let (_deleted, errors) = store + .delete_objects_with_tier_delete_journal( + bucket, + vec![ObjectToDelete { + object_name: "batch.bin".to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "legacy batch delete should succeed: {errors:?}"); + let prefix_error = store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("legacy recursive prefix delete must fail closed before physical removal"); + assert!( + prefix_error.to_string().contains("legacy Unknown"), + "unexpected prefix failure: {prefix_error}" + ); + + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "legacy cleanup must not create an unsafe journal" + ); + assert_eq!( + backend.object_count().await, + 3, + "failed free-version cleanup must retain all remote objects" + ); + for object in ["single.bin", "batch.bin"] { + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("legacy post-delete metadata should decode") + .expect("legacy cleanup owner should remain on disk"); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "legacy {object} must retain exactly one hidden cleanup owner" + ); + assert_eq!( + exact.versions.iter().filter(|version| !version.tier_free_version()).count(), + 0, + "legacy {object} must have no visible source after delete" + ); + } + let retained_prefix = store.pools[0] + .get_disks_by_key("prefix/legacy.bin") + .load_file_info_versions_exact(bucket, "prefix/legacy.bin") + .await + .expect("rejected prefix metadata should decode") + .expect("rejected prefix delete must retain its source"); + assert_eq!( + retained_prefix + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 1 + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn stable_transitioned_recursive_prefix_delete_uses_journal_owners() { + let temp_dir = tempfile::tempdir().expect("create stable transitioned prefix-delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stable-transitioned-prefix-delete", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "STABLE-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "stable-transitioned-prefix-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("stable prefix source bucket should be created"); + + let objects = ["prefix/first.bin", "prefix/second.bin"]; + for (index, object) in objects.into_iter().enumerate() { + let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("stable prefix source should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("stable prefix source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("stable prefix source should transition"); + } + assert_eq!(backend.object_count().await, objects.len()); + backend.set_remove_failure(true); + + store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("stable recursive prefix delete should use journal ownership"); + + assert_eq!(tier_delete_journal_count(store.clone()).await, objects.len()); + assert_eq!(backend.object_count().await, objects.len()); + for object in objects { + assert!( + store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("stable prefix metadata lookup should succeed") + .is_none(), + "stable recursive delete must leave neither a visible source nor a free-version for {object}" + ); + } + + backend.set_remove_failure(false); + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("stable prefix journals should recover"); + assert_eq!(stats.failed, 0); + assert!(stats.scanned <= objects.len() && stats.deleted <= objects.len()); + assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), objects.len()); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn authorized_prefix_retry_replays_predecessor_before_newcomer() { + let handle = std::thread::Builder::new() + .name("authorized-prefix-predecessor-replay-test".to_string()) + .stack_size(32 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("authorized predecessor replay test runtime should build"); + runtime.block_on(authorized_prefix_retry_replays_predecessor_before_newcomer_case()); + }) + .expect("authorized predecessor replay test thread should spawn"); + if let Err(payload) = handle.join() { + std::panic::resume_unwind(payload); + } + } + + #[cfg(feature = "test-util")] + async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() { + let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "AUTHORIZED-PREFIX-REPLAY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("authorized replay tier lease should resolve"); + let bucket = "authorized-prefix-predecessor-replay-bucket"; + let prefix = "prefix/"; + let predecessor = "prefix/predecessor.bin"; + let newcomer = "prefix/newcomer.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("authorized replay bucket should be created"); + + let mut predecessor_reader = PutObjReader::from_vec(vec![b'p'; 1024 * 1024]); + let predecessor_source = store + .put_object(bucket, predecessor, &mut predecessor_reader, &ObjectOptions::default()) + .await + .expect("predecessor source should be written"); + store + .transition_object( + bucket, + predecessor, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: predecessor_source.etag.clone().expect("predecessor should have an etag"), + ..Default::default() + }, + mod_time: predecessor_source.mod_time, + ..Default::default() + }, + ) + .await + .expect("predecessor should transition"); + let predecessor_info = store + .get_object_info(bucket, predecessor, &ObjectOptions::default()) + .await + .expect("transitioned predecessor should be readable"); + let mut predecessor_entry = + transitioned_delete_journal_entry_for_source(None, false, false, bucket, predecessor, &predecessor_info) + .expect("transitioned predecessor should produce a dispatch journal"); + predecessor_entry.backend_identity = Some(lease.backend_identity()); + let incarnation = store + .bucket_incarnation_id(bucket) + .await + .expect("authorized replay bucket incarnation should resolve"); + let (manifest_name, _) = install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + prefix, + vec![(predecessor_entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized predecessor fixture should persist"); + + let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]); + let newcomer_source = store + .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default()) + .await + .expect("newcomer source should be written after authorization"); + store + .transition_object( + bucket, + newcomer, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: newcomer_source.etag.clone().expect("newcomer should have an etag"), + ..Default::default() + }, + mod_time: newcomer_source.mod_time, + ..Default::default() + }, + ) + .await + .expect("newcomer should transition"); + backend.set_remove_failure(true); + + let publication_epoch = store + .scanner_data_usage_publication_epoch() + .await + .expect("authorized replay should observe the current scanner publication epoch"); + let publication_scope = store + .scanner_data_usage_publication_commit_scope( + publication_epoch, + tokio::time::Instant::now() + Duration::from_secs(30), + Vec::new(), + ) + .await + .expect("authorized replay should acquire a scanner publication scope"); + + let retry = store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + scanner_publication_commit_scope: Some(publication_scope.clone()), + ..Default::default() + }, + ) + .await + .expect_err("the predecessor-only replay must request a successor retry"); + assert!( + retry.to_string().contains("retry the successor dispatch"), + "unexpected predecessor replay result: {retry}" + ); + assert_eq!( + publication_scope.state(), + crate::object_api::ScannerPublicationCommitState::Committed, + "an exact predecessor replay must report its local mutation as committed" + ); + assert!(publication_scope.release_movement_permit().await); + assert!( + store.pools[0] + .get_disks_by_key(predecessor) + .load_file_info_versions_exact(bucket, predecessor) + .await + .expect("predecessor metadata should remain readable") + .is_none(), + "the authorized predecessor must be removed exactly" + ); + assert!( + store.pools[0] + .get_disks_by_key(newcomer) + .load_file_info_versions_exact(bucket, newcomer) + .await + .expect("newcomer metadata should remain readable") + .is_some(), + "the predecessor authorization must not delete the newcomer" + ); + assert_eq!( + test_tier_delete_dispatch_manifest_state(store.clone(), &manifest_name) + .await + .expect("completed predecessor manifest should remain readable"), + Some(TierDeleteDispatchManifestState::Completed) + ); + + backend.set_remove_failure(false); + let predecessor_journals = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("completed predecessor journal should recover"); + assert_eq!(predecessor_journals.failed, 0); + assert!(predecessor_journals.scanned <= 1 && predecessor_journals.deleted <= 1); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 1, "only the newcomer remote object should remain"); + assert_eq!(backend.remove_versions().await.len(), 1); + let predecessor_manifest = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("completed predecessor manifest should be collected"); + assert_eq!(predecessor_manifest.failed, 0); + assert!(predecessor_manifest.deleted <= 1); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + + store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("a successor dispatch should delete the remaining newcomer"); + assert!( + store.pools[0] + .get_disks_by_key(newcomer) + .load_file_info_versions_exact(bucket, newcomer) + .await + .expect("newcomer metadata lookup should succeed after successor") + .is_none() + ); + drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await; + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), 2); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn multi_pool_same_tuple_recursive_prefix_uses_one_journal_owner() { + let temp_dir = tempfile::tempdir().expect("create shared-tuple prefix-delete store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "multi-pool-shared-tuple-prefix-delete", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "SHARED-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "multi-pool-shared-tuple-prefix-delete-bucket"; + let object = "prefix/shared.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("shared-tuple prefix bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]); + let source = store.pools[0] + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("shared-tuple prefix source should be written"); + store.pools[0] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("shared-tuple prefix source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("shared-tuple prefix source should transition"); + copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await; + assert_eq!(backend.object_count().await, 1); + backend.set_remove_failure(true); + + store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("shared-tuple recursive prefix delete should commit"); + + assert_eq!(tier_delete_journal_count(store.clone()).await, 1); + for pool_idx in 0..2 { + assert!( + store.pools[pool_idx] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("shared-tuple post-delete metadata should decode") + .is_none(), + "pool {pool_idx} must be physically empty" + ); + } + backend.set_remove_failure(false); + ctx.wake_tier_delete_journal_recovery(); + wait_for_tier_delete_journal_recovery(store.clone(), &backend, 1).await; + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), 1); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn multi_pool_recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete() { + let temp_dir = tempfile::tempdir().expect("create merge-loser prefix-delete store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "multi-pool-merge-loser-prefix-delete", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "MERGE-LOSER-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "multi-pool-merge-loser-prefix-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("merge-loser prefix bucket should be created"); + + for (object, byte) in [("legacy/item.bin", b'l'), ("hidden/item.bin", b'h')] { + let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]); + let source = store.pools[0] + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("merge-loser source should be written"); + store.pools[0] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("merge-loser source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("merge-loser source should transition"); + copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await; + } + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await; + backend.set_remove_failure(true); + store.pools[1] + .delete_object(bucket, "hidden/item.bin", ObjectOptions::default()) + .await + .expect("second-pool source delete should retain a hidden free-version"); + + let legacy_error = store + .delete_object_with_tier_delete_journal( + bucket, + "legacy/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a legacy merge loser must reject recursive deletion"); + assert!(legacy_error.to_string().contains("legacy Unknown"), "unexpected error: {legacy_error}"); + let hidden_error = store + .delete_object_with_tier_delete_journal( + bucket, + "hidden/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a hidden free-version merge loser must reject recursive deletion"); + assert!( + hidden_error.to_string().contains("free-version cleanup obligation"), + "unexpected error: {hidden_error}" + ); + + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 2); + assert_eq!(backend.remove_count().await, 0); + for object in ["legacy/item.bin", "hidden/item.bin"] { + let pool0 = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("winner metadata should decode") + .expect("winner source must remain before destructive deletion"); + assert_eq!(pool0.versions.iter().filter(|version| !version.tier_free_version()).count(), 1); + } + let hidden_pool1 = store.pools[1] + .get_disks_by_key("hidden/item.bin") + .load_file_info_versions_exact(bucket, "hidden/item.bin") + .await + .expect("hidden loser metadata should decode") + .expect("hidden loser cleanup owner must remain"); + assert_eq!( + hidden_pool1 + .versions + .iter() + .chain(hidden_pool1.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1 + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn recursive_prefix_partial_pool_failure_keeps_prepared_cleanup_owners() { + let temp_dir = tempfile::tempdir().expect("create partial prefix-delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "partial-pool-prefix-delete", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "PARTIAL-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "partial-pool-prefix-delete-bucket"; + let object = "prefix/item.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("partial prefix bucket should be created"); + for (pool_idx, byte) in [(0, b'a'), (1, b'b')] { + let mut reader = PutObjReader::from_vec(vec![byte; 1024 * 1024]); + let source = store.pools[pool_idx] + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("pool-local prefix source should be written"); + store.pools[pool_idx] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("pool-local source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("pool-local prefix source should transition"); + } + assert_eq!(backend.object_count().await, 2); + backend.set_remove_failure(true); + + let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/"); + let delete_store = store.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused()) + .await + .expect("prefix delete should pause after every journal is Prepared"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 2); + + let failed_set = store.pools[1].get_disks_by_key(object); + let offline_disks = force_set_disks_offline_for_test(&failed_set).await; + barrier.release(); + let error = delete + .await + .expect("partial prefix delete task should join") + .expect_err("an offline second pool must fail the recursive prefix delete"); + assert!( + matches!( + error, + StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _) + ), + "unexpected error: {error:?}" + ); + drop(offline_disks); + drop(barrier); + + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 2, + "an ambiguous partial mutation must retain every Dispatched owner" + ); + let first_pool_state = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("first-pool metadata should remain decodable after an ambiguous failure"); + if let Some(first_pool_state) = first_pool_state { + assert_eq!( + first_pool_state + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 1, + "a retained first-pool source must remain a visible source, not an unjournaled fallback owner" + ); + } + assert!( + store.pools[1] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("failed-pool metadata should decode") + .is_some(), + "the failed pool should retain its live source" + ); + + let _ = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("Dispatched owners should reconcile after the failed pool returns"); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 2, + "recovery must never abort or delete a Dispatched journal while any source is still live" + ); + store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("retry should delete the remaining pool source"); + backend.set_remove_failure(false); + for _ in 0..3 { + let _ = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("remaining pool journals should settle after backend recovery"); + if tier_delete_journal_count(store.clone()).await == 0 { + break; + } + } + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), 2); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn recursive_prefix_partial_set_failure_keeps_prepared_cleanup_owners() { + let temp_dir = tempfile::tempdir().expect("create partial-set prefix-delete store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "partial-set-prefix-delete", + &[(2, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "PARTIAL-SET-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "partial-set-prefix-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("partial-set prefix bucket should be created"); + + let objects = (0..2) + .map(|target_set| { + (0..512) + .map(|index| format!("prefix/set-{target_set}-{index}.bin")) + .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == target_set) + .expect("a deterministic key should map to each test set") + }) + .collect::>(); + for (index, object) in objects.iter().enumerate() { + let mut reader = PutObjReader::from_vec(vec![b'0' + index as u8; 1024 * 1024]); + let source = store.pools[0] + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("set-local prefix source should be written"); + store.pools[0] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("set-local source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("set-local prefix source should transition"); + } + assert_eq!(backend.object_count().await, 2); + backend.set_remove_failure(true); + + let barrier = crate::store::object::PrefixDeleteAfterJournalPrepareBarrier::install(bucket, "prefix/"); + let delete_store = store.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused()) + .await + .expect("set-scoped prefix delete should pause after journal Prepare"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 2); + + let failed_set = store.pools[0].disk_set[1].clone(); + let offline_disks = force_set_disks_offline_for_test(&failed_set).await; + barrier.release(); + let error = delete + .await + .expect("partial-set prefix delete task should join") + .expect_err("an offline second set must fail recursive prefix deletion"); + assert!( + matches!( + error, + StorageError::InsufficientWriteQuorum(_, _) | StorageError::InsufficientReadQuorum(_, _) + ), + "unexpected error: {error:?}" + ); + drop(offline_disks); + drop(barrier); + + assert_eq!(tier_delete_journal_count(store.clone()).await, 2); + let first_set_state = store.pools[0] + .get_disks_by_key(&objects[0]) + .load_file_info_versions_exact(bucket, &objects[0]) + .await + .expect("first-set metadata should remain decodable after an ambiguous failure"); + if let Some(first_set_state) = first_set_state { + assert_eq!( + first_set_state + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 1, + "a retained first-set source must remain visible under its Dispatched journal" + ); + } + assert!( + store.pools[0] + .get_disks_by_key(&objects[1]) + .load_file_info_versions_exact(bucket, &objects[1]) + .await + .expect("failed-set metadata should decode") + .is_some() + ); + + let _ = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("set-scoped Dispatched owners should reconcile"); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 2, + "recovery must retain the complete Authorized dispatch until every source is absent" + ); + store + .delete_object_with_tier_delete_journal( + bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("set-scoped retry should delete the remaining source"); + backend.set_remove_failure(false); + for _ in 0..3 { + let _ = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("remaining set journals should settle after backend recovery"); + if tier_delete_journal_count(store.clone()).await == 0 { + break; + } + } + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.remove_versions().await.len(), 2); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete() { + let temp_dir = tempfile::tempdir().expect("create versioned transitioned delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-transitioned-delete-owner", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "VERSIONED-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "versioned-transitioned-delete-owner-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("versioned source bucket should be created"); + crate::bucket::metadata_sys::update_in( + &ctx, + bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("bucket versioning should be enabled"); + + let mut first_reader = PutObjReader::from_vec(vec![b'1'; 1024 * 1024]); + let first = store + .put_object( + bucket, + object, + &mut first_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("first version should be written"); + let first_version = first.version_id.expect("versioned PUT should return an identity"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + version_id: Some(first_version.to_string()), + versioned: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: first.etag.clone().expect("first version should have an etag"), + ..Default::default() + }, + mod_time: first.mod_time, + ..Default::default() + }, + ) + .await + .expect("first version should transition"); + let mut second_reader = PutObjReader::from_vec(vec![b'2'; 1024 * 1024]); + let second = store + .put_object( + bucket, + object, + &mut second_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("second version should be written"); + let second_version = second.version_id.expect("second PUT should return an identity"); + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + version_id: Some(first_version.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("explicit transitioned version delete should commit"); + store + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: Some(second_version.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the unrelated second version must remain readable"); + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("remaining version metadata should decode") + .expect("the second version should remain"); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "ordinary exact-version delete must leave one hidden free-version owner" + ); + assert!( + exact + .versions + .iter() + .any(|version| version.version_id == Some(second_version)), + "the explicit delete must not remove the other version" + ); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary exact-version delete must not create a journal" + ); + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-cleanup metadata should decode") + .expect("the unrelated second version should remain after cleanup"); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 0, + "free-version recovery must remove only its exact owner" + ); + assert!( + exact + .versions + .iter() + .any(|version| version.version_id == Some(second_version)), + "free-version recovery must preserve the unrelated version" + ); + + store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + version_id: Some(second_version.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("last ordinary version should be removed"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("bucket delete should succeed after the last version is removed"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn suspended_null_transition_delete_uses_free_version_as_sole_owner() { + let temp_dir = tempfile::tempdir().expect("create suspended transitioned delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "suspended-transitioned-delete-owner", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "SUSPENDED-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "suspended-transitioned-delete-owner-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("suspended source bucket should be created"); + crate::bucket::metadata_sys::update_in( + &ctx, + bucket, + BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec(), + ) + .await + .expect("bucket versioning should be suspended"); + + let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]); + let source = store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect("suspended null source should be written"); + let null_version = source.version_id.expect("suspended source should expose the null identity"); + assert!(null_version.is_nil(), "suspended source identity must be null"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + version_id: Some(null_version.to_string()), + version_suspended: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("suspended source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("suspended null source should transition"); + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + version_id: Some(null_version.to_string()), + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect("explicit null transitioned version delete should commit"); + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-delete null metadata should decode") + .expect("ordinary null-version delete must retain a cleanup owner"); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "ordinary null-version delete must retain exactly one hidden free-version owner" + ); + assert_eq!( + exact.versions.iter().filter(|version| !version.tier_free_version()).count(), + 0, + "ordinary null-version delete must remove the visible source" + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("suspended bucket should be empty after exact null-version cleanup"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn versioned_delete_marker_keeps_transitioned_source_and_remote_object() { + let temp_dir = tempfile::tempdir().expect("create versioned delete-marker store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-marker", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "DELETE-MARKER"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "transitioned-delete-marker-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("versioned source bucket should be created"); + crate::bucket::metadata_sys::update_in( + &ctx, + bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("bucket versioning should be enabled"); + + let mut reader = PutObjReader::from_vec(vec![b'm'; 1024 * 1024]); + let source = store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("versioned source should be written"); + let source_version = source.version_id.expect("versioned source should have an identity"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + version_id: Some(source_version.to_string()), + versioned: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("versioned source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("versioned source should transition"); + + let marker = store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("versioned DELETE without versionId should create a marker"); + assert!(marker.delete_marker); + let marker_version = marker.version_id.expect("delete marker should have an identity"); + assert_eq!(backend.object_count().await, 1, "delete-marker creation must retain the remote object"); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "delete-marker creation must not schedule cleanup" + ); + let exact = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("versioned delete-marker metadata should decode") + .expect("source and delete marker should remain on disk"); + assert!( + exact.versions.iter().any(|version| { + version.version_id == Some(source_version) && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE + }), + "the transitioned source must remain behind the delete marker" + ); + assert_eq!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 0, + "delete-marker creation must not create a cleanup free-version" + ); + + store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + version_id: Some(marker_version.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("delete marker should be removable by exact identity"); + store + .delete_object_with_tier_delete_journal( + bucket, + object, + ObjectOptions { + version_id: Some(source_version.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("transitioned source should be removable by exact identity"); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + if let Some(exact) = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-recovery version metadata should decode") + { + assert!( + exact + .versions + .iter() + .chain(exact.free_versions.iter()) + .all(|version| !version.tier_free_version()), + "free-version recovery must remove the all-physical cleanup owner" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("versioned source bucket should be empty after explicit cleanup"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn batch_transitioned_delete_uses_free_version_per_item() { + let temp_dir = tempfile::tempdir().expect("create batch transitioned delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-delete-owner", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "BATCH-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "batch-transitioned-delete-owner-bucket"; + let transitioned = "transitioned.bin"; + let ordinary = "ordinary.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("batch source bucket should be created"); + let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]); + let transitioned_source = store + .put_object(bucket, transitioned, &mut transitioned_reader, &ObjectOptions::default()) + .await + .expect("transitioned batch source should be written"); + store + .transition_object( + bucket, + transitioned, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: transitioned_source + .etag + .clone() + .expect("transitioned batch source should have an etag"), + ..Default::default() + }, + mod_time: transitioned_source.mod_time, + ..Default::default() + }, + ) + .await + .expect("batch source transition should commit"); + let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]); + store + .put_object(bucket, ordinary, &mut ordinary_reader, &ObjectOptions::default()) + .await + .expect("ordinary batch source should be written"); + + backend.set_remove_failure(true); + let (_deleted, errors) = store + .delete_objects_with_tier_delete_journal( + bucket, + vec![ + ObjectToDelete { + object_name: transitioned.to_string(), + ..Default::default() + }, + ObjectToDelete { + object_name: ordinary.to_string(), + ..Default::default() + }, + ObjectToDelete { + object_name: "missing.bin".to_string(), + ..Default::default() + }, + ], + ObjectOptions::default(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "S3 batch delete should be idempotent: {errors:?}"); + let transitioned_versions = store.pools[0] + .get_disks_by_key(transitioned) + .load_file_info_versions_exact(bucket, transitioned) + .await + .expect("transitioned batch metadata should remain decodable") + .expect("transitioned batch item must retain a cleanup owner"); + assert_eq!( + transitioned_versions + .versions + .iter() + .chain(transitioned_versions.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "transitioned batch item must retain exactly one hidden free-version owner" + ); + assert_eq!( + transitioned_versions + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "transitioned batch item must have no visible source" + ); + assert!( + store.pools[0] + .get_disks_by_key(ordinary) + .load_file_info_versions_exact(bucket, ordinary) + .await + .expect("ordinary batch metadata should remain decodable") + .is_none(), + "ordinary batch item should have no local metadata" + ); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary batch delete must not create a tier-delete journal" + ); + assert_eq!(backend.object_count().await, 1, "failed remote cleanup must retain its object"); + + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + assert!( + store.pools[0] + .get_disks_by_key(transitioned) + .load_file_info_versions_exact(bucket, transitioned) + .await + .expect("cleaned batch metadata should remain decodable") + .is_none(), + "free-version recovery must remove the exact cleanup owner" + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("batch source bucket should be physically empty"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn transitioned_delete_free_version_replays_after_store_restart() { + let temp_dir = tempfile::tempdir().expect("create restarted free-version store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-before", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "DELETE-RESTART"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let tier_config = ctx + .tier_config_mgr() + .read() + .await + .tiers + .get(tier_name) + .expect("mock tier config should exist") + .clone_with_credentials(); + let bucket = "transitioned-delete-restart-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("restart source bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b'r'; 1024 * 1024]); + let original = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("restart source object should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: original.etag.clone().expect("restart source should have an etag"), + ..Default::default() + }, + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("restart source transition should commit"); + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default()) + .await + .expect("local source delete should commit while remote cleanup is retryable"); + let retained = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-delete local metadata should decode") + .expect("ordinary delete must retain a free-version across restart"); + assert_eq!( + retained + .versions + .iter() + .chain(retained.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "ordinary delete must retain exactly one free-version across restart" + ); + assert_eq!( + retained + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "ordinary delete must remove the visible source before restart" + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 1); + + shutdown.cancel(); + drop(store); + drop(ctx); + let (restarted_ctx, restarted_store, restarted_shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-restart-after", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + { + let restarted_tier_config_mgr = restarted_ctx.tier_config_mgr(); + let mut manager = restarted_tier_config_mgr.write().await; + manager.tiers.insert(tier_name.to_string(), tier_config); + manager + .install_test_driver(tier_name, Box::new(backend.clone())) + .expect("restarted store should bind the original tier destination"); + } + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(restarted_store.clone(), &backend, 1).await; + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert!( + restarted_store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("restarted cleanup metadata should decode") + .is_none(), + "restarted recovery must remove the exact free-version owner" + ); + restarted_store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("source bucket should be empty after restarted free-version recovery"); + restarted_shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn transitioned_delete_local_quorum_failure_rolls_back_without_cleanup_owner() { + let temp_dir = tempfile::tempdir().expect("create failed local delete store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-local-failure", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "DELETE-LOCAL-FAIL"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "transitioned-delete-local-failure-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("failed-delete source bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b'q'; 1024 * 1024]); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("failed-delete source should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("failed-delete source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("failed-delete source should transition"); + + let set = store.pools[0].get_disks_by_key(object); + let saved_disk = { + let mut disks = set.disks.write().await; + let first = disks[0].as_ref().expect("first disk should be online"); + crate::disk::local::set_delete_version_fail_after_commit(first.path().as_path(), object); + disks[1].take().expect("second disk should be online before fault injection") + }; + let err = store + .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default()) + .await + .expect_err("one post-commit error plus one offline disk must miss delete quorum"); + assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}"); + assert!( + err.to_string() + .contains("Storage resources are insufficient for the write operation"), + "the caller must observe the wrapped quorum failure: {err}" + ); + set.disks.write().await[1] = Some(saved_disk); + + let retained = store + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("failed local delete must roll back the transitioned source"); + assert_eq!(retained.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE); + let retained_versions = set + .load_file_info_versions_exact(bucket, object) + .await + .expect("rolled-back metadata should decode") + .expect("rolled-back source should remain on disk"); + assert_eq!( + retained_versions + .versions + .iter() + .chain(retained_versions.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 0, + "failed local quorum must not create a cleanup owner" + ); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary delete must not create a journal, including on rollback" + ); + assert_eq!(backend.object_count().await, 1, "failed local commit must retain the remote object"); + assert_eq!(backend.remove_count().await, 0, "failed local commit must not dispatch remote cleanup"); + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default()) + .await + .expect("retry after disk recovery should commit"); + let cleanup_owner = set + .load_file_info_versions_exact(bucket, object) + .await + .expect("retry metadata should decode") + .expect("successful retry must leave a free-version owner"); + assert_eq!( + cleanup_owner + .versions + .iter() + .chain(cleanup_owner.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "successful retry must leave exactly one free-version owner" + ); + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("retry should leave the source bucket empty"); + } + + #[cfg(feature = "test-util")] + async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) { + let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir"); + let case = if batch { "batch" } else { "single" }; + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + &format!("multi-pool-same-tuple-{case}"), + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = if batch { "SAME-TUPLE-BATCH" } else { "SAME-TUPLE-SINGLE" }; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = format!("multi-pool-same-tuple-{case}-bucket"); + let object = "archive.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("shared-tuple bucket should be created"); + let mut reader = PutObjReader::from_vec(vec![b's'; 1024 * 1024]); + let source = store.pools[0] + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("first pool source should be written"); + store.pools[0] + .transition_object( + &bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("shared source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("first pool source should transition"); + let first = store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("first pool transitioned source should be readable"); + + // Model the exact metadata duplication produced by data movement: + // both pools refer to the same remote object/version/backend and carry + // the same stable source identity. + for disk_index in 0..4 { + let source_meta = temp_dir + .path() + .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); + let target_meta = temp_dir + .path() + .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); + tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent")) + .await + .expect("second pool object directory should be created"); + tokio::fs::copy(&source_meta, &target_meta) + .await + .expect("transitioned xl.meta should copy exactly to the second pool"); + } + let second = store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("copied second-pool source should be readable"); + assert_eq!( + (first.transitioned_object.name.clone(), first.transitioned_object.version_id.clone()), + (second.transitioned_object.name.clone(), second.transitioned_object.version_id.clone()) + ); + assert_eq!(backend.object_count().await, 1); + + let mut delete_opts = ObjectOptions { + tier_delete_journal_api: Some(store.clone()), + ..Default::default() + }; + if batch { + delete_opts.delete_replication_config_snapshot = Some(Arc::new( + ReplicationObjectBridge::delete_request_config_in(&store.ctx, &bucket) + .await + .expect("batch delete replication snapshot should load"), + )); + let (_deleted, errors, _accounting) = store.pools[0] + .delete_objects_with_accounting( + &bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + delete_opts.clone(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "first pool batch delete should commit: {errors:?}"); + } else { + store.pools[0] + .delete_object(&bucket, object, delete_opts.clone()) + .await + .expect("first pool single delete should commit"); + } + + let first_pool_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("first-pool metadata should decode") + .expect("first pool must retain a free-version owner"); + assert_eq!( + first_pool_versions + .versions + .iter() + .chain(first_pool_versions.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "first pool must retain exactly one free-version owner" + ); + assert_eq!( + first_pool_versions + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "first pool must remove its visible source" + ); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary pool-local delete must not create a journal" + ); + + ExpiryState::resize_workers(1, store.clone()).await; + let first_recovery = recover_tier_free_versions(store.clone(), 100, None, None) + .await + .expect("first-pool free-version should be recoverable"); + assert_eq!( + (first_recovery.enqueued, first_recovery.failed), + (0, 0), + "the live second-pool source is the merged walk winner, so its hidden loser must not dispatch cleanup" + ); + wait_for_expiry_workers_idle(&store).await; + assert_eq!(backend.remove_count().await, 0, "live second-pool source must gate remote DELETE"); + assert_eq!(backend.object_count().await, 1, "live source must retain its remote body"); + assert!( + store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("blocked cleanup metadata should decode") + .is_some(), + "blocked cleanup must retain the first-pool free-version" + ); + let surviving = store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("second-pool source must remain GET-addressable"); + assert_eq!(surviving.transitioned_object.name, first.transitioned_object.name); + + backend.set_remove_failure(true); + if batch { + let (_deleted, errors, _accounting) = store.pools[1] + .delete_objects_with_accounting( + &bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + delete_opts, + ) + .await; + assert!(errors.iter().all(Option::is_none), "second pool batch retry should commit: {errors:?}"); + } else { + store.pools[1] + .delete_object(&bucket, object, delete_opts) + .await + .expect("second pool single retry should commit"); + } + wait_for_expiry_workers_idle(&store).await; + let mut free_version_ids = Vec::new(); + for pool_idx in 0..2 { + let retained = store.pools[pool_idx] + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("shared-tuple free-version metadata should decode") + .expect("remote failure must retain every shared-tuple cleanup owner"); + let owners = retained + .versions + .iter() + .chain(retained.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .collect::>(); + assert_eq!(owners.len(), 1, "pool {pool_idx} must retain one free-version owner"); + free_version_ids.push(owners[0].version_id); + } + assert_ne!( + free_version_ids[0], free_version_ids[1], + "ordinary pool-local deletes must model distinct local free-version identities" + ); + + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await; + assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once"); + for pool_idx in 0..2 { + assert!( + store.pools[pool_idx] + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("post-delete metadata should decode") + .is_none(), + "pool {pool_idx} must not retain source or free-version metadata" + ); + } + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("shared-tuple bucket should be physically empty"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn multi_pool_same_remote_tuple_single_delete_waits_for_all_sources() { + run_multi_pool_same_remote_tuple_delete_case(false).await; + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn multi_pool_same_remote_tuple_batch_delete_waits_for_all_sources() { + run_multi_pool_same_remote_tuple_delete_case(true).await; + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn multi_pool_transitioned_delete_persists_one_free_version_per_remote_tuple() { + let temp_dir = tempfile::tempdir().expect("create multi-pool transitioned delete store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "multi-pool-transitioned-delete-owner", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "MULTIPOOL-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "multi-pool-transitioned-delete-owner-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("multi-pool source bucket should be created"); + + let mut remote_tuples = Vec::new(); + for pool_idx in 0..2 { + let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]); + let source = store.pools[pool_idx] + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(pool_idx as i64)), + ..Default::default() + }, + ) + .await + .expect("pool-local source should be written"); + store.pools[pool_idx] + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("pool-local source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("pool-local source should transition"); + let transitioned = store.pools[pool_idx] + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("pool-local transitioned source should be readable"); + remote_tuples.push(( + transitioned.transitioned_object.name.clone(), + transitioned.transitioned_object.version_id.clone(), + )); + } + assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples"); + assert_eq!(backend.object_count().await, 2); + + backend.set_remove_failure(true); + store + .delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default()) + .await + .expect("store-level delete should remove both pool copies"); + for pool_idx in 0..2 { + let retained = store.pools[pool_idx] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("pool-local post-delete metadata should decode") + .expect("each pool-specific remote tuple must retain a cleanup owner"); + assert_eq!( + retained + .versions + .iter() + .chain(retained.free_versions.iter()) + .filter(|version| version.tier_free_version()) + .count(), + 1, + "pool {pool_idx} must retain exactly one free-version owner" + ); + assert_eq!( + retained + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "pool {pool_idx} must not retain a visible source" + ); + } + assert_eq!( + tier_delete_journal_count(store.clone()).await, + 0, + "ordinary multi-pool delete must not create tier-delete journals" + ); + assert_eq!(backend.object_count().await, 2, "failed remote cleanup must retain both remote objects"); + + backend.set_remove_failure(false); + wait_for_tier_free_version_recovery(store.clone(), &backend, 2).await; + let mut removed = backend.remove_versions().await; + removed.sort(); + remote_tuples.sort(); + assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple"); + for pool_idx in 0..2 { + assert!( + store.pools[pool_idx] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("post-recovery pool metadata should decode") + .is_none(), + "pool {pool_idx} must not retain source or free-version metadata" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("multi-pool source bucket should be physically empty"); + } + // Phase 5 follow-up (backlog#1052): building a real store through the // ctx-explicit constructor lands every construction-time write — object // graph adoption, local-disk registry, deployment id — on the passed @@ -9202,27 +14192,16 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn tier_delete_journal_recovery_spawns_for_each_store() { + async fn v4_tier_delete_journal_recovery_is_exact_and_store_scoped() { let temp_a = tempfile::tempdir().expect("create temp store dir a"); let temp_b = tempfile::tempdir().expect("create temp store dir b"); - let (ctx_a, store_a, shutdown_a) = + let (ctx_a, store_a, _shutdown_a) = without_storage_class_env(build_isolated_test_store(temp_a.path(), "tier-journal-recovery-a", &[4])).await; - let (ctx_b, store_b, shutdown_b) = + let (ctx_b, store_b, _shutdown_b) = without_storage_class_env(build_isolated_test_store(temp_b.path(), "tier-journal-recovery-b", &[4])).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await; - assert!( - !ctx_a.mark_tier_delete_journal_recovery_started(store_a.id), - "store A should have claimed its production recovery worker" - ); - assert!( - !ctx_b.mark_tier_delete_journal_recovery_started(store_b.id), - "store B should have claimed its production recovery worker" - ); - assert!(!shutdown_a.is_cancelled()); - assert!(!shutdown_b.is_cancelled()); - let tier_a = "JOURNAL-A"; let tier_b = "JOURNAL-B"; let backend_a = register_mock_tier(&ctx_a.tier_config_mgr(), tier_a).await; @@ -9236,6 +14215,7 @@ mod tests { .expect("store B tier lease should resolve") .backend_identity(); let entry_a = Jentry { + persisted_version: 0, obj_name: "remote-a".to_string(), version_id: "version-a".to_string(), tier_name: tier_a.to_string(), @@ -9244,8 +14224,10 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; let entry_b = Jentry { + persisted_version: 0, obj_name: "remote-b".to_string(), version_id: "version-b".to_string(), tier_name: tier_b.to_string(), @@ -9254,68 +14236,238 @@ mod tests { version_state: rustfs_filemeta::TransitionVersionState::Exact, state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed, source: None, + dispatch: None, }; - let remove_a = backend_a.arm_failing_remove_barrier().await; persist_tier_delete_journal_entry(store_a.clone(), &entry_a) .await - .expect("store A journal should persist"); + .expect("store A v4 compatibility fixture should persist"); persist_tier_delete_journal_entry(store_b.clone(), &entry_b) .await - .expect("store B journal should persist"); - - ctx_a.wake_tier_delete_journal_recovery(); - ctx_b.wake_tier_delete_journal_recovery(); - remove_a.wait_until_paused().await; - wait_for_tier_delete_journal_recovery(store_b.clone(), &backend_b, 1).await; - - shutdown_a.cancel(); - remove_a.wait_until_operation_dropped().await; - assert!( - ctx_a - .background_cancel_token() - .expect("store A shutdown token should be bound") - .is_cancelled() - ); - assert!( - !ctx_b - .background_cancel_token() - .expect("store B shutdown token should be bound") - .is_cancelled(), - "cancelling store A must not stop store B" - ); - assert_eq!(tier_delete_journal_count(store_a.clone()).await, 1); - + .expect("store B v4 compatibility fixture should persist"); + let entry_a_v3 = Jentry { + obj_name: "remote-a-v3".to_string(), + version_id: "version-a-v3".to_string(), + ..entry_a.clone() + }; + let mut raw_v3: serde_json::Value = serde_json::from_slice( + &encode_tier_delete_journal_entry(&entry_a_v3).expect("store A v3 fixture should encode from the v4 shape"), + ) + .expect("store A v3 fixture should be JSON"); + raw_v3["version"] = serde_json::json!(3); + raw_v3 + .as_object_mut() + .expect("store A v3 fixture should be an object") + .remove("version_state"); + let path_a_v3 = tier_delete_journal_object_name(&entry_a_v3); + com::save_config( + store_a.clone(), + &path_a_v3, + serde_json::to_vec(&raw_v3).expect("store A raw v3 fixture should encode"), + ) + .await + .expect("store A raw v3 compatibility fixture should persist"); + let path_a = tier_delete_journal_object_name(&entry_a); + let path_b = tier_delete_journal_object_name(&entry_b); let recovered_a = recover_tier_delete_journal_entries(store_a.clone(), 100, None) .await - .expect("the cancelled store A worker must leave its journal recoverable"); - assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (1, 1, 0)); - assert_eq!(backend_a.remove_versions().await, vec![("remote-a".to_string(), "version-a".to_string())]); - - let second_entry_b = Jentry { - obj_name: "remote-b-2".to_string(), - version_id: "version-b-2".to_string(), - ..entry_b - }; - persist_tier_delete_journal_entry(store_b.clone(), &second_entry_b) + .expect("store A recovery scan should finish"); + let recovered_b = recover_tier_delete_journal_entries(store_b.clone(), 100, None) .await - .expect("store B second journal should persist"); - ctx_b.wake_tier_delete_journal_recovery(); - wait_for_tier_delete_journal_recovery(store_b.clone(), &backend_b, 2).await; + .expect("store B recovery scan should finish"); + assert_eq!((recovered_a.scanned, recovered_a.deleted, recovered_a.failed), (2, 2, 0)); + assert_eq!((recovered_b.scanned, recovered_b.deleted, recovered_b.failed), (1, 1, 0)); + let mut removed_a = backend_a.remove_versions().await; + removed_a.sort(); assert_eq!( - backend_b.remove_versions().await, + removed_a, vec![ - ("remote-b".to_string(), "version-b".to_string()), - ("remote-b-2".to_string(), "version-b-2".to_string()), + ("remote-a".to_string(), "version-a".to_string()), + ("remote-a-v3".to_string(), "version-a-v3".to_string()), ] ); - - shutdown_b.cancel(); + assert_eq!(backend_b.remove_versions().await, vec![("remote-b".to_string(), "version-b".to_string())]); + assert!(matches!(com::read_config(store_a.clone(), &path_a).await, Err(Error::ConfigNotFound))); + assert!(matches!(com::read_config(store_a.clone(), &path_a_v3).await, Err(Error::ConfigNotFound))); + assert!(matches!(com::read_config(store_b.clone(), &path_b).await, Err(Error::ConfigNotFound))); + assert_eq!(tier_delete_journal_count(store_a).await, 0); + assert_eq!(tier_delete_journal_count(store_b).await, 0); } #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn prepared_tier_delete_recovery_finds_directory_source_on_encoded_set() { + async fn v5_tier_delete_recovery_retains_live_sources_and_aborts_only_prepared() { + let temp_dir = tempfile::tempdir().expect("create v5 live-source store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-live-source-recovery", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "V5-LIVE-SOURCE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("v5 live-source tier lease should resolve") + .backend_identity(); + let bucket = "v5-live-source-recovery-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("v5 live-source bucket should be created"); + + let mut paths = Vec::new(); + for (index, state) in [ + TierDeleteJournalState::Prepared, + TierDeleteJournalState::Dispatched, + TierDeleteJournalState::Committed, + ] + .into_iter() + .enumerate() + { + let object = format!("live-{index}.bin"); + let remote_version = uuid::Uuid::new_v4().to_string(); + backend.set_put_remote_version(Some(remote_version)).await; + let mut reader = PutObjReader::from_vec(format!("v5 live source {index}").into_bytes()); + let source = store + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("v5 live source should be written"); + store + .transition_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("v5 live source should have an ETag"), + ..Default::default() + }, + version_id: source.version_id.map(|version| version.to_string()), + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("v5 live source should transition"); + let transitioned = store + .get_object_info( + bucket, + &object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + ) + .await + .expect("v5 transitioned source should be readable"); + let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &transitioned) + .expect("v5 transitioned source should produce a stable journal identity"); + entry.persisted_version = 5; + entry.backend_identity = Some(backend_identity); + entry.state = state; + entry.dispatch = None; + let path = tier_delete_journal_object_name(&entry); + com::save_config( + store.clone(), + &path, + encode_tier_delete_journal_entry(&entry).expect("raw v5 live-source journal should encode"), + ) + .await + .expect("raw v5 live-source journal should persist"); + paths.push((state, path)); + } + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("v5 live-source recovery should finish"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 1, 2)); + assert_eq!(backend.remove_count().await, 0, "a live source must block every remote DELETE"); + for (state, path) in paths { + let observed = com::read_config(store.clone(), &path).await; + if state == TierDeleteJournalState::Prepared { + assert!(matches!(observed, Err(Error::ConfigNotFound)), "Prepared v5 evidence should abort"); + } else { + assert!(observed.is_ok(), "{state:?} v5 evidence must remain while its exact source exists"); + } + } + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn v5_tier_delete_recovery_replays_all_crash_states_without_sources() { + let temp_dir = tempfile::tempdir().expect("create v5 crash-replay store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v5-crash-replay", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "V5-CRASH-REPLAY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("v5 crash-replay tier lease should resolve"); + let backend_identity = lease.backend_identity(); + let bucket = "v5-crash-replay-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("v5 crash-replay bucket should be created"); + + for (index, state, seed_remote) in [ + (0, TierDeleteJournalState::Prepared, true), + (1, TierDeleteJournalState::Dispatched, true), + (2, TierDeleteJournalState::Committed, true), + // Models a restart after exact remote DELETE succeeded but before + // the local Committed journal was removed. + (3, TierDeleteJournalState::Committed, false), + ] { + let remote_version = uuid::Uuid::new_v4().to_string(); + let mut entry = synthetic_v6_dispatch_entry( + bucket, + &format!("missing-{index}.bin"), + tier_name, + backend_identity, + &remote_version, + ); + entry.persisted_version = 5; + entry.state = state; + entry.dispatch = None; + if seed_remote { + backend.set_put_remote_version(Some(remote_version)).await; + let body = bytes::Bytes::from(format!("v5 remote candidate {index}")); + lease + .put( + &entry.obj_name, + ReaderImpl::Body(body.clone()), + i64::try_from(body.len()).expect("v5 candidate length should fit i64"), + ) + .await + .expect("v5 remote candidate should be seeded"); + } + let path = tier_delete_journal_object_name(&entry); + com::save_config( + store.clone(), + &path, + encode_tier_delete_journal_entry(&entry).expect("raw v5 crash journal should encode"), + ) + .await + .expect("raw v5 crash journal should persist"); + } + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("v5 crash-state recovery should finish"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (4, 4, 0)); + assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.exact_remove_count(), 4, "every v5 state must use exact-version cleanup"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn dispatched_tier_delete_recovery_finds_directory_source_on_encoded_set() { let temp_dir = tempfile::tempdir().expect("create temp store dir"); let shutdown = CancellationToken::new(); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( @@ -9386,16 +14538,27 @@ mod tests { let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed) .expect("transitioned source should produce a prepared journal entry"); entry.backend_identity = Some(backend_identity); - persist_tier_delete_journal_entry(store.clone(), &entry) + let incarnation = store + .bucket_incarnation_id(bucket) .await - .expect("prepared journal should persist"); + .expect("bucket incarnation should resolve"); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + &object, + vec![(entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized dispatched journal should persist"); let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) .await .expect("prepared recovery should complete"); - assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); - assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1)); + assert_eq!(tier_delete_journal_count(store).await, 1); assert_eq!(backend.remove_count().await, 0); assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object"); } @@ -9403,7 +14566,7 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn prepared_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() { + async fn dispatched_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() { let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir"); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await; @@ -9449,20 +14612,31 @@ mod tests { let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed) .expect("transitioned source should produce a prepared journal"); entry.backend_identity = Some(backend_identity); - persist_tier_delete_journal_entry(store.clone(), &entry) + let incarnation = store + .bucket_incarnation_id(bucket) .await - .expect("prepared journal should persist"); + .expect("bucket incarnation should resolve"); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + object, + vec![(entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized dispatched journal should persist"); let retained = recover_tier_delete_journal_entries(store.clone(), 100, None) .await .expect("recovery should scan the later pool"); - assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 1, 0)); + assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1)); assert_eq!(backend.remove_count().await, 0); assert_eq!(backend.object_count().await, 1); assert_eq!( tier_delete_journal_count(store.clone()).await, - 0, - "live source should abort its prepared journal" + 1, + "an authorized journal must remain while its live source is durable" ); store.pools[1] @@ -9472,18 +14646,26 @@ mod tests { ObjectOptions { version_id: committed.version_id.map(|version| version.to_string()), expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true }, + skip_free_version: true, ..Default::default() }, ) .await .expect("source version should be removed before recovery retry"); - persist_tier_delete_journal_entry(store.clone(), &entry) + let committed_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) .await - .expect("prepared journal should persist for the absent source"); + .expect("journal recovery should commit an absent stable source"); + assert_eq!((committed_stats.scanned, committed_stats.deleted, committed_stats.failed), (1, 0, 1)); + assert_eq!(backend.remove_count().await, 0); + + let manifest_stats = recover_tier_delete_dispatch_manifests(store.clone(), 100, None) + .await + .expect("manifest recovery should authorize committed remote cleanup"); + assert_eq!((manifest_stats.advanced, manifest_stats.failed), (1, 0)); let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None) .await - .expect("recovery should commit an absent stable source"); + .expect("completed journal should delete the exact remote version"); assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0)); assert_eq!(tier_delete_journal_count(store).await, 0); assert_eq!(backend.remove_count().await, 1); @@ -9493,7 +14675,7 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn prepared_tier_delete_recovery_retains_journal_on_source_metadata_error() { + async fn dispatched_tier_delete_recovery_retains_journal_on_source_metadata_error() { let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir"); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await; @@ -9539,9 +14721,20 @@ mod tests { let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed) .expect("transitioned source should produce a prepared journal"); entry.backend_identity = Some(backend_identity); - persist_tier_delete_journal_entry(store.clone(), &entry) + let incarnation = store + .bucket_incarnation_id(bucket) .await - .expect("prepared journal should persist"); + .expect("bucket incarnation should resolve"); + install_test_tier_delete_dispatch_fixture( + store.clone(), + bucket, + incarnation, + object, + vec![(entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized dispatched journal should persist"); for disk_index in 0..4 { let metadata_path = temp_dir @@ -9557,7 +14750,7 @@ mod tests { .expect("recovery scan should complete despite the entry failure"); assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1)); - assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain prepared for retry"); + assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry"); assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion"); assert_eq!(backend.object_count().await, 1, "remote source must remain intact"); } @@ -9582,6 +14775,7 @@ mod tests { .await .expect("remote body should be seeded"); let entry = Jentry { + persisted_version: 0, obj_name: "remote/original".to_string(), version_id: remote_version, tier_name: tier_name.to_string(), @@ -9599,11 +14793,19 @@ mod tests { etag: Some("etag".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()), }), + dispatch: None, }; - persist_tier_delete_journal_entry(store.clone(), &entry) - .await - .expect("prepared journal should persist"); - let journal_name = tier_delete_journal_object_name(&entry); + let (_manifest_name, bound) = install_test_tier_delete_dispatch_fixture( + store.clone(), + "absent-source-bucket", + uuid::Uuid::new_v4(), + "absent-source-object", + vec![(entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized dispatched journal should persist"); + let journal_name = tier_delete_journal_object_name(&bound[0]); let data = com::read_config(store.clone(), &journal_name) .await .expect("prepared journal should be readable"); @@ -9630,7 +14832,7 @@ mod tests { #[cfg(feature = "test-util")] #[test] #[serial_test::serial(storage_class_env)] - fn transitioned_history_expiry_journals_real_source_without_free_version() { + fn transitioned_history_expiry_uses_journal_or_legacy_free_version() { std::thread::Builder::new() .name("transitioned-delete-all-test".to_string()) .stack_size(32 * 1024 * 1024) @@ -9654,14 +14856,16 @@ mod tests { .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("bucket should be created"); - crate::bucket::metadata_sys::update( + crate::bucket::metadata_sys::update_in( + &ctx, bucket, BUCKET_VERSIONING_CONFIG, b"Enabled".to_vec(), ) .await .expect("bucket versioning should be enabled"); - crate::bucket::metadata_sys::update( + crate::bucket::metadata_sys::update_in( + &ctx, bucket, BUCKET_LIFECYCLE_CONFIG, br#" @@ -9726,8 +14930,8 @@ mod tests { .await .expect("historical version should transition"); assert_eq!(backend.object_count().await, 1); - let transitioned_remote_versions = backend.put_versions().await; - assert_eq!(transitioned_remote_versions.len(), 1); + assert_eq!(backend.put_versions().await.len(), 1); + backend.set_remove_failure(true); let incarnation = store .bucket_incarnation_id_from_disk(bucket) @@ -9764,60 +14968,101 @@ mod tests { rule_id: "delete-all-versions".to_string(), ..Default::default() }; - let rejected = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects( - store.clone(), - ¤t, - &lifecycle_event, - &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner, - incarnation, - ) - .await; - assert!(!rejected, "legacy unknown transition identity must fail before local mutation"); + let legacy_applied = + crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects( + store.clone(), + ¤t, + &lifecycle_event, + &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner, + incarnation, + ) + .await; + assert!(legacy_applied, "legacy Unknown transition metadata should use the free-version fallback"); assert_eq!(tier_delete_journal_count(store.clone()).await, 0); assert_eq!(backend.remove_count().await, 0); let retained = store.pools[0].disk_set[0] .load_file_info_versions_exact(bucket, object) .await - .expect("rejected delete-all metadata should remain readable") - .expect("rejected delete-all should retain both versions"); + .expect("fallback delete-all metadata should remain readable") + .expect("fallback delete-all should retain a hidden cleanup owner"); assert_eq!( retained .versions .iter() .filter(|version| !version.tier_free_version()) .count(), - 2 + 0 ); + assert_eq!( + retained.versions.iter().filter(|version| version.tier_free_version()).count(), + 1, + "legacy cleanup must retain exactly one hidden free-version" + ); + assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending"); - for disk_index in 0..4 { - let metadata_path = temp_dir - .path() - .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")); - let encoded = tokio::fs::read(&metadata_path) - .await - .expect("unknown transition metadata should be readable"); - let mut metadata = FileMeta::load(&encoded).expect("unknown transition metadata should decode"); - let mut transitioned = metadata - .get_all_file_info_versions(bucket, object, true) - .expect("unknown transition versions should decode") - .versions - .into_iter() - .find(|version| version.version_id == history.version_id) - .expect("unknown transitioned history should exist"); - transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; - metadata - .add_version(transitioned) - .expect("exact state should replace the transitioned version"); - tokio::fs::write( - &metadata_path, - metadata.marshal_msg().expect("exact transition metadata should encode"), + // Use a distinct tier/backend so the legacy worker can be + // held in a deterministic remote-failure state while the + // exact journal path independently proves convergence. + let exact_tier_name = "DELETEALLTRANSITIONEDEXACT"; + let exact_backend = register_mock_tier(&ctx.tier_config_mgr(), exact_tier_name).await; + let exact_object = "exact-object"; + let mut exact_history_reader = PutObjReader::from_vec(b"exact transitioned history".to_vec()); + let exact_history = store + .put_object( + bucket, + exact_object, + &mut exact_history_reader, + &ObjectOptions { + versioned: true, + mod_time: Some(old_time - time::Duration::hours(1)), + ..Default::default() + }, ) .await - .expect("exact transition metadata should be written"); - } + .expect("second historical version should be written"); + let mut exact_current_reader = PutObjReader::from_vec(b"exact current version".to_vec()); + let exact_current = store + .put_object( + bucket, + exact_object, + &mut exact_current_reader, + &ObjectOptions { + versioned: true, + mod_time: Some(old_time), + ..Default::default() + }, + ) + .await + .expect("second current version should be written"); + store + .transition_object( + bucket, + exact_object, + &ObjectOptions { + versioned: true, + version_id: exact_history.version_id.map(|version_id| version_id.to_string()), + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: exact_tier_name.to_string(), + etag: exact_history.etag.clone().expect("second history should have an ETag"), + ..Default::default() + }, + mod_time: exact_history.mod_time, + ..Default::default() + }, + ) + .await + .expect("second historical version should transition exactly"); + let exact_remote_version = exact_backend + .put_versions() + .await + .last() + .cloned() + .expect("second transition should publish a remote version"); + exact_backend.set_remove_failure(true); let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects( store.clone(), - ¤t, + &exact_current, &lifecycle_event, &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner, incarnation, @@ -9825,22 +15070,52 @@ mod tests { .await; assert!(applied, "delete-all should remove current and transitioned history"); + assert!( + store.pools[0].disk_set[0] + .load_file_info_versions_exact(bucket, exact_object) + .await + .expect("exact journal-owned metadata lookup should succeed") + .is_none(), + "exact journal ownership must remove the source key without leaving a free-version" + ); let versions = store.pools[0].disk_set[0] .load_file_info_versions_exact(bucket, object) .await - .expect("remaining exact metadata should be readable"); - assert!(versions.is_none(), "delete-all must not leave a tier free-version"); + .expect("remaining fallback metadata should be readable") + .expect("the prior legacy cleanup owner should remain"); + assert_eq!( + versions + .versions + .iter() + .filter(|version| !version.tier_free_version()) + .count(), + 0, + "exact delete-all must remove every visible version" + ); + assert_eq!( + versions.versions.iter().filter(|version| version.tier_free_version()).count(), + 1, + "the exact v6 journal must not add another free-version" + ); assert_eq!(tier_delete_journal_count(store.clone()).await, 1); - assert_eq!(backend.object_count().await, 1, "remote deletion must remain journal-driven"); + assert_eq!(backend.object_count().await, 1, "legacy remote cleanup must remain pending"); + assert_eq!(exact_backend.object_count().await, 1, "exact remote cleanup must remain journal-owned"); + exact_backend.set_remove_failure(false); let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) .await .expect("committed journal should recover"); - assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + assert_eq!(stats.failed, 0); + assert!(stats.scanned <= 1 && stats.deleted <= 1); assert_eq!(tier_delete_journal_count(store).await, 0); - assert_eq!(backend.object_count().await, 0); - assert_eq!(backend.exact_remove_count(), 1); - assert_eq!(backend.remove_versions().await, transitioned_remote_versions); + assert_eq!( + backend.object_count().await, + 1, + "journal recovery must leave the legacy free-version remote" + ); + assert_eq!(exact_backend.object_count().await, 0); + assert!(exact_backend.exact_remove_count() >= 1); + assert_eq!(exact_backend.remove_versions().await, vec![exact_remote_version]); }); }) .expect("test thread should spawn") @@ -9851,7 +15126,7 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn prepared_tier_delete_recovery_requires_namespace_locking() { + async fn dispatched_tier_delete_recovery_requires_namespace_locking() { temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async { let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir"); let (ctx, store, _shutdown) = @@ -9867,6 +15142,7 @@ mod tests { .expect("tier lease should resolve") .backend_identity(); let entry = Jentry { + persisted_version: 0, obj_name: "remote/lock-disabled".to_string(), version_id: uuid::Uuid::new_v4().to_string(), tier_name: tier_name.to_string(), @@ -9884,17 +15160,25 @@ mod tests { etag: Some("etag".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()), }), + dispatch: None, }; - persist_tier_delete_journal_entry(store.clone(), &entry) - .await - .expect("prepared journal should persist"); + install_test_tier_delete_dispatch_fixture( + store.clone(), + "absent-source-bucket", + uuid::Uuid::new_v4(), + "absent-source-object", + vec![(entry, Some(TierDeleteJournalState::Dispatched))], + TierDeleteDispatchManifestState::DispatchAuthorized, + ) + .await + .expect("authorized dispatched journal should persist"); let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) .await .expect("recovery scan should complete"); assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1)); - assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain prepared for retry"); + assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain dispatched for retry"); assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely"); }) .await; @@ -10201,14 +15485,252 @@ mod tests { old_backend_identity: Some([1; 32]), new_backend_identity: Some([2; 32]), }], - expires_at_unix_nanos: 1_780_000_000_000_000_000, + expires_at_unix_nanos: i64::MAX, } } #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() { + async fn tier_mutation_abort_tombstone_rejects_delayed_prepare() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-abort-tombstone", &[4])).await; + register_mock_tier(&store.tier_config_mgr(), "COLD-A").await; + store + .tier_config_mgr() + .read() + .await + .save_tiering_config(store.clone()) + .await + .expect("base tier config should persist"); + let config_info = store + .get_object_info( + RUSTFS_META_BUCKET, + &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE), + &ObjectOptions::default(), + ) + .await + .expect("base tier config metadata should load"); + let mutation_id = uuid::Uuid::new_v4(); + let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [7; 32]); + intent.old_config_etag = config_info.etag; + intent.expires_at_unix_nanos = + i64::try_from((OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp_nanos()) + .expect("future tombstone expiry should fit i64"); + let payload = intent.encode().expect("prepare identity should encode"); + + let mismatch_id = uuid::Uuid::new_v4(); + let mut mismatch = intent.clone(); + mismatch.mutation_id = mismatch_id; + mismatch.old_config_etag = Some("not-the-current-config-etag".to_string()); + let mismatch_error = handle_tier_mutation_peer_request( + store.clone(), + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Abort, + mismatch_id, + &mismatch.encode().expect("proof-mismatch Abort should encode"), + ) + .await + .expect_err("a missing-record Abort with the wrong config proof must fail closed"); + assert!(matches!(mismatch_error, TierMutationPeerError::AbortProofMismatch)); + assert!(matches!( + load_tier_mutation_intent_record(store.clone(), mismatch_id).await, + Err(Error::ConfigNotFound) + )); + + let aborted = handle_tier_mutation_peer_request( + store.clone(), + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Abort, + mutation_id, + &payload, + ) + .await + .expect("missing-record abort should persist a tombstone"); + assert!(aborted.applied); + assert_eq!(aborted.state, TierMutationPeerState::Aborted); + let tombstone = load_tier_mutation_intent_record(store.clone(), mutation_id) + .await + .expect("abort tombstone should be durable"); + assert_eq!(tombstone.state, TierMutationIntentState::Aborted); + assert!(tombstone.same_identity_as(&intent)); + + let manager = store.tier_config_mgr(); + TierConfigMgr::reload_handle(&manager, store.clone()) + .await + .expect("reload should retain an unexpired abort tombstone"); + let tombstone_after_reload = load_tier_mutation_intent_record(store.clone(), mutation_id) + .await + .expect("reload must not clean an unexpired abort tombstone"); + assert_eq!(tombstone_after_reload.state, TierMutationIntentState::Aborted); + assert!(tombstone_after_reload.same_identity_as(&intent)); + + register_mock_tier(&store.tier_config_mgr(), "COLD-B").await; + store + .tier_config_mgr() + .read() + .await + .save_tiering_config(store.clone()) + .await + .expect("later config change should persist before idempotent Abort replay"); + let retried_abort = handle_tier_mutation_peer_request( + store.clone(), + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Abort, + mutation_id, + &payload, + ) + .await + .expect("terminal Abort retry must not depend on the mutable current config proof"); + assert!(!retried_abort.applied); + assert_eq!(retried_abort.state, TierMutationPeerState::Aborted); + + let delayed = handle_tier_mutation_peer_request( + store.clone(), + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + &payload, + ) + .await + .expect("delayed matching prepare should converge on the tombstone"); + assert!(!delayed.applied); + assert_eq!(delayed.state, TierMutationPeerState::Aborted); + TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A") + .await + .expect("delayed prepare must not reinstall the runtime block"); + + let mut conflicting = intent.clone(); + conflicting.candidate_digest = [8; 32]; + let conflict = handle_tier_mutation_peer_request( + store, + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + &conflicting.encode().expect("conflicting prepare should encode"), + ) + .await + .expect_err("different identity must not reuse an abort tombstone"); + assert!(matches!(conflict, TierMutationPeerError::ConflictingIntent)); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_mutation_expired_prepare_cannot_recreate_a_cleaned_tombstone() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-expired-replay", &[4])).await; + register_mock_tier(&store.tier_config_mgr(), "COLD-A").await; + let mutation_id = uuid::Uuid::new_v4(); + let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [11; 32]); + intent.expires_at_unix_nanos = + i64::try_from((OffsetDateTime::now_utc() - time::Duration::seconds(1)).unix_timestamp_nanos()) + .expect("expired timestamp should fit i64"); + let payload = intent.encode().expect("expired signed Prepare should still encode"); + + let error = handle_tier_mutation_peer_request( + store.clone(), + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + &payload, + ) + .await + .expect_err("a missing-record expired Prepare must fail closed"); + assert!(matches!(error, TierMutationPeerError::ExpiredIntent)); + assert!(matches!( + load_tier_mutation_intent_record(store.clone(), mutation_id).await, + Err(Error::ConfigNotFound) + )); + TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A") + .await + .expect("expired replay must not install a peer-only runtime fence"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_mutation_peer_prepare_waits_for_inflight_reference_lease() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-drain", &[4])).await; + register_mock_tier(&store.tier_config_mgr(), "COLD-A").await; + let old_lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A") + .await + .expect("old tier operation lease should be available"); + let mutation_id = uuid::Uuid::new_v4(); + let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-A", [9; 32]); + intent.affected_targets[0].old_backend_identity = Some(old_lease.backend_identity()); + let payload = intent.encode().expect("prepare intent should encode"); + let prepare_store = store.clone(); + let prepare = tokio::spawn(async move { + handle_tier_mutation_peer_request( + prepare_store, + TIER_MUTATION_RPC_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + &payload, + ) + .await + }); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-A") + .await + .is_err() + { + return; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("peer Prepare should install its runtime block"); + assert!( + !prepare.is_finished(), + "peer Prepare must not acknowledge while a pre-existing reference creator still holds a lease" + ); + + drop(old_lease); + let outcome = prepare + .await + .expect("peer Prepare task should join") + .expect("peer Prepare should finish after the lease drains"); + assert_eq!(outcome.state, TierMutationPeerState::Prepared); + TierConfigMgr::clear_prepared_mutation_intent_block(&store.tier_config_mgr(), mutation_id) + .await + .expect("test should clear the prepared runtime block"); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently() { + // This end-to-end state-machine scenario intentionally keeps many + // durable fixtures live at once. Give its debug future the same + // dedicated stack used by the other large ecstore scenarios so the + // assertions run under the default test command without RUST_MIN_STACK. + std::thread::Builder::new() + .name("tier-mutation-peer-handler-test".to_string()) + .stack_size(32 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("tier mutation peer handler test runtime should build"); + runtime.block_on(tier_mutation_peer_handler_case()); + }) + .expect("tier mutation peer handler test thread should spawn") + .join() + .expect("tier mutation peer handler test thread should complete"); + } + + #[cfg(feature = "test-util")] + async fn tier_mutation_peer_handler_case() { let temp_dir = tempfile::tempdir().expect("create temp store dir"); let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-peer-handler", &[4])).await; @@ -10324,7 +15846,7 @@ mod tests { TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase::Abort, bad_digest_id, - b"", + &bad_digest_intent.encode().expect("bad digest abort identity should encode"), ) .await .expect("the negative digest proof fixture should clean up through abort"); @@ -10506,48 +16028,27 @@ mod tests { TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase::Abort, abort_id, - b"", + &abort_prepare_payload, ) .await .expect("abort should advance the prepared peer intent"); assert!(aborted.applied); assert_eq!(aborted.state, TierMutationPeerState::Aborted); - let aborted_blocked = match TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await { - Ok(_) => panic!("aborted peer mutation must remain blocked until local recovery cleans it up"), - Err(err) => err, - }; - assert!(aborted_blocked.message.contains("being replaced"), "{aborted_blocked}"); + TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B") + .await + .expect("durable abort must clear the peer runtime fence immediately"); let retried_abort = handle_tier_mutation_peer_request( store.clone(), TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase::Abort, abort_id, - b"", + &abort_prepare_payload, ) .await .expect("same abort retry should be idempotent before recovery cleanup"); assert!(!retried_abort.applied); assert_eq!(retried_abort.state, TierMutationPeerState::Aborted); - - let refresh_store = store.clone(); - let refresh_manager = store.tier_config_mgr(); - let refresh_worker = tokio::spawn(async move { - TierConfigMgr::refresh_tier_config_handle_with(refresh_manager, refresh_store).await; - }); - tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Ok(lease) = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), "COLD-B").await { - drop(lease); - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("abort notification should drive cleanup before clearing the prepared fence"); - refresh_worker.abort(); - let _ = refresh_worker.await; } #[cfg(feature = "test-util")] @@ -10584,12 +16085,13 @@ mod tests { let mutation_id = uuid::Uuid::new_v4(); let mut intent = tier_mutation_peer_test_intent(mutation_id, "COLD-B", candidate_digest); intent.old_config_etag = Some(base_etag.clone()); + let prepare_payload = intent.encode().expect("late abort prepare intent should encode"); handle_tier_mutation_peer_request( store.clone(), TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase::Prepare, mutation_id, - &intent.encode().expect("late abort prepare intent should encode"), + &prepare_payload, ) .await .expect("prepare should install the runtime fence"); @@ -10619,7 +16121,7 @@ mod tests { TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase::Abort, mutation_id, - b"", + &prepare_payload, ) .await .expect_err("an abort after the candidate config commit must fail closed"); @@ -10859,7 +16361,7 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn cancelled_transition_cleanup_journals_to_its_own_instance_store() { + async fn cancelled_transition_cleanup_recovers_from_its_own_instance_transaction() { struct ResolverReset(Arc>>>); impl Drop for ResolverReset { @@ -10911,7 +16413,7 @@ mod tests { ); let tier_name = "CROSSCTXA"; - let backend = register_mock_tier(&ctx_a.tier_config_mgr(), tier_name).await; + let backend = register_transition_reconcile_test_tier(&ctx_a.tier_config_mgr(), tier_name).await; backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await; backend.reject_next_non_empty_remote_version_validation(); let remove_barrier = backend.arm_failing_remove_barrier().await; @@ -10953,23 +16455,37 @@ mod tests { ); remove_barrier.wait_until_paused().await; + let transaction_counts = ( + transition_transaction_record_count(store_a.clone()).await, + transition_transaction_record_count(store_b.clone()).await, + ); + assert_eq!( + transaction_counts, + (1, 0), + "the transition transaction must remain only on store A even while the process resolver points at store B" + ); let journal_counts = ( tier_delete_journal_count(store_a.clone()).await, tier_delete_journal_count(store_b.clone()).await, ); assert_eq!( journal_counts, - (1, 0), - "the journal must land only on store A even while the process resolver points at store B" + (0, 0), + "rejected transition uploads must not create a tier-delete journal" ); assert_eq!(backend.object_count().await, 1, "failed cleanup should retain the remote candidate"); remove_barrier.release(); remove_barrier.wait_until_operation_dropped().await; - let recovered = recover_tier_delete_journal_entries(store_a.clone(), 100, None) + let recovered = recover_transition_transaction_records(store_a.clone(), 100, None) .await - .expect("store A should recover its own cancelled-transition journal"); - assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0)); + .expect("store A should recover its own cancelled-transition transaction"); + assert_eq!( + (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed), + (1, 1, 0, 0) + ); + assert_eq!(transition_transaction_record_count(store_a.clone()).await, 0); + assert_eq!(transition_transaction_record_count(store_b.clone()).await, 0); assert_eq!(tier_delete_journal_count(store_a.clone()).await, 0); assert_eq!(tier_delete_journal_count(store_b.clone()).await, 0); assert_eq!( diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index a7a9dc4ee..e34776e10 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -65,7 +65,7 @@ use http::HeaderMap; use lazy_static::lazy_static; use rand::RngExt as _; use rustfs_config::server_config::Config; -use rustfs_filemeta::FileInfo; +use rustfs_filemeta::{FileInfo, FileMeta}; use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts}; use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper}; use rustfs_madmin::heal_commands::HealResultItem; @@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr; type WalkOptions = StorageWalkOptions bool>; pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000; +pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024; +pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096; +pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100); + +#[derive(Debug)] +pub(crate) struct BucketDeleteDiagnosticBudget { + deadline: Option, + max_elapsed: Duration, + entries_remaining: usize, +} + +impl BucketDeleteDiagnosticBudget { + pub(crate) fn new() -> Self { + Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED) + } + + fn with_limits(entries: usize, elapsed: Duration) -> Self { + Self { + deadline: None, + max_elapsed: elapsed, + entries_remaining: entries, + } + } + + fn deadline(&mut self) -> tokio::time::Instant { + let max_elapsed = self.max_elapsed; + *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed) + } + + fn claim_entry(&mut self) -> bool { + if self.entries_remaining == 0 { + return false; + } + let deadline = self.deadline(); + if tokio::time::Instant::now() >= deadline { + return false; + } + self.entries_remaining -= 1; + true + } + + async fn run_io(&mut self, future: F) -> std::io::Result> + where + F: std::future::Future>, + { + let deadline = self.deadline(); + if tokio::time::Instant::now() >= deadline { + return Ok(None); + } + match tokio::time::timeout_at(deadline, future).await { + Ok(result) => result.map(Some), + Err(_) => Ok(None), + } + } +} #[derive(Debug, Default, Clone, PartialEq, Eq)] pub(crate) struct BucketMetadataLessResidue { pub(crate) xlmeta_found: bool, + pub(crate) xlmeta_blocker: Option, pub(crate) files: usize, pub(crate) uuid_data_dirs: usize, + pub(crate) entries_scanned: usize, + pub(crate) diagnostic_bytes_read: u64, + pub(crate) diagnostic_truncated: bool, pub(crate) sample: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BucketDeleteBlockerKind { + VisibleVersion, + TierFreeVersion, + UnknownXlMeta, + OrphanDirectory, + DiagnosticBudgetExceeded, +} + +impl BucketDeleteBlockerKind { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::VisibleVersion => "visible_version", + Self::TierFreeVersion => "tier_free_version", + Self::UnknownXlMeta => "unknown_xlmeta", + Self::OrphanDirectory => "orphan_directory", + Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded", + } + } +} + impl BucketMetadataLessResidue { pub(crate) fn has_residue_without_xlmeta(&self) -> bool { - !self.xlmeta_found && self.files > 0 + !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated) } pub(crate) fn describe(&self) -> String { let sample = self.sample.as_deref().unwrap_or(""); format!( - "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}", - self.files, self.uuid_data_dirs + "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}", + self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated, ) } } @@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result< Ok(false) } +#[cfg(test)] pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result { + let mut budget = BucketDeleteDiagnosticBudget::new(); + scan_metadata_less_residue_with_budget(path, &mut budget).await +} + +async fn scan_metadata_less_residue_with_budget( + path: &std::path::Path, + budget: &mut BucketDeleteDiagnosticBudget, +) -> std::io::Result { use crate::disk::STORAGE_FORMAT_FILE; use tokio::fs; + use tokio::io::AsyncReadExt as _; let mut scan = BucketMetadataLessResidue::default(); let mut stack = vec![path.to_path_buf()]; + let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| { + scan.diagnostic_truncated = true; + scan.sample.get_or_insert_with(|| "".to_string()); + }; + while let Some(current_path) = stack.pop() { - let mut entries = match fs::read_dir(¤t_path).await { - Ok(entries) => entries, + let mut entries = match budget.run_io(fs::read_dir(¤t_path)).await { + Ok(Some(entries)) => entries, + Ok(None) => { + mark_budget_exhausted(&mut scan); + return Ok(scan); + } Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, Err(err) => return Err(err), }; - while let Some(entry) = entries.next_entry().await? { - let file_type = entry.file_type().await?; + loop { + let entry = match budget.run_io(entries.next_entry()).await? { + Some(Some(entry)) => entry, + Some(None) => break, + None => { + mark_budget_exhausted(&mut scan); + return Ok(scan); + } + }; + if !budget.claim_entry() { + mark_budget_exhausted(&mut scan); + return Ok(scan); + } + scan.entries_scanned = scan.entries_scanned.saturating_add(1); + let Some(file_type) = budget.run_io(entry.file_type()).await? else { + mark_budget_exhausted(&mut scan); + return Ok(scan); + }; let file_name = entry.file_name(); let file_name_str = file_name.to_string_lossy(); if file_name_str == STORAGE_FORMAT_FILE { scan.xlmeta_found = true; - continue; + if scan.xlmeta_blocker.is_none() { + let entry_path = entry.path(); + let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else { + mark_budget_exhausted(&mut scan); + scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded); + return Ok(scan); + }; + scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES { + BucketDeleteBlockerKind::UnknownXlMeta + } else { + match budget.run_io(fs::File::open(&entry_path)).await { + Ok(Some(file)) => { + let mut data = Vec::new(); + let read = budget + .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data)) + .await; + scan.diagnostic_bytes_read = data.len() as u64; + match read { + Ok(Some(_)) => match FileMeta::load(&data) { + Ok(meta) + if !meta.versions.is_empty() + && meta.versions.iter().all(|version| version.header.free_version()) => + { + BucketDeleteBlockerKind::TierFreeVersion + } + Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion, + Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta, + }, + Ok(None) => { + mark_budget_exhausted(&mut scan); + BucketDeleteBlockerKind::DiagnosticBudgetExceeded + } + Err(_) => BucketDeleteBlockerKind::UnknownXlMeta, + } + } + Ok(None) => { + mark_budget_exhausted(&mut scan); + BucketDeleteBlockerKind::DiagnosticBudgetExceeded + } + Err(_) => BucketDeleteBlockerKind::UnknownXlMeta, + } + }); + let sample = entry_path + .strip_prefix(path) + .unwrap_or(entry_path.as_path()) + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"); + scan.sample = Some(sample); + } + return Ok(scan); } if file_type.is_dir() { @@ -222,7 +386,10 @@ pub(crate) mod init_format; pub(crate) mod list_objects; mod multipart; mod object; -pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches}; +pub(crate) use object::{ + DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence, + SourceCleanupMutationFence, tiered_data_movement_source_matches, +}; pub use object::{ PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError, SnapshotConsistencyError, diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 0c74cad5f..cbddc8299 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -846,6 +846,7 @@ impl ECStore { .await } + #[cfg(all(test, feature = "test-util"))] pub(crate) async fn complete_multipart_upload_for_data_movement( self: Arc, target: (usize, Option<&ObjectLockDiagGuard>), @@ -854,6 +855,44 @@ impl ECStore { upload_id: &str, uploaded_parts: Vec, opts: &ObjectOptions, + ) -> Result { + self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence( + self: Arc, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_id: &str, + uploaded_parts: Vec, + opts: &ObjectOptions, + publication_fence: RemoteTuplePublicationFence, + ) -> Result { + self.complete_multipart_upload_for_data_movement_inner( + (target_pool_idx, None), + bucket, + object, + upload_id, + uploaded_parts, + opts, + Some(publication_fence), + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn complete_multipart_upload_for_data_movement_inner( + self: Arc, + target: (usize, Option<&ObjectLockDiagGuard>), + bucket: &str, + object: &str, + upload_id: &str, + uploaded_parts: Vec, + opts: &ObjectOptions, + publication_fence: Option, ) -> Result { let (target_pool_idx, mutation_fence) = target; check_complete_multipart_args(bucket, object, upload_id)?; @@ -885,8 +924,36 @@ impl ECStore { snapshot.add_lock_fences(&mut opts); opts.object_lock_config_snapshot = Some(snapshot); } - self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence) + let fixed_read_anchor = publication_fence + .as_ref() + .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard); + self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor)) .await; + // NewMultipart/UploadPart are staging only. Acquire and consume the + // non-cloneable publication capability immediately before Complete, + // then retain its guards until Complete has drained the commit path. + let publication_object = encode_dir_object(object); + let publication_guard = match publication_fence { + Some(publication_fence) => { + let guard = publication_fence + .into_commit_guard(target_pool_idx, bucket, &publication_object) + .await?; + guard.add_namespace_lock_fence(&mut opts); + opts.no_lock = true; + Some(guard) + } + None => { + if rustfs_utils::http::metadata_compat::contains_key_str( + &opts.user_defined, + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + ) { + return Err(Error::other( + "data movement multipart completion cannot publish transition ownership without a publication capability", + )); + } + None + } + }; #[cfg(test)] pause_data_movement_multipart_before_selected_completion(bucket).await; let pool = self @@ -911,6 +978,7 @@ impl ECStore { }, ) .await; + drop(publication_guard); let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await; if result.is_ok() { list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await; diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index bdc9930c1..26a933797 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -17,12 +17,13 @@ use crate::bucket::lifecycle::{ bucket_lifecycle_ops::eval_action_from_lifecycle, get_expiry_configs, tier_delete_journal::{ - abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry, - enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry, - record_tier_delete_journal_backend_identity, + ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE, + complete_tier_delete_dispatch, prepare_tier_delete_dispatch, record_tier_delete_journal_backend_identity, + tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, }, tier_sweeper::{ - Jentry, attach_tier_delete_source, transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry, + Jentry, TierDeleteSourceIdentity, attach_tier_delete_source, transitioned_delete_journal_entry_for_source, + transitioned_force_delete_journal_entry, }, }; use crate::bucket::metadata_sys::{ @@ -36,12 +37,17 @@ use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObj use crate::bucket::versioning::VersioningApi; use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id}; use crate::disk::OldCurrentSize; -use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot}; +use crate::object_api::{ + NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState, +}; +use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof; +use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::{ SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain, }; use crate::storage_api_contracts::{ + list::ListOperations as _, namespace::NamespaceLocking as _, object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _}, }; @@ -63,7 +69,6 @@ use tokio::io::{AsyncRead, ReadBuf}; const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000; #[cfg(test)] const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2; -const FORCE_DELETE_LIST_PAGE_SIZE: i32 = 1_000; fn build_tier_delete_journal_entry( bucket: &str, @@ -73,7 +78,7 @@ fn build_tier_delete_journal_entry( ) -> Result> { let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?; let source_object = decode_dir_object(object); - let Some(mut je) = (if opts.delete_prefix { + let entry = if opts.delete_prefix { transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| { attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended); je @@ -87,127 +92,245 @@ fn build_tier_delete_journal_entry( source_object.as_str(), source, ) - }) else { + }; + let Some(mut je) = entry else { + let lifecycle_exact_object_delete_all = opts.delete_prefix_object && opts.lifecycle_delete_all.is_some(); + if opts.delete_prefix + && !lifecycle_exact_object_delete_all + && source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE + { + return Err(Error::other( + "recursive prefix delete cannot preserve a transitioned source with legacy Unknown remote-version state", + )); + } return Ok(None); }; record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?; + if opts.delete_prefix && !je.can_replace_tier_free_version() { + return Err(Error::other( + "recursive prefix delete requires a stable transitioned source and destination identity", + )); + } Ok(Some(je)) } -async fn prepare_tier_delete_journal_entry( - api: &Arc, - bucket: &str, - object: &str, - opts: &ObjectOptions, - source: &ObjectInfo, -) -> Result> { - let Some(je) = build_tier_delete_journal_entry(bucket, object, opts, source)? else { - return Ok(None); - }; - persist_tier_delete_journal_entry(Arc::clone(api), &je) - .await - .map_err(Error::other)?; - Ok(Some(je)) -} - -async fn abort_prepared_tier_delete_journal_entry(api: &Arc, je: &Jentry) { - if let Err(err) = abort_prepared_journal_entry_if_current(Arc::clone(api), je).await { - warn!( - object = %je.obj_name, - tier = %je.tier_name, - error = ?err, - "failed to remove aborted tier delete journal" - ); - } -} - -async fn abort_prepared_tier_delete_journal_entries(api: &Arc, entries: &[Jentry]) { - for entry in entries { - abort_prepared_tier_delete_journal_entry(api, entry).await; - } -} - -async fn commit_prepared_tier_delete_journal_entry(api: &Arc, je: &Jentry) { - if let Err(err) = commit_tier_delete_journal_entry(Arc::clone(api), je).await { - warn!( - object = %je.obj_name, - tier = %je.tier_name, - error = ?err, - "tier delete committed locally but journal commit failed; recovery will retry" - ); - return; - } - let mut committed = je.clone(); - committed.state = crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed; - if let Err(err) = enqueue_committed_tier_delete_journal_entry(&committed).await { - warn!( - object = %je.obj_name, - tier = %je.tier_name, - error = ?err, - "tier delete journal committed but could not be queued; recovery will retry" - ); - } -} - -async fn commit_prepared_tier_delete_journal_entries(api: &Arc, entries: &[Jentry]) { - for entry in entries { - commit_prepared_tier_delete_journal_entry(api, entry).await; - } -} - async fn prepare_prefix_tier_delete_journal_entries( api: &Arc, bucket: &str, prefix: &str, opts: &ObjectOptions, -) -> Result> { - let mut marker = None; - let mut version_marker = None; - let mut entries = Vec::new(); +) -> Result { + Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await +} - loop { - let page = Arc::clone(api) - .list_object_versions_for_lifecycle( - bucket, - prefix, - marker.clone(), - version_marker.clone(), - None, - FORCE_DELETE_LIST_PAGE_SIZE, - ) - .await?; +async fn prepare_prefix_tier_delete_journal_entries_inner( + api: &Arc, + bucket: &str, + prefix: &str, + opts: &ObjectOptions, +) -> Result { + let mut tier_references = std::collections::HashSet::<(String, Option)>::new(); + let mut entries_by_name = std::collections::BTreeMap::new(); + let logical_prefix = decode_dir_object(prefix); + let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone()); + let physical_sets = api + .pools + .iter() + .flat_map(|pool| pool.disk_set.iter().cloned()) + .collect::>(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(100); + let cancellation = tokio_util::sync::CancellationToken::new(); + let walk_cancel = cancellation.clone(); + let bucket_owned = bucket.to_string(); + let prefix_owned = prefix.to_string(); + let walk = async move { + use futures::StreamExt as _; - for source in page.objects { - if let Some(entry) = build_tier_delete_journal_entry(bucket, &source.name, opts, &source)? { - entries.push(entry); + let results = futures::stream::iter(physical_sets.into_iter().map(|set| { + let tx = tx.clone(); + let cancellation = walk_cancel.clone(); + let bucket = bucket_owned.clone(); + let prefix = prefix_owned.clone(); + async move { + let result = set + .walk( + cancellation.clone(), + &bucket, + &prefix, + tx, + WalkOptions { + include_free_versions: true, + walkdir_timeout: Some(Duration::ZERO), + ..Default::default() + }, + ) + .await; + if result.is_err() { + cancellation.cancel(); + } + result + } + })) + .buffer_unordered(4) + .collect::>() + .await; + drop(tx); + results.into_iter().collect::>>().map(|_| ()) + }; + let collect = async { + while let Some(result) = rx.recv().await { + if let Some(err) = result.err { + cancellation.cancel(); + return Err(err); + } + let Some(source) = result.item else { + continue; + }; + let object = decode_dir_object(&source.name); + if exact_object.as_ref().is_some_and(|expected| expected != &object) { + continue; + } + if exact_object.is_none() && !object.starts_with(&logical_prefix) { + continue; + } + if source.transitioned_object.free_version { + cancellation.cancel(); + return Err(Error::other( + "recursive prefix delete cannot discard an existing tier free-version cleanup obligation", + )); + } + if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE { + let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?; + tier_references.insert((source.transitioned_object.tier.clone(), backend_identity)); + } + if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? { + entries_by_name + .entry(tier_delete_journal_object_name(&entry)) + .or_insert(entry); } } + Ok(()) + }; + let (walk_result, collect_result) = tokio::join!(walk, collect); + collect_result?; + walk_result?; + let entries = entries_by_name.into_values().collect::>(); - if !page.is_truncated { - break; + let mut tier_references = tier_references.into_iter().collect::>(); + tier_references.sort_unstable(); + let mut leases = Vec::with_capacity(tier_references.len()); + for (tier_name, backend_identity) in tier_references { + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, } - - let next_marker = page - .next_marker - .ok_or_else(|| Error::other("truncated force delete listing has no next marker"))?; - let next_version_marker = page.next_version_idmarker; - if marker.as_deref() == Some(next_marker.as_str()) && version_marker == next_version_marker { - return Err(Error::other("force delete listing marker did not advance")); - } - marker = Some(next_marker); - version_marker = next_version_marker; + .map_err(Error::other)?; + leases.push(lease); + } + if entries.is_empty() { + return Ok(PreparedPrefixTierDelete { + dispatch: None, + _leases: leases, + }); } - let mut persisted = Vec::with_capacity(entries.len()); - for entry in entries { - if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &entry).await { - abort_prepared_tier_delete_journal_entries(api, &persisted).await; - return Err(Error::other(err)); - } - persisted.push(entry); - } - Ok(persisted) + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + let dispatch = + prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence) + .await?; + Ok(PreparedPrefixTierDelete { + dispatch: Some(dispatch), + _leases: leases, + }) } + +struct PreparedPrefixTierDelete { + dispatch: Option, + _leases: Vec, +} + +#[cfg(all(test, feature = "test-util"))] +struct PrefixDeleteAfterJournalPrepareBarrierState { + bucket: String, + prefix: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct PrefixDeleteAfterJournalPrepareBarrier { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl PrefixDeleteAfterJournalPrepareBarrier { + pub(crate) fn install(bucket: &str, prefix: &str) -> Self { + let state = Arc::new(PrefixDeleteAfterJournalPrepareBarrierState { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("prefix journal barrier mutex should not poison"); + assert!(slot.is_none(), "prefix journal barrier must not already be installed"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + self.state.arrived.notified().await; + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for PrefixDeleteAfterJournalPrepareBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + if let Some(slot) = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER.get() { + let mut slot = slot.lock().expect("prefix journal barrier mutex should not poison"); + if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) { + *slot = None; + } + } + } +} + +#[cfg(all(test, feature = "test-util"))] +async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) { + let state = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("prefix journal barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.prefix == prefix) + .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + } +} + async fn delete_prefix_with_tier_delete_journal( store: &ECStore, bucket: &str, @@ -215,48 +338,221 @@ async fn delete_prefix_with_tier_delete_journal( opts: &ObjectOptions, tier_journal_api: Option<&Arc>, ) -> Result<()> { - let lifecycle_delete_all = opts.lifecycle_delete_all.is_some(); - let journal_entry = if !lifecycle_delete_all && let Some(api) = tier_journal_api { - Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?) - } else { - None + let Some(api) = tier_journal_api else { + return store.delete_prefix(bucket, object, opts).await; }; + let PreparedPrefixTierDelete { dispatch, _leases } = + prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?; + let Some(dispatch) = dispatch else { + // There is no remote-cleanup candidate, so no v6 manifest or fleet + // proof is required. Keep any compatibility-path tier leases alive + // until the local delete has committed. + let _tier_leases = _leases; + let mut operation_opts = opts.clone(); + // `tier_delete_journal_api` means a v6 dispatch authorization must be + // present at every destructive phase. With no v6 candidate, clear + // that marker so ordinary objects use the local path and legacy + // transitioned metadata retains its FreeVersion fallback. + operation_opts.tier_delete_journal_api = None; + operation_opts.tier_delete_dispatch_authorization = None; + return store.delete_prefix(bucket, object, &operation_opts).await; + }; + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + let active = dispatch.consume(bucket, bucket_incarnation, object)?; + let mut operation_opts = opts.clone(); + operation_opts.tier_delete_dispatch_authorization = Some(active.authorization()); - let result = store.delete_prefix(bucket, object, opts).await; + #[cfg(all(test, feature = "test-util"))] + pause_prefix_delete_after_journal_prepare(bucket, object).await; + + // Keep every backend generation lease until the whole local operation has + // either committed its journal set or returned an ambiguous mutation. + let _tier_leases = _leases; + if active.predecessor_replay_required() { + replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?; + complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?; + return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch")); + } + let result = store.delete_prefix(bucket, object, &operation_opts).await; match result { Ok(()) => { - let lifecycle_entries = if lifecycle_delete_all { - opts.lifecycle_delete_all_journal() - .ok_or(StorageError::PreconditionFailed)? - .lock() - .prepared_entries() - } else { - Vec::new() - }; - let entries = journal_entry.as_deref().unwrap_or(&lifecycle_entries); - if let Some(api) = tier_journal_api { - commit_prepared_tier_delete_journal_entries(api, entries).await; + if let Err(err) = complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket, + prefix = object, + error = ?err, + "Prefix deletion committed with durable tier cleanup pending" + ); } Ok(()) } - Err(err) => { - if let Some(api) = tier_journal_api { - if lifecycle_delete_all { - let (abort, entries) = { - let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?; - let state = journal.lock(); - (!state.mutation_started(), state.prepared_entries()) - }; - if abort { - abort_prepared_tier_delete_journal_entries(api, &entries).await; - } - } else if let Some(entries) = journal_entry.as_ref() { - abort_prepared_tier_delete_journal_entries(api, entries).await; - } + // Once DispatchAuthorized exists, every failure is mutation-ambiguous. + // Recovery decides from physical proof; never abort the journals here. + Err(err) => Err(err), + } +} + +async fn replay_authorized_tier_delete_sources( + store: &ECStore, + bucket: &str, + prefix: &str, + active: &ActiveTierDeleteDispatch, + opts: &ObjectOptions, +) -> Result<()> { + let _publication_scope_guard = opts + .scanner_publication_commit_scope + .clone() + .map(ScannerPublicationCommitScopeGuard::new); + let publication_scope = opts.scanner_publication_commit_scope.as_ref(); + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + let authorization = active.authorization(); + authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?; + + let mut source_objects = std::collections::BTreeSet::new(); + for entry in active.entries() { + let source = entry + .source + .as_ref() + .filter(|source| source.has_stable_identity()) + .ok_or_else(|| Error::other("authorized tier delete predecessor has no stable source"))?; + if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) { + return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope")); + } + source_objects.insert(source.object.clone()); + } + + let mut deleted = 0; + for object in source_objects { + if bucket_fence.is_lock_lost() { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + } + let encoded_object = encode_dir_object(&object); + let guards = if opts.delete_prefix_object { + store + .acquire_remaining_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) + .await? + } else { + store + .acquire_all_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) + .await? + }; + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + if let Some(scope) = publication_scope { + if scope.state() == ScannerPublicationCommitState::Admitted { + scope + .try_begin() + .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?; } - Err(err) + if !scope.can_commit() { + let _ = scope.mark_indeterminate(); + return Err(StorageError::OperationCanceled); + } + } + let mut replay_opts = opts.clone(); + replay_opts.no_lock = true; + replay_opts.delete_prefix = false; + replay_opts.delete_prefix_object = false; + for guard in &guards { + guard.add_namespace_lock_fence(&mut replay_opts); + } + for pool in &store.pools { + for set in &pool.disk_set { + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + deleted += set + .replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts) + .await?; + } + } + if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); } } + if let Some(scope) = publication_scope { + let _ = scope.mark_committed(); + } + if deleted > 0 { + super::list_objects::observe_list_objects_mutation(store, bucket).await; + } + Ok(()) +} + +fn tier_delete_source_matches_replay_scope( + source: &TierDeleteSourceIdentity, + bucket: &str, + persisted_prefix: &str, + exact_object: bool, +) -> bool { + tier_delete_source_matches_dispatch_scope(source, bucket, persisted_prefix) + && (!exact_object || source.object == decode_dir_object(persisted_prefix)) +} + +async fn delete_recursive_prefix_with_tier_delete_journal( + store: &ECStore, + bucket: &str, + object: &str, + opts: &ObjectOptions, + tier_journal_api: Option<&Arc>, +) -> Result<()> { + // Prefix deletes cover multiple object keys; an exact lock on the prefix + // string would not protect child objects. + if !is_meta_bucketname(bucket) { + let state = opts + .object_lock_config_snapshot + .as_deref() + .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))? + .state(); + ensure_recursive_force_delete_allowed_for_state(bucket, state)?; + let bypass_governance = opts + .object_lock_delete + .as_ref() + .is_some_and(|delete_opts| delete_opts.bypass_governance); + for pool in &store.pools { + for set in &pool.disk_set { + let mut marker = None; + let mut version_marker = None; + loop { + let page = set + .clone() + .inner_list_object_versions_for_recursive_delete( + bucket, + object, + marker.clone(), + version_marker.clone(), + RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE, + ) + .await?; + for object_info in &page.objects { + if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone())); + } + } + if !page.is_truncated { + break; + } + let next_marker = page + .next_marker + .ok_or_else(|| Error::other("recursive delete version scan did not return a continuation marker"))?; + if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker { + return Err(Error::other("recursive delete version scan did not advance")); + } + marker = Some(next_marker); + version_marker = page.next_version_idmarker; + } + } + } + } + delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await } /// A GET whose object identity has been resolved while its namespace read lock @@ -397,7 +693,18 @@ impl ObjectLockDiagGuard { } pub(crate) fn is_lock_lost(&self) -> bool { - self.guard.is_lock_lost() + self.guard.is_lock_lost() || { + #[cfg(test)] + { + self.test_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + } + #[cfg(not(test))] + { + false + } + } } pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { @@ -407,7 +714,7 @@ impl ObjectLockDiagGuard { } #[cfg(test)] if let Some(fence) = self.test_namespace_lock_fence.as_ref() { - opts.add_namespace_lock_fence_for_test(fence); + opts.add_namespace_lock_fence(fence); } } } @@ -1358,6 +1665,10 @@ fn resolve_batch_delete_pool_results<'a>( fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions { let mut lookup_opts = opts.clone(); + // `no_lock` is an internal implementation detail, not transferable + // authority. Restore deliberately does not hold an outer object write lock, + // so every final local commit must acquire its own commit-late lock. + lookup_opts.no_lock = false; lookup_opts.skip_decommissioned = true; lookup_opts.skip_rebalancing = true; lookup_opts @@ -1684,7 +1995,364 @@ fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> { object_names } +/// A CopyObject branch that writes a fresh local data stream owns new bytes and +/// must not inherit the source version's remote-tier ownership record. FileMeta +/// exposes internal transition fields through `ObjectInfo::user_defined` for +/// compatibility, so cloning that map verbatim would publish a second owner of +/// the same remote tuple even though the destination has local data. +/// +/// Keep this normalization in the store layer so every full-copy branch in +/// `handle_copy_object` shares the same fail-safe boundary. Metadata-only and +/// version-only same-key copies do not use this helper and continue to preserve +/// the existing version's protected state. +fn materialized_copy_user_defined(source: &HashMap) -> HashMap { + let mut metadata = source.clone(); + for suffix in [ + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_utils::http::SUFFIX_TRANSITION_TIER, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, + rustfs_utils::http::SUFFIX_FREE_VERSION, + rustfs_utils::http::SUFFIX_TIER_FV_ID, + rustfs_utils::http::SUFFIX_TIER_FV_MARKER, + ] { + rustfs_utils::http::metadata_compat::remove_str(&mut metadata, suffix); + } + metadata +} + +enum RemoteTuplePublicationSource { + Object(ObjectInfo), + Tiered(rustfs_filemeta::FileInfo), +} + +/// A decommission migration may retain the fixed store domain as a read +/// anchor while it streams bytes. This type cannot authorize publication: it +/// must be consumed and dropped before the commit path acquires the fixed +/// domain as a write lock. +#[must_use = "the fixed read anchor must remain live until publication or migration abort"] +pub(crate) struct DecommissionFixedReadAnchor { + guard: ObjectLockDiagGuard, +} + +impl DecommissionFixedReadAnchor { + pub(crate) fn guard(&self) -> &ObjectLockDiagGuard { + &self.guard + } + + fn is_lock_lost(&self) -> bool { + self.guard.is_lock_lost() + } +} + +/// Opaque, non-cloneable capability retained by every background path that can +/// publish an existing object identity into a new physical metadata owner. +/// +/// The exact tier destination is captured before the copy, but its operation +/// lease and namespace write locks are deliberately deferred until the final +/// metadata commit. The source is then re-read under the same lock set. This +/// keeps large migrations compatible with the commit-late locking contract and +/// avoids pinning a tier generation throughout multipart staging. +#[must_use = "a data-movement publication fence must be consumed by the final target commit"] +pub(crate) struct RemoteTuplePublicationFence { + store: Arc, + bucket: String, + object: String, + source_pool_idx: usize, + source: RemoteTuplePublicationSource, + include_fixed_domain: bool, + fixed_read_anchor: Option, + backend_target: Option<(String, Option)>, +} + +/// Locks and tier generation retained from source revalidation through the +/// target rename quorum (and any rename-tail guard handoff). +#[must_use = "the publication commit guard must live through the target metadata commit"] +pub(crate) struct RemoteTuplePublicationCommitGuard { + guards: Vec, + backend_lease: Option, + revalidated_tiered_source: Option, +} + +impl RemoteTuplePublicationCommitGuard { + pub(crate) fn is_lock_lost(&self) -> bool { + self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost) + || self + .backend_lease + .as_ref() + .is_some_and(|lease| !lease.is_current_generation()) + } + + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + for guard in &self.guards { + guard.add_namespace_lock_fence(opts); + } + } + + pub(crate) fn revalidated_tiered_source(&self) -> Option<&rustfs_filemeta::FileInfo> { + self.revalidated_tiered_source.as_ref() + } +} + +fn remote_tuple_publication_target(source: &ObjectInfo) -> std::io::Result)>> { + if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !source.transitioned_object.free_version { + return Ok(None); + } + if source.transitioned_object.tier.is_empty() || source.transitioned_object.name.is_empty() { + return Err(std::io::Error::other( + "remote tuple publication source is missing its tier or remote object identity", + )); + } + let backend_identity = tier_destination_id_from_metadata(source.user_defined.as_ref())?; + Ok(Some((source.transitioned_object.tier.clone(), backend_identity))) +} + +fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current: &ObjectInfo) -> bool { + let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else { + return false; + }; + let parts_match = expected.parts.len() == current.parts.len() + && expected.parts.iter().all(|expected_part| { + current + .parts + .iter() + .find(|current_part| current_part.number == expected_part.number) + .is_some_and(|current_part| { + current_part.size == expected_part.size + && current_part.actual_size == expected_part.actual_size + && current_part.etag == expected_part.etag + }) + }); + + expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil()) + && expected.data_dir == current.data_dir + && expected.version_id == current.version_id + && expected.delete_marker == current.delete_marker + && expected.size == current.size + && expected_actual_size == current_actual_size + && expected.checksum == current.checksum + && expected.mod_time == current.mod_time + && expected.storage_class == current.storage_class + && crate::data_movement::is_equivalent_data_movement_metadata( + expected, + current, + expected_actual_size, + current_actual_size, + ) + && expected.user_tags == current.user_tags + && expected.expires == current.expires + && expected.replication_status_internal == current.replication_status_internal + && expected.replication_status == current.replication_status + && expected.version_purge_status_internal == current.version_purge_status_internal + && expected.version_purge_status == current.version_purge_status + && expected.transitioned_object.name == current.transitioned_object.name + && expected.transitioned_object.version_id == current.transitioned_object.version_id + && expected.transitioned_object.tier == current.transitioned_object.tier + && expected.transitioned_object.free_version == current.transitioned_object.free_version + && expected.transitioned_object.status == current.transitioned_object.status + && expected.transition_version_state == current.transition_version_state + && parts_match +} + +impl RemoteTuplePublicationFence { + pub(crate) fn under_fixed_read_anchor(mut self, anchor: DecommissionFixedReadAnchor) -> Result { + if self.include_fixed_domain { + return Err(Error::other( + "data movement publication capability cannot attach a fixed read anchor after requesting fixed write", + )); + } + if anchor.is_lock_lost() { + return Err(Error::other( + "data movement publication capability received an already-lost fixed read anchor", + )); + } + self.include_fixed_domain = self.backend_target.is_some(); + self.fixed_read_anchor = Some(anchor); + Ok(self) + } + + pub(crate) fn fixed_read_anchor_guard(&self) -> Option<&ObjectLockDiagGuard> { + self.fixed_read_anchor.as_ref().map(DecommissionFixedReadAnchor::guard) + } + + pub(crate) async fn into_commit_guard( + self, + target_pool_idx: usize, + bucket: &str, + object: &str, + ) -> Result { + let Self { + store, + bucket: expected_bucket, + object: expected_object, + source_pool_idx, + source, + include_fixed_domain, + fixed_read_anchor, + backend_target, + } = self; + if bucket != expected_bucket || object != expected_object { + return Err(Error::other_with_context( + "data movement publication capability target mismatch", + format!("expected {expected_bucket}/{expected_object}, got {bucket}/{object}"), + )); + } + if fixed_read_anchor + .as_ref() + .is_some_and(DecommissionFixedReadAnchor::is_lock_lost) + { + return Err(Error::other("data movement publication fixed read anchor was lost before commit")); + } + // Never upgrade a held read lock in place. Releasing here keeps the + // long transfer on a read anchor while the commit obtains a short + // fixed-domain write lock in the global tier -> namespace order. + drop(fixed_read_anchor); + // Tier generation is acquired at the commit boundary, before any + // namespace lock, preserving bucket -> tier -> namespace lock order. + let backend_lease = if let Some((tier_name, backend_identity)) = backend_target { + let manager = store.tier_config_mgr(); + Some( + match backend_identity { + Some(identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, &tier_name, identity).await + } + None => TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await, + } + .map_err(Error::other)?, + ) + } else { + None + }; + + let guards = store + .acquire_data_movement_publication_write_locks(bucket, object, source_pool_idx, target_pool_idx, include_fixed_domain) + .await?; + if guards.iter().any(ObjectLockDiagGuard::is_lock_lost) + || backend_lease.as_ref().is_some_and(|lease| !lease.is_current_generation()) + { + return Err(Error::other("data movement publication fence was lost before source revalidation")); + } + + let source_pool = store.pools.get(source_pool_idx).ok_or_else(|| { + Error::other_with_context("invalid data movement source pool", format!("pool index {source_pool_idx}")) + })?; + let mut revalidated_tiered_source = None; + let source_matches = match source { + RemoteTuplePublicationSource::Object(expected) => { + let lookup_opts = ObjectOptions { + versioned: expected.version_id.is_some(), + version_id: expected.version_id.map(|version_id| version_id.to_string()), + incl_free_versions: expected.transitioned_object.free_version, + include_part_checksums: true, + no_lock: true, + ..Default::default() + }; + match source_pool.get_object_info(bucket, object, &lookup_opts).await { + Ok(current) => remote_tuple_publication_object_source_matches(&expected, ¤t), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => false, + Err(err) => return Err(err), + } + } + RemoteTuplePublicationSource::Tiered(expected) => { + let logical_object = decode_dir_object(object); + let current_versions = source_pool + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, &logical_object) + .await?; + let is_free_version = expected.tier_free_version(); + let Some(current) = current_versions.as_ref().and_then(|versions| { + versions.versions.iter().find(|current| { + current.version_id == expected.version_id && current.tier_free_version() == is_free_version + }) + }) else { + return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object])); + }; + if tiered_data_movement_source_matches(&expected, current)? { + revalidated_tiered_source = Some(current.clone()); + true + } else { + false + } + } + }; + if !source_matches { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_string(), + object.to_string(), + String::new(), + )); + } + + let commit_guard = RemoteTuplePublicationCommitGuard { + guards, + backend_lease, + revalidated_tiered_source, + }; + if commit_guard.is_lock_lost() { + return Err(Error::other("data movement publication fence was lost after source revalidation")); + } + Ok(commit_guard) + } +} + impl ECStore { + /// Acquire the pre-copy half of a data-movement publication fence. Legacy + /// remote objects without a destination identity pin the named tier; an + /// ordinary local source takes the zero-tier-manager-work `None` lease + /// branch but still receives source-at-commit validation. + pub(crate) async fn acquire_remote_tuple_publication_fence( + self: &Arc, + bucket: &str, + source_pool_idx: usize, + source: &ObjectInfo, + include_fixed_domain: bool, + ) -> Result { + if self.pools.get(source_pool_idx).is_none() { + return Err(Error::other_with_context( + "data movement source pool is out of range", + format!("pool index {source_pool_idx} for {bucket}/{}", source.name), + )); + } + let backend_target = remote_tuple_publication_target(source).map_err(Error::other)?; + let include_fixed_domain = include_fixed_domain && backend_target.is_some(); + Ok(RemoteTuplePublicationFence { + store: Arc::clone(self), + bucket: bucket.to_string(), + object: encode_dir_object(&source.name), + source_pool_idx, + source: RemoteTuplePublicationSource::Object(source.clone()), + include_fixed_domain, + fixed_read_anchor: None, + backend_target, + }) + } + + async fn acquire_tiered_remote_tuple_publication_fence( + self: &Arc, + bucket: &str, + object: &str, + source_pool_idx: usize, + source: &rustfs_filemeta::FileInfo, + ) -> Result { + let object_info = ObjectInfo::from_file_info(source, bucket, object, true); + let Some((tier_name, backend_identity)) = remote_tuple_publication_target(&object_info).map_err(Error::other)? else { + return Err(Error::other("tiered data movement source has no publishable remote tuple")); + }; + Ok(RemoteTuplePublicationFence { + store: Arc::clone(self), + bucket: bucket.to_string(), + object: encode_dir_object(object), + source_pool_idx, + source: RemoteTuplePublicationSource::Tiered(source.clone()), + include_fixed_domain: true, + fixed_read_anchor: None, + backend_target: Some((tier_name, backend_identity)), + }) + } + /// Captures Object Lock state once for a batch of PUTs to the same bucket. /// `handle_put_object` only reuses the token for the same store, bucket, /// bucket incarnation, and Object Lock configuration revision. @@ -2215,7 +2883,7 @@ impl ECStore { &self, bucket: &str, object: &str, - ) -> Result { + ) -> Result { if self.ctx.lock_manager().is_disabled() { return Err(Error::other("decommission object migration requires namespace locking")); } @@ -2235,7 +2903,7 @@ impl ECStore { guard.test_namespace_lock_fence = test_namespace_lock_fence; guard }; - Ok(guard) + Ok(DecommissionFixedReadAnchor { guard }) } pub(super) async fn apply_decommission_target_mutation_fence( @@ -2353,39 +3021,201 @@ impl ECStore { Ok(guards) } - async fn acquire_data_movement_object_write_locks( + /// Acquire the fixed store domain followed by every physical set domain in + /// stable `(pool_idx, set_idx)` order. This stronger form is reserved for + /// remote-tuple destruction proofs: old topology generations and failed + /// movements may leave a valid source outside the current hash-selected + /// set, so a hashed-only lock/read is not authoritative. + pub(crate) async fn acquire_all_physical_object_read_locks( + &self, + op: &'static str, + bucket: &str, + object: &str, + opts: &mut ObjectOptions, + ) -> Result> { + let diag_enabled = is_object_lock_diag_enabled(); + let distributed = self.ctx.is_dist_erasure().await; + let mut guards = Vec::new(); + if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? { + guards.push(guard); + } + + let fixed_set = Arc::clone(&self.pools[0].disk_set[0]); + let mut locked_sets = vec![fixed_set]; + for pool in &self.pools { + for set in &pool.disk_set { + let lock_domain_already_held = !distributed + || locked_sets + .iter() + .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers)); + if lock_domain_already_held { + continue; + } + let ns_lock = set.new_ns_lock(bucket, object).await?; + let acquire_start = Instant::now(); + let guard = ns_lock + .get_read_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?; + let owner = diag_enabled.then(|| ns_lock.owner().to_string()); + log_object_lock_acquire_if_slow( + op, + bucket, + object, + owner.as_deref(), + ObjectLockDiagMode::Read, + acquire_start.elapsed(), + diag_enabled, + ); + guards.push(ObjectLockDiagGuard::new( + guard, + diag_enabled, + op, + diag_enabled.then(|| bucket.to_string()), + diag_enabled.then(|| object.to_string()), + owner, + ObjectLockDiagMode::Read, + )); + locked_sets.push(Arc::clone(set)); + } + } + Ok(guards) + } + + pub(crate) async fn acquire_all_physical_object_write_locks( + &self, + op: &'static str, + bucket: &str, + object: &str, + ) -> Result> { + self.acquire_physical_object_write_locks(op, bucket, object, false).await + } + + async fn acquire_remaining_physical_object_write_locks( + &self, + op: &'static str, + bucket: &str, + object: &str, + ) -> Result> { + self.acquire_physical_object_write_locks(op, bucket, object, true).await + } + + async fn acquire_physical_object_write_locks( + &self, + op: &'static str, + bucket: &str, + object: &str, + store_lock_already_held: bool, + ) -> Result> { + // Caller-held bucket lifecycle and tier-generation guards precede the + // fixed store domain, followed by every distinct physical set domain + // in stable `(pool_idx, set_idx)` order. Exact-object predecessor + // replay sets `store_lock_already_held` only while retaining that same + // fixed-domain write lock. + if self.ctx.lock_manager().is_disabled() { + return Err(Error::other("physical object mutation requires namespace locking")); + } + let diag_enabled = is_object_lock_diag_enabled(); + let distributed = self.ctx.is_dist_erasure().await; + let mut guards = Vec::new(); + if !store_lock_already_held { + guards.push(self.acquire_object_write_lock(op, bucket, object).await?); + } + let fixed_set = Arc::clone(&self.pools[0].disk_set[0]); + let mut locked_sets = vec![fixed_set]; + for pool in &self.pools { + for set in &pool.disk_set { + let lock_domain_already_held = !distributed + || locked_sets + .iter() + .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers)); + if lock_domain_already_held { + continue; + } + let ns_lock = set.new_ns_lock(bucket, object).await?; + let acquire_start = Instant::now(); + let guard = ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?; + let owner = diag_enabled.then(|| ns_lock.owner().to_string()); + log_object_lock_acquire_if_slow( + op, + bucket, + object, + owner.as_deref(), + ObjectLockDiagMode::Write, + acquire_start.elapsed(), + diag_enabled, + ); + guards.push(ObjectLockDiagGuard::new( + guard, + diag_enabled, + op, + diag_enabled.then(|| bucket.to_string()), + diag_enabled.then(|| object.to_string()), + owner, + ObjectLockDiagMode::Write, + )); + locked_sets.push(Arc::clone(set)); + } + } + Ok(guards) + } + + pub(crate) async fn acquire_data_movement_publication_write_locks( &self, bucket: &str, object: &str, source_pool_idx: usize, target_pool_idx: usize, - opts: &mut ObjectOptions, + include_fixed_domain: bool, ) -> Result> { if self.ctx.lock_manager().is_disabled() { - return Err(Error::other("tiered data movement requires namespace locking")); + return Err(Error::other("data movement publication requires namespace locking")); } let distributed = self.ctx.is_dist_erasure().await; let diag_enabled = is_object_lock_diag_enabled(); let mut pool_indices = [source_pool_idx, target_pool_idx]; pool_indices.sort_unstable(); let fixed_set = Arc::clone(&self.pools[0].disk_set[0]); - let mut locked_sets = vec![fixed_set]; + let mut locked_sets = Vec::with_capacity(3); let mut guards = Vec::with_capacity(3); + #[cfg(test)] + let test_namespace_lock_fence = decommission_mutation_fence_for_test( + bucket, + &decode_dir_object(object), + DecommissionMutationFenceTestPhase::Migration, + ); - // Lock order matches journal recovery: fixed store domain first, then - // hashed domains by ascending pool index. This also serializes source - // revalidation and target publication against ordinary object deletes. - guards.push(self.acquire_object_write_lock("tiered_data_movement", bucket, object).await?); + // Decommission publishers take the fixed store domain first to match + // tier-delete recovery. Rebalance publishers do not need that global + // serialization, but both paths take source/target domains in stable + // pool order and revalidate the source before target publication. + if include_fixed_domain { + let guard = self + .acquire_object_write_lock("data_movement_publication", bucket, object) + .await?; + #[cfg(test)] + let guard = { + let mut guard = guard; + guard.test_namespace_lock_fence = test_namespace_lock_fence.clone(); + guard + }; + guards.push(guard); + locked_sets.push(fixed_set); + } for pool_idx in pool_indices { let pool = self .pools .get(pool_idx) - .ok_or_else(|| Error::other(format!("invalid tiered data movement pool {pool_idx}")))?; + .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?; let set = pool.get_disks_by_key(object); - let lock_domain_already_held = !distributed - || locked_sets.iter().any(|locked_set: &Arc| { - same_distributed_lock_domain(&locked_set.lockers, &set.lockers) - }); + let lock_domain_already_held = !locked_sets.is_empty() + && (!distributed + || locked_sets.iter().any(|locked_set: &Arc| { + same_distributed_lock_domain(&locked_set.lockers, &set.lockers) + })); if lock_domain_already_held { continue; } @@ -2397,7 +3227,7 @@ impl ECStore { .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?; let owner = diag_enabled.then(|| ns_lock.owner().to_string()); log_object_lock_acquire_if_slow( - "tiered_data_movement", + "data_movement_publication", bucket, object, owner.as_deref(), @@ -2405,22 +3235,26 @@ impl ECStore { acquire_start.elapsed(), diag_enabled, ); - guards.push(ObjectLockDiagGuard::new( + let guard = ObjectLockDiagGuard::new( guard, diag_enabled, - "tiered_data_movement", + "data_movement_publication", diag_enabled.then(|| bucket.to_string()), diag_enabled.then(|| object.to_string()), owner, ObjectLockDiagMode::Write, - )); + ); + #[cfg(test)] + let guard = { + let mut guard = guard; + if guards.is_empty() { + guard.test_namespace_lock_fence = test_namespace_lock_fence.clone(); + } + guard + }; + guards.push(guard); locked_sets.push(set); } - opts.no_lock = true; - for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) { - opts.add_namespace_lock_lost_signal(signal); - } - opts.ensure_namespace_lock_fence(); Ok(guards) } @@ -2583,7 +3417,7 @@ impl ECStore { #[instrument(skip(self, fi, opts))] pub(crate) async fn decommission_tiered_object( - &self, + self: &Arc, bucket: &str, object: &str, fi: &rustfs_filemeta::FileInfo, @@ -2609,6 +3443,12 @@ impl ECStore { Some(guard) }; + // Capture exact source/tier identity now; the tier lease and namespace + // writes are acquired only at the final metadata publication boundary. + let publication_fence = self + .acquire_tiered_remote_tuple_publication_fence(bucket, object, opts.src_pool_idx, fi) + .await?; + let logical_object = object; let object = encode_dir_object(logical_object); ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts); @@ -2638,30 +3478,13 @@ impl ECStore { if is_free_version { pause_decommission_free_version_before_source_lock(bucket, logical_object).await; } - let _object_guards = self - .acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts) - .await?; - let source_pool = self - .pools - .get(opts.src_pool_idx) - .ok_or_else(|| Error::other(format!("invalid tiered data movement source pool {}", opts.src_pool_idx)))?; - let source_versions = source_pool - .get_disks_by_key(&object) - .load_file_info_versions_exact(bucket, logical_object) - .await?; - let current_source = source_versions - .as_ref() - .and_then(|versions| { - versions - .versions - .iter() - .find(|current| current.version_id == fi.version_id && current.tier_free_version() == is_free_version) - }) - .ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?; - if !tiered_data_movement_source_matches(fi, current_source)? { - return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()])); - } - let mut fi = current_source.clone(); + let publication_guard = publication_fence.into_commit_guard(idx, bucket, &object).await?; + publication_guard.add_namespace_lock_fence(&mut opts); + opts.no_lock = true; + let mut fi = publication_guard + .revalidated_tiered_source() + .ok_or_else(|| Error::other("tiered data movement publication guard is missing its revalidated source"))? + .clone(); if opts.data_movement { crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?; } @@ -2853,13 +3676,39 @@ impl ECStore { Ok(idx) } + #[cfg(test)] pub(crate) async fn put_object_for_data_movement( - &self, + self: &Arc, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions, mutation_fence: Option<&ObjectLockDiagGuard>, + ) -> Result<(usize, Result)> { + self.put_object_for_data_movement_inner(bucket, object, data, opts, mutation_fence, None) + .await + } + + pub(crate) async fn put_object_for_data_movement_with_publication_fence( + self: &Arc, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + publication_fence: RemoteTuplePublicationFence, + ) -> Result<(usize, Result)> { + self.put_object_for_data_movement_inner(bucket, object, data, opts, None, Some(publication_fence)) + .await + } + + async fn put_object_for_data_movement_inner( + self: &Arc, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, + publication_fence: Option, ) -> Result<(usize, Result)> { if !opts.data_movement { return Err(Error::other("data movement PUT requires data_movement options")); @@ -2869,7 +3718,10 @@ impl ECStore { let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; - self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence) + let fixed_read_anchor = publication_fence + .as_ref() + .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard); + self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor)) .await; let expected_data_bytes = usize::try_from(data.size()).ok(); let result = self @@ -2881,10 +3733,25 @@ impl ECStore { if let Some(capacity_lease) = capacity_lease { opts.add_namespace_lock_lost_signal(capacity_lease); } - self.pools[idx] - .put_object_with_old_current_size(bucket, &object, data, &opts) - .await - .map(|(object_info, _)| object_info) + let result = match publication_fence { + Some(publication_fence) => { + self.pools[idx] + .put_object_with_old_current_size_for_data_movement( + bucket, + &object, + data, + &opts, + publication_fence, + ) + .await + } + None => { + self.pools[idx] + .put_object_with_old_current_size(bucket, &object, data, &opts) + .await + } + }; + result.map(|(object_info, _)| object_info) }, ) .await; @@ -3074,7 +3941,7 @@ impl ECStore { } // Transitioned object self-copy: restore from tier into the same pool. let mut put_opts = ObjectOptions { - user_defined: (*src_info.user_defined).clone(), + user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, @@ -3110,7 +3977,7 @@ impl ECStore { // a metadata-only version copy would corrupt SSE/compressed objects (issue #4238). if let Some(reader) = src_info.put_object_reader.as_mut() { let mut put_opts = ObjectOptions { - user_defined: (*src_info.user_defined).clone(), + user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, @@ -3156,7 +4023,7 @@ impl ECStore { let dst_object_name = dst_object.as_str(); let mut put_opts = ObjectOptions { - user_defined: (*src_info.user_defined).clone(), + user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, @@ -3270,6 +4137,16 @@ impl ECStore { object: &str, opts: ObjectOptions, tier_journal_api: Option>, + ) -> Result { + Box::pin(self.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)).await + } + + async fn handle_delete_object_with_journal_inner( + &self, + bucket: &str, + object: &str, + opts: ObjectOptions, + tier_journal_api: Option>, ) -> Result { check_del_obj_args(bucket, object)?; @@ -3332,59 +4209,14 @@ impl ECStore { { return Err(StorageError::BucketNotFound(bucket.to_string())); } + if opts.delete_prefix && opts.expected_bucket_incarnation_id.is_none() { + opts.expected_bucket_incarnation_id = current_bucket_incarnation_id; + } #[cfg(test)] pause_delete_after_object_lock_snapshot(bucket).await; if opts.delete_prefix && !opts.delete_prefix_object { - // Prefix deletes cover multiple object keys; an exact lock on the prefix string - // would not protect child objects. - if !is_meta_bucketname(bucket) { - let state = opts - .object_lock_config_snapshot - .as_deref() - .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))? - .state(); - ensure_recursive_force_delete_allowed_for_state(bucket, state)?; - let bypass_governance = opts - .object_lock_delete - .as_ref() - .is_some_and(|delete_opts| delete_opts.bypass_governance); - for pool in &self.pools { - for set in &pool.disk_set { - let mut marker = None; - let mut version_marker = None; - loop { - let page = set - .clone() - .inner_list_object_versions_for_recursive_delete( - bucket, - object, - marker.clone(), - version_marker.clone(), - RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE, - ) - .await?; - for object_info in &page.objects { - if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() { - return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone())); - } - } - if !page.is_truncated { - break; - } - let next_marker = page.next_marker.ok_or_else(|| { - Error::other("recursive delete version scan did not return a continuation marker") - })?; - if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker { - return Err(Error::other("recursive delete version scan did not advance")); - } - marker = Some(next_marker); - version_marker = page.next_version_idmarker; - } - } - } - } - delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?; + delete_recursive_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?; return Ok(ObjectInfo::default()); } @@ -3607,25 +4439,8 @@ impl ECStore { )); } - let journal_entry = if let Some(api) = tier_journal_api.as_ref() { - prepare_tier_delete_journal_entry(api, bucket, object, &opts, &pinfo.object_info).await? - } else { - None - }; - if should_delete_from_all_pools(&opts, errs.len()) { - let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await { - Ok(obj) => obj, - Err(err) => { - if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { - abort_prepared_tier_delete_journal_entry(api, je).await; - } - return Err(err); - } - }; - if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { - commit_prepared_tier_delete_journal_entry(api, je).await; - } + let mut obj = self.delete_object_from_all_pools(bucket, object, &opts, errs).await?; obj.name = decode_dir_object(object); return Ok(obj); } @@ -3651,9 +4466,6 @@ impl ECStore { Ok(res) => { #[cfg(test)] pause_versioned_delete_marker_after_commit(bucket, object).await; - if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { - commit_prepared_tier_delete_journal_entry(api, je).await; - } let mut obj = res; obj.name = decode_dir_object(object); return Ok(obj); @@ -3666,10 +4478,6 @@ impl ECStore { } } - if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { - abort_prepared_tier_delete_journal_entry(api, je).await; - } - if let Some(ver) = opts.version_id { return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver)); } @@ -4379,11 +5187,33 @@ mod tests { use crate::storage_api_contracts::bucket::MakeBucketOptions; use crate::storage_api_contracts::lifecycle::TransitionedObject; use bytes::Bytes; + use std::future::Future as _; use std::io::Cursor; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::io::AsyncReadExt; + #[test] + fn predecessor_replay_scope_distinguishes_exact_object_from_recursive_prefix() { + let source = TierDeleteSourceIdentity { + bucket: "bucket".to_string(), + object: "directory/child".to_string(), + version_id: Some("version".to_string()), + versioned: true, + version_suspended: false, + data_dir: Some("data-dir".to_string()), + etag: Some("etag".to_string()), + mod_time: Some("mod-time".to_string()), + }; + let persisted_prefix = encode_dir_object("directory/"); + + assert!(tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, false,)); + assert!(!tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, true,)); + let mut exact_source = source; + exact_source.object = "directory/".to_string(); + assert!(tier_delete_source_matches_replay_scope(&exact_source, "bucket", &persisted_prefix, true,)); + } + struct WaitForLockLossReader { inner: Cursor>, poll_started: Option>, @@ -4419,6 +5249,33 @@ mod tests { } } + struct CommitLateBodyReader { + inner: Cursor>, + poll_started: Option>, + release: Option>, + } + + impl AsyncRead for CommitLateBodyReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if let Some(poll_started) = self.poll_started.take() { + let _ = poll_started.send(()); + } + if let Some(release) = self.release.as_mut() { + match Pin::new(release).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => self.release = None, + Poll::Ready(Err(_)) => { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "commit-late body release sender was dropped", + ))); + } + } + } + Pin::new(&mut self.inner).poll_read(cx, buf) + } + } + struct CountingMissHook { calls: AtomicUsize, } @@ -4502,7 +5359,7 @@ mod tests { } #[tokio::test] - async fn decommission_fence_covers_dist_sets_with_same_clients_despite_different_namespaces() { + async fn publication_commit_locks_dedupe_dist_sets_with_same_clients() { let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await; let mut second_set = (*original_sets.disk_set[1]).clone(); @@ -4528,10 +5385,10 @@ mod tests { .map(|index| format!("decommission-dist-domain-{index}.bin")) .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1])) .expect("a key should hash to the second set namespace"); - let mutation_fence = store - .acquire_decommission_object_mutation_fence("bucket", &object) + let publication_guards = store + .acquire_data_movement_publication_write_locks("bucket", &encode_dir_object(&object), 0, 0, true) .await - .expect("the fixed distributed mutation fence should be acquired"); + .expect("the publication commit lock set should be acquired"); let target_lock = sets.disk_set[1] .new_ns_lock("bucket", &object) .await @@ -4539,21 +5396,10 @@ mod tests { let target_err = target_lock .get_write_lock(Duration::from_millis(50)) .await - .expect_err("the fixed read fence must conflict through the shared clients"); + .expect_err("the fixed write fence must conflict through the shared clients"); assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. })); - let mut put_opts = ObjectOptions::default(); - store - .apply_decommission_target_mutation_fence(0, &object, &mut put_opts, Some(&mutation_fence)) - .await; - assert!(put_opts.no_lock, "migration target PUT must reuse the covering fixed fence"); - - let mut multipart_opts = ObjectOptions::default(); - store - .apply_decommission_target_mutation_fence(0, &object, &mut multipart_opts, Some(&mutation_fence)) - .await; - assert!(multipart_opts.no_lock, "migration target multipart must reuse the covering fixed fence"); - drop(mutation_fence); + drop(publication_guards); let cleanup_object = (0..1_000) .map(|index| format!("decommission-dist-cleanup-{index}.bin")) @@ -4575,6 +5421,438 @@ mod tests { assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. })); } + #[tokio::test] + async fn physical_absence_proof_blocks_decommission_remote_tuple_publication() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await; + ctx.update_erasure_type(SetupType::DistErasure).await; + let pool_config = sets.endpoints.clone(); + let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx); + let bucket = "remote-tuple-publication-fence"; + let object = "transitioned.bin"; + let encoded = encode_dir_object(object); + let mut proof_opts = ObjectOptions::default(); + let proof = store + .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts) + .await + .expect("the all-physical absence proof should acquire every read domain"); + + // Keep polling the same distributed-lock request after the bounded + // observation below. Dropping a timed-out request can race with the + // lock service granting it and would turn the test itself into an + // abandoned-owner scenario. + let mut publication = Box::pin(store.acquire_data_movement_publication_write_locks(bucket, &encoded, 0, 0, true)); + let blocked = tokio::time::timeout(Duration::from_millis(50), &mut publication).await; + assert!( + blocked.is_err(), + "a decommission publisher must remain blocked while the remote-delete absence proof is held" + ); + + drop(proof); + publication + .await + .expect("the publisher should acquire its write fence after the proof releases"); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn restored_transitioned_rebalance_revalidates_source_after_recovery_lock() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new( + new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], Arc::clone(&ctx)).await, + ); + let bucket = "restored-transitioned-rebalance-publication"; + let object = "restored.bin"; + for set in [&first_set, &second_set] { + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("publication-race bucket should be created"); + } + + let tier_name = "RESTORED-REBALANCE"; + crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), tier_name).await; + let lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), tier_name) + .await + .expect("mock tier generation should be available"); + let backend_identity = lease.backend_identity(); + drop(lease); + + let mut metadata = HashMap::new(); + for (suffix, value) in [ + (rustfs_utils::http::SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.to_string()), + (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/restored.bin".to_string()), + (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()), + (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-version".to_string()), + ( + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(), + ), + ( + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(backend_identity), + ), + ] { + rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, value); + } + metadata.insert( + "x-amz-restore".to_string(), + "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(), + ); + let mut body = PutObjReader::from_vec(b"restored local body".to_vec()); + first_set + .put_object( + bucket, + object, + &mut body, + &ObjectOptions { + user_defined: metadata, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("restored-transitioned source should be written"); + let source = first_set + .get_object_info( + bucket, + object, + &ObjectOptions { + include_part_checksums: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("restored-transitioned source metadata should be readable"); + assert_eq!(source.transitioned_object.status, TRANSITION_COMPLETE); + assert!(source.data_dir.is_some(), "fixture must retain restored local data"); + + let publisher_first = store + .acquire_remote_tuple_publication_fence(bucket, 0, &source, false) + .await + .expect("rebalance should capture an exact publication capability"); + let encoded = encode_dir_object(object); + let publisher_guard = publisher_first + .into_commit_guard(1, bucket, &encoded) + .await + .expect("unchanged source should validate at the publication boundary"); + let mut publisher_first_proof_opts = ObjectOptions::default(); + let mut publisher_first_proof = Box::pin(store.acquire_all_physical_object_read_locks( + "tier_delete_journal_recovery", + bucket, + &encoded, + &mut publisher_first_proof_opts, + )); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut publisher_first_proof) + .await + .is_err(), + "recovery must wait while a validated publisher owns the commit scope" + ); + drop(publisher_guard); + drop( + publisher_first_proof + .await + .expect("recovery proof should acquire after publisher commit scope releases"), + ); + + let publication = store + .acquire_remote_tuple_publication_fence(bucket, 0, &source, false) + .await + .expect("recovery-first race should capture the old source identity"); + + // The pre-copy capability itself must not hold a namespace write lock. + let pre_commit_reader = first_set + .new_ns_lock(bucket, &encoded) + .await + .expect("source read lock should be created") + .get_read_lock(Duration::from_millis(50)) + .await + .expect("body staging must not be blocked by an early publication write lock"); + drop(pre_commit_reader); + + first_set + .delete_object( + bucket, + &encoded, + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + data_movement: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("source cleanup should remove the old physical owner"); + + let mut proof_opts = ObjectOptions::default(); + let proof = store + .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts) + .await + .expect("recovery should hold its all-physical absence proof"); + let mut commit = Box::pin(publication.into_commit_guard(1, bucket, &encoded)); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut commit).await.is_err(), + "publisher must wait while recovery owns the physical absence proof" + ); + drop(proof); + let err = match commit.await { + Ok(_) => panic!("publisher must reject its stale source snapshot after recovery releases"), + Err(err) => err, + }; + assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _))); + assert!( + second_set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "stale publication must not create a target owner" + ); + } + + #[tokio::test] + async fn data_movement_put_defers_publication_locks_until_body_complete() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = + Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await); + let bucket = "commit-late-data-movement-put"; + let object = "slow-body.bin"; + for set in [&first_set, &second_set] { + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("commit-late PUT bucket should be created"); + } + + let payload = b"data movement body".to_vec(); + let mut source_body = PutObjReader::from_vec(payload.clone()); + first_set + .put_object(bucket, object, &mut source_body, &ObjectOptions::default()) + .await + .expect("data-movement source should be written"); + let source = first_set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("data-movement source should be readable"); + let publication = store + .acquire_remote_tuple_publication_fence(bucket, 0, &source, true) + .await + .expect("decommission should capture its publication capability"); + + let payload_len = i64::try_from(payload.len()).expect("test payload length should fit in i64"); + let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let slow_body = CommitLateBodyReader { + inner: Cursor::new(payload), + poll_started: Some(poll_started_tx), + release: Some(release_rx), + }; + let target_set = Arc::clone(&store.pools[1]); + let target_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + mod_time: source.mod_time, + preserve_etag: source.etag.clone(), + user_defined: source.user_defined.as_ref().clone(), + ..Default::default() + }; + let worker = tokio::spawn(async move { + let hash_reader = rustfs_rio::HashReader::from_stream(slow_body, payload_len, payload_len, None, None, false) + .expect("slow body should produce a valid hash reader"); + let mut reader = PutObjReader::new(hash_reader); + target_set + .put_object_with_old_current_size_for_data_movement(bucket, object, &mut reader, &target_opts, publication) + .await + }); + + tokio::time::timeout(Duration::from_secs(5), poll_started_rx) + .await + .expect("target PUT should start polling its body") + .expect("target PUT should retain the poll observer"); + let source_head = tokio::time::timeout( + Duration::from_millis(250), + first_set.get_object_info(bucket, object, &ObjectOptions::default()), + ) + .await + .expect("same-key HEAD must not wait behind body staging") + .expect("the source must remain readable during body staging"); + assert_eq!(source_head.etag, source.etag); + let target_head = tokio::time::timeout( + Duration::from_millis(250), + second_set.get_object_info(bucket, object, &ObjectOptions::default()), + ) + .await + .expect("target HEAD must not wait behind body staging") + .expect_err("the target must remain unpublished before body EOF"); + assert!(is_err_object_not_found(&target_head) || is_err_version_not_found(&target_head)); + + release_tx + .send(()) + .expect("target PUT should still be waiting on the body gate"); + worker + .await + .expect("target PUT task should join") + .expect("target PUT should commit after body EOF"); + second_set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("target metadata should be visible after the commit boundary"); + } + + #[test] + fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() { + let protected_suffixes = [ + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_utils::http::SUFFIX_TRANSITION_TIER, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID, + rustfs_utils::http::SUFFIX_FREE_VERSION, + rustfs_utils::http::SUFFIX_TIER_FV_ID, + rustfs_utils::http::SUFFIX_TIER_FV_MARKER, + ]; + let mut source = HashMap::from([("content-type".to_string(), "application/octet-stream".to_string())]); + for suffix in protected_suffixes { + rustfs_utils::http::metadata_compat::insert_str(&mut source, suffix, format!("source-{suffix}")); + } + // Compatibility readers are case-insensitive, so the scrubber must + // remove non-canonical casing as well as both canonical prefixes. + source.insert( + format!( + "{}{}", + rustfs_utils::http::MINIO_INTERNAL_PREFIX.to_ascii_uppercase(), + rustfs_utils::http::SUFFIX_TRANSITION_STATUS.to_ascii_uppercase() + ), + "complete".to_string(), + ); + + let copied = materialized_copy_user_defined(&source); + + assert_eq!(copied.get("content-type").map(String::as_str), Some("application/octet-stream")); + for suffix in protected_suffixes { + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str(&copied, suffix), + "materialized copy retained protected internal suffix {suffix}" + ); + } + } + + #[test] + fn remote_tuple_publication_target_is_exact_legacy_compatible_and_ordinary_zero_cost() { + let mut ordinary_metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut ordinary_metadata, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + "malformed-but-irrelevant".to_string(), + ); + let ordinary = ObjectInfo { + user_defined: Arc::new(ordinary_metadata), + ..Default::default() + }; + assert!( + remote_tuple_publication_target(&ordinary) + .expect("ordinary objects must not parse or lock tier metadata") + .is_none() + ); + + let identity = [0xabu8; 32]; + let mut exact_metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut exact_metadata, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(identity), + ); + let transitioned = ObjectInfo { + transitioned_object: TransitionedObject { + status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(), + name: "remote/object".to_string(), + tier: "WARM".to_string(), + ..Default::default() + }, + user_defined: Arc::new(exact_metadata), + ..Default::default() + }; + let (tier, recorded_identity) = remote_tuple_publication_target(&transitioned) + .expect("exact transitioned metadata should parse") + .expect("transitioned object should require a publication fence"); + assert_eq!(tier, "WARM"); + assert_eq!(recorded_identity, Some(identity)); + + let legacy = ObjectInfo { + user_defined: Arc::new(HashMap::new()), + ..transitioned.clone() + }; + assert_eq!( + remote_tuple_publication_target(&legacy) + .expect("legacy transitioned metadata should remain compatible") + .map(|(_, identity)| identity), + Some(None) + ); + + let mut malformed = transitioned; + malformed.transitioned_object.tier.clear(); + assert!(remote_tuple_publication_target(&malformed).is_err()); + } + + #[test] + fn publication_source_match_rejects_new_physical_generation_with_same_logical_identity() { + let expected = ObjectInfo { + data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + assert!(remote_tuple_publication_object_source_matches(&expected, &expected)); + + let current = ObjectInfo { + data_dir: Some(Uuid::new_v4()), + ..expected.clone() + }; + assert!( + crate::data_movement::is_equivalent_data_movement_object_identity(&expected, ¤t, true, true), + "the logical target-equivalence check deliberately ignores target data-dir allocation" + ); + assert!( + !remote_tuple_publication_object_source_matches(&expected, ¤t), + "publication revalidation must additionally bind the physical source generation" + ); + } + + #[tokio::test] + async fn generic_data_movement_put_rejects_transition_ownership_without_capability() { + let (_dirs, set) = make_local_set_disks(4, 2).await; + let bucket = "unfenced-transition-publication"; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("unfenced publication bucket should be created"); + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_TRANSITION_STATUS, + TRANSITION_COMPLETE.to_string(), + ); + let mut body = PutObjReader::from_vec(b"must not publish".to_vec()); + let err = set + .put_object( + bucket, + "unfenced.bin", + &mut body, + &ObjectOptions { + data_movement: true, + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect_err("transition ownership must require the typed publication entry point"); + assert!(err.to_string().contains("publication capability")); + } + #[test] fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() { let nil = Uuid::nil(); @@ -4735,7 +6013,7 @@ mod tests { } #[test] - fn tier_delete_entry_is_prepared_and_bound_to_source_generation() { + fn tier_delete_entry_is_prepared_and_prefix_legacy_fails_closed() { let identity = [9_u8; 32]; let mut metadata = HashMap::new(); rustfs_utils::http::metadata_compat::insert_str( @@ -4781,6 +6059,65 @@ mod tests { entry.source.as_ref().and_then(|source| source.data_dir.as_deref()), Some(data_dir_string.as_str()) ); + + let mut unknown = source.clone(); + unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; + assert!( + build_tier_delete_journal_entry( + "bucket", + "object", + &ObjectOptions { + delete_prefix: true, + versioned: true, + ..Default::default() + }, + &unknown, + ) + .expect_err("recursive prefix delete must fail closed for legacy Unknown metadata") + .to_string() + .contains("legacy Unknown") + ); + + assert!( + build_tier_delete_journal_entry( + "bucket", + "object", + &ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + versioned: true, + lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { + version_id: Some(version_id), + delete_marker: false, + action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, + rule_id: "delete-all".to_string(), + phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, + }), + ..Default::default() + }, + &unknown, + ) + .expect("lifecycle exact-object delete-all may use its free-version fallback") + .is_none() + ); + + let mut identity_less = source; + identity_less.user_defined = Arc::new(HashMap::new()); + assert!( + build_tier_delete_journal_entry( + "bucket", + "object", + &ObjectOptions { + delete_prefix: true, + versioned: true, + ..Default::default() + }, + &identity_less, + ) + .expect_err("recursive prefix delete must fail closed without destination identity") + .to_string() + .contains("requires a stable transitioned source") + ); } impl Drop for BodyCacheHookGuard { @@ -5829,7 +7166,7 @@ mod tests { } #[test] - fn transition_restore_pool_opts_preserves_existing_no_lock() { + fn transition_restore_pool_opts_rejects_ambient_no_lock() { let lookup_opts = transition_restore_pool_opts(&ObjectOptions { no_lock: true, ..Default::default() @@ -5837,7 +7174,7 @@ mod tests { assert!(lookup_opts.skip_decommissioned); assert!(lookup_opts.skip_rebalancing); - assert!(lookup_opts.no_lock); + assert!(!lookup_opts.no_lock); } #[test] @@ -5895,6 +7232,26 @@ mod tests { } } + #[tokio::test] + async fn delete_object_handler_futures_remain_stack_bounded() { + let store = new_read_lock_test_store().await; + let unified_future = store.handle_delete_object_with_journal("bucket", "object", ObjectOptions::default(), None); + let unified_future_size = std::mem::size_of_val(&unified_future); + + assert!( + unified_future_size <= 4 * 1024, + "unified delete handler future must remain stack-bounded; measured {unified_future_size} bytes" + ); + drop(unified_future); + + let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default()); + let outer_future_size = std::mem::size_of_val(&outer_future); + assert!( + outer_future_size <= 16 * 1024, + "outer delete handler future must remain stack-bounded; measured {outer_future_size} bytes" + ); + } + async fn new_prepared_reader_test_store(set_disks: &[Arc]) -> ECStore { new_prepared_reader_test_store_with_ctx(set_disks, crate::runtime::instance::bootstrap_ctx()).await } diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 2bfe36af8..32ae15c3e 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -309,6 +309,17 @@ impl ECStore { } pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let dispatch_scope = if opts.tier_delete_journal_api.is_some() { + let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let authorization = opts + .tier_delete_dispatch_authorization + .as_ref() + .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?; + authorization.ensure_current(bucket, incarnation, object)?; + Some((authorization, incarnation)) + } else { + None + }; if opts.lifecycle_delete_all.is_some() { let mut preflight_opts = opts.clone(); preflight_opts @@ -317,6 +328,9 @@ impl ECStore { .ok_or(StorageError::PreconditionFailed)? .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight; for pool in &self.pools { + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.ensure_current(bucket, incarnation, object)?; + } #[cfg(test)] lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?; pool.delete_object(bucket, object, preflight_opts.clone()).await?; @@ -326,6 +340,9 @@ impl ECStore { .ok_or(StorageError::PreconditionFailed)? .lock() .mark_mutation_started(); + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.mark_mutation_started(bucket, incarnation, object)?; + } let mut non_trigger_opts = opts.clone(); non_trigger_opts .lifecycle_delete_all @@ -333,6 +350,9 @@ impl ECStore { .ok_or(StorageError::PreconditionFailed)? .phase = crate::object_api::LifecycleDeleteAllPhase::History; for pool in &self.pools { + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.ensure_current(bucket, incarnation, object)?; + } #[cfg(test)] lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?; let mut pool_opts = non_trigger_opts.clone(); @@ -348,6 +368,9 @@ impl ECStore { .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight; let mut trigger_pools = Vec::new(); for (pool_index, pool) in self.pools.iter().enumerate() { + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.ensure_current(bucket, incarnation, object)?; + } #[cfg(test)] lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?; let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?; @@ -366,6 +389,9 @@ impl ECStore { .ok_or(StorageError::PreconditionFailed)? .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger; for pool_index in trigger_pools { + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.ensure_current(bucket, incarnation, object)?; + } #[cfg(test)] lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?; let mut pool_opts = trigger_opts.clone(); @@ -378,7 +404,13 @@ impl ECStore { let mut first_error = None; let mut first_volume_error = None; let mut has_success = false; + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.mark_mutation_started(bucket, incarnation, object)?; + } for pool in &self.pools { + if let Some((authorization, incarnation)) = dispatch_scope { + authorization.ensure_current(bucket, incarnation, object)?; + } let mut opts = opts.clone(); opts.delete_prefix = true; match pool.delete_object(bucket, object, opts).await { diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 7e2d38de0..eb03aab9b 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -697,6 +697,11 @@ impl HealChannelProcessor { | HealRequestSource::Mrf => true, }); + // `no_lock` is an internal coordination hint. An admin request can + // carry the legacy field over the wire, but cannot use it as ambient + // authority to bypass storage namespace locking. + let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin; + // Build HealOptions with all available fields let options = HealOptions { scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal), @@ -705,7 +710,7 @@ impl HealChannelProcessor { update_parity: request.update_parity.unwrap_or(true), recursive, dry_run: request.dry_run.unwrap_or(false), - no_lock: request.no_lock.unwrap_or(false), + no_lock, timeout: request.timeout_seconds.map(std::time::Duration::from_secs), pool_index: request.pool_index, set_index: request.set_index, @@ -990,6 +995,23 @@ mod tests { assert!(heal_request.options.no_lock); } + #[tokio::test] + async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() { + let heal_manager = create_test_heal_manager(); + let processor = HealChannelProcessor::new(heal_manager); + let channel_request = HealChannelRequest { + id: "admin-no-lock".to_string(), + bucket: "test-bucket".to_string(), + object_prefix: Some("test-object".to_string()), + no_lock: Some(true), + source: HealRequestSource::Admin, + ..Default::default() + }; + + let heal_request = processor.convert_to_heal_request(channel_request).unwrap(); + assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority"); + } + #[tokio::test] async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() { let heal_manager = create_test_heal_manager(); diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 4843e7bf9..7a5058db0 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse { pub error_info: ::core::option::Option<::prost::alloc::string::String>, #[prost(bytes = "bytes", tag = "5")] pub response_proof: ::prost::bytes::Bytes, + #[prost(enumeration = "TierMutationFailureClass", tag = "6")] + pub failure_class: i32, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetLiveEventsRequest { @@ -1611,6 +1613,35 @@ impl TierMutationPeerState { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TierMutationFailureClass { + Unspecified = 0, + PreDispatchRejected = 1, + Ambiguous = 2, +} +impl TierMutationFailureClass { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED", + Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED", + Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified), + "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected), + "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous), + _ => None, + } + } +} /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index e50fdf031..691f9a6f3 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0"; pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024; pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024; +pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE; pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096; pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result { @@ -342,7 +343,16 @@ impl TierMutationRpcPhase { } } -pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1; +// Version 2 required peer Prepare to block new tier-reference creators and +// drain their in-flight operation leases. Version 3 additionally binds Abort +// to the canonical Prepare intent so a missing-record Abort can persist an +// identity-bound tombstone and linearize against a delayed Prepare. Version 4 +// signs a typed failure classification while retaining the exact v3 proof +// bytes for rolling compatibility. +pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3; +pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4; +pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024; +pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096; pub fn canonical_tier_mutation_rpc_body( version: u32, @@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> { pub state: i32, pub applied: bool, pub error_info: Option<&'a str>, + pub failure_class: i32, } pub fn canonical_tier_mutation_rpc_response_body( input: TierMutationRpcResponseProofInput<'_>, ) -> Result, std::num::TryFromIntError> { - const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0"; + const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0"; + const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0"; let phase = input.phase.as_wire_str().as_bytes(); let mutation_id = input.mutation_id.as_bytes(); let error_info = input.error_info.map(str::as_bytes); let error_info_len = error_info.map_or(0, <[u8]>::len); + let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION; + let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN }; let mut body = Vec::with_capacity( - DOMAIN.len() + domain.len() + 4 + 8 + phase.len() @@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body( + 1 + 1 + 8 - + error_info_len, + + error_info_len + + if is_v4 { 4 } else { 0 }, ); - body.extend_from_slice(DOMAIN); + body.extend_from_slice(domain); body.extend_from_slice(&input.version.to_be_bytes()); body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes()); body.extend_from_slice(phase); @@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body( if let Some(error_info) = error_info { body.extend_from_slice(error_info); } + if is_v4 { + body.extend_from_slice(&input.failure_class.to_be_bytes()); + } Ok(body) } @@ -2137,10 +2155,10 @@ mod heal_control_tests { #[cfg(test)] mod tier_mutation_rpc_tests { use super::{ - TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput, - canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body, + TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, + TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body, }; - use crate::proto_gen::node_service::TierMutationPeerState; + use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState}; use uuid::uuid; #[test] @@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests { ) .expect("small mutation body should encode"); let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec(); - golden.extend_from_slice(&1_u32.to_be_bytes()); + golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes()); golden.extend_from_slice(&7_u64.to_be_bytes()); golden.extend_from_slice(b"prepare"); golden.extend_from_slice(mutation_id.as_bytes()); @@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests { assert_ne!( baseline, - canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload) - .expect("small mutation body should encode") + canonical_tier_mutation_rpc_body( + TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, + TierMutationRpcPhase::Prepare, + mutation_id, + payload, + ) + .expect("small mutation body should encode") ); assert_ne!( baseline, @@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests { } #[test] - fn canonical_tier_mutation_response_binds_request_state_and_error() { + fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() { let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678"); let payload = b"canonical-intent-record"; - let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput { - version: TIER_MUTATION_RPC_PROTOCOL_VERSION, + let request = canonical_tier_mutation_rpc_body( + TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, + TierMutationRpcPhase::Prepare, + mutation_id, + payload, + ) + .expect("v3 request should encode"); + let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec(); + request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes()); + request_golden.extend_from_slice(&7_u64.to_be_bytes()); + request_golden.extend_from_slice(b"prepare"); + request_golden.extend_from_slice(mutation_id.as_bytes()); + request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes()); + request_golden.extend_from_slice(payload); + assert_eq!(request, request_golden); + + let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput { + version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Prepare, mutation_id, canonical_payload: payload, @@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests { state: TierMutationPeerState::Prepared as i32, applied: true, error_info: None, + // v3 must ignore the field so its authenticated bytes stay exact. + failure_class: TierMutationFailureClass::PreDispatchRejected as i32, }) - .expect("small mutation response should encode"); + .expect("v3 response should encode"); + let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec(); + response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes()); + response_golden.extend_from_slice(&7_u64.to_be_bytes()); + response_golden.extend_from_slice(b"prepare"); + response_golden.extend_from_slice(mutation_id.as_bytes()); + response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes()); + response_golden.extend_from_slice(payload); + response_golden.push(1); + response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes()); + response_golden.push(1); + response_golden.push(0); + response_golden.extend_from_slice(&0_u64.to_be_bytes()); + assert_eq!(response, response_golden); + } + + #[test] + fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() { + let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678"); + let payload = b"canonical-intent-record"; + let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput { + version: TIER_MUTATION_RPC_PROTOCOL_VERSION, + phase: TierMutationRpcPhase::Prepare, + mutation_id, + canonical_payload: payload, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, + }) + .expect("small v4 mutation response should encode"); let cases = [ TierMutationRpcResponseProofInput { - version: 2, + version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Prepare, mutation_id, canonical_payload: payload, - success: true, - state: TierMutationPeerState::Prepared as i32, - applied: true, - error_info: None, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }, TierMutationRpcResponseProofInput { version: TIER_MUTATION_RPC_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Commit, mutation_id, canonical_payload: payload, - success: true, - state: TierMutationPeerState::Prepared as i32, - applied: true, - error_info: None, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }, TierMutationRpcResponseProofInput { version: TIER_MUTATION_RPC_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Prepare, mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"), canonical_payload: payload, - success: true, - state: TierMutationPeerState::Prepared as i32, - applied: true, - error_info: None, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }, TierMutationRpcResponseProofInput { version: TIER_MUTATION_RPC_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Prepare, mutation_id, canonical_payload: b"tampered-intent-record", + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, + }, + TierMutationRpcResponseProofInput { + version: TIER_MUTATION_RPC_PROTOCOL_VERSION, + phase: TierMutationRpcPhase::Prepare, + mutation_id, + canonical_payload: payload, success: true, - state: TierMutationPeerState::Prepared as i32, - applied: true, - error_info: None, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }, TierMutationRpcResponseProofInput { version: TIER_MUTATION_RPC_PROTOCOL_VERSION, @@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests { mutation_id, canonical_payload: payload, success: false, - state: TierMutationPeerState::Prepared as i32, - applied: true, - error_info: None, - }, - TierMutationRpcResponseProofInput { - version: TIER_MUTATION_RPC_PROTOCOL_VERSION, - phase: TierMutationRpcPhase::Prepare, - mutation_id, - canonical_payload: payload, - success: true, state: TierMutationPeerState::Committed as i32, - applied: true, - error_info: None, - }, - TierMutationRpcResponseProofInput { - version: TIER_MUTATION_RPC_PROTOCOL_VERSION, - phase: TierMutationRpcPhase::Prepare, - mutation_id, - canonical_payload: payload, - success: true, - state: TierMutationPeerState::Prepared as i32, applied: false, - error_info: None, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }, TierMutationRpcResponseProofInput { version: TIER_MUTATION_RPC_PROTOCOL_VERSION, phase: TierMutationRpcPhase::Prepare, mutation_id, canonical_payload: payload, - success: true, - state: TierMutationPeerState::Prepared as i32, + success: false, + state: TierMutationPeerState::Unspecified as i32, applied: true, - error_info: Some("error"), + error_info: Some("failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, + }, + TierMutationRpcResponseProofInput { + version: TIER_MUTATION_RPC_PROTOCOL_VERSION, + phase: TierMutationRpcPhase::Prepare, + mutation_id, + canonical_payload: payload, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("other failure"), + failure_class: TierMutationFailureClass::Ambiguous as i32, + }, + TierMutationRpcResponseProofInput { + version: TIER_MUTATION_RPC_PROTOCOL_VERSION, + phase: TierMutationRpcPhase::Prepare, + mutation_id, + canonical_payload: payload, + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("failure"), + failure_class: TierMutationFailureClass::PreDispatchRejected as i32, }, ]; for case in cases { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 7347e17ca..d1e1b0ebc 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -1068,12 +1068,19 @@ enum TierMutationPeerState { TIER_MUTATION_PEER_STATE_ABORTED = 3; } +enum TierMutationFailureClass { + TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0; + TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1; + TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2; +} + message TierMutationControlResponse { bool success = 1; TierMutationPeerState state = 2; bool applied = 3; optional string error_info = 4; bytes response_proof = 5; + TierMutationFailureClass failure_class = 6; } message GetLiveEventsRequest { diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 12d928513..71e2a9a62 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -46,7 +46,7 @@ use storage_api::lifecycle::{ STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent, enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found, - is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries, + is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records, register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence, }; @@ -562,6 +562,29 @@ where } } +// Deep transition futures can overflow libtest's default stack before their +// first assertion, so the serial ILM cases use one dedicated test thread. +fn run_large_stack_async_test(thread_name: &'static str, test_fn: F) +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + 'static, +{ + let handle = std::thread::Builder::new() + .name(thread_name.to_string()) + .stack_size(32 * 1024 * 1024) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("large-stack scanner test runtime should build"); + runtime.block_on(test_fn()); + }) + .expect("large-stack scanner test thread should spawn"); + if let Err(payload) = handle.join() { + std::panic::resume_unwind(payload); + } +} + mod serial_tests { use super::*; @@ -737,10 +760,17 @@ mod serial_tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[test] #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] - async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() { + fn rejected_transition_candidate_is_recovered_from_persisted_transaction() { + run_large_stack_async_test( + "scanner-rejected-transition-transaction", + rejected_transition_candidate_is_recovered_from_persisted_transaction_case, + ); + } + + async fn rejected_transition_candidate_is_recovered_from_persisted_transaction_case() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); @@ -790,21 +820,20 @@ mod serial_tests { "no cleanup path may delete the candidate while remove failures are enabled" ); - let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let retained = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("tier delete journal recovery should scan the persisted candidate"); - assert_eq!(retained.scanned, 1); - assert_eq!(retained.deleted, 0); - assert_eq!(retained.failed, 1); + .expect("transition transaction recovery should scan the persisted candidate"); + assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1)); assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate"); backend.set_remove_failure(false); - let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("tier delete journal recovery should delete the retained candidate"); - assert_eq!(recovered.scanned, 1); - assert_eq!(recovered.deleted, 1); - assert_eq!(recovered.failed, 0); + .expect("transition transaction recovery should delete the retained candidate"); + assert_eq!( + (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed), + (1, 1, 0, 0) + ); let removed_versions = backend.remove_versions().await; assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete"); assert!( @@ -813,10 +842,10 @@ mod serial_tests { ); assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate"); - let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let empty = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("a removed tier delete journal entry should no longer be listed"); - assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry"); + .expect("a removed transition transaction should no longer be listed"); + assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted transaction"); assert_eq!( read_object_fully(&ecstore, bucket_name.as_str(), object_name).await, payload, @@ -824,10 +853,17 @@ mod serial_tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[test] #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] - async fn cancelled_before_cleanup_store_resolution_persists_journal() { + fn cancelled_before_cleanup_store_resolution_persists_transaction() { + run_large_stack_async_test( + "scanner-cancelled-transition-transaction", + cancelled_before_cleanup_store_resolution_persists_transaction_case, + ); + } + + async fn cancelled_before_cleanup_store_resolution_persists_transaction_case() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&tier_name).await; @@ -876,9 +912,9 @@ mod serial_tests { let retained = tokio::time::timeout(Duration::from_secs(30), async { loop { - let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("the cancelled transition journal should be readable"); + .expect("the cancelled transition transaction should be readable"); if recovery.scanned > 0 { break recovery; } @@ -887,29 +923,36 @@ mod serial_tests { }) .await .expect("Drop should persist the rejected candidate through the saved instance context"); - assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1)); + assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1)); tokio::time::timeout(Duration::from_secs(5), async { while backend.exact_remove_count() < 2 { tokio::task::yield_now().await; } }) .await - .expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint"); + .expect("Drop cleanup and failed transaction recovery must both preserve the exact version constraint"); let failed_exact_attempts = backend.exact_remove_count(); + assert_eq!( + failed_exact_attempts, 2, + "the cancelled task and the first failed recovery must each preserve the exact delete constraint" + ); assert_eq!(backend.object_count().await, 1); assert!(backend.remove_versions().await.is_empty()); backend.set_remove_failure(false); - let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None) .await .expect("recovery should delete the candidate retained by the cancelled transition"); - assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0)); + assert_eq!( + (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed), + (1, 1, 0, 0) + ); assert_eq!(backend.remove_versions().await, backend.put_versions().await); assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1); assert_eq!(backend.object_count().await, 0); - let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let empty = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("successful recovery should remove the cancellation journal"); + .expect("successful recovery should remove the cancellation transaction"); assert_eq!(empty.scanned, 0); assert_eq!( read_object_fully(&ecstore, bucket_name.as_str(), object_name).await, @@ -918,10 +961,14 @@ mod serial_tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[test] #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] - async fn rejected_transition_cleanup_durability_matrix() { + fn rejected_transition_cleanup_durability_matrix() { + run_large_stack_async_test("scanner-transition-cleanup-matrix", rejected_transition_cleanup_durability_matrix_case); + } + + async fn rejected_transition_cleanup_durability_matrix_case() { #[derive(Clone, Copy)] enum CleanupCase { Persisted, @@ -1010,44 +1057,67 @@ mod serial_tests { CleanupCase::Persisted | CleanupCase::DeleteFallback => { assert_eq!(backend.remove_versions().await, backend.put_versions().await); assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate"); - let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("successful cleanup must not retain a journal entry"); - assert_eq!(recovery.scanned, 0); + .expect("successful cleanup must leave no failed transition transaction"); + assert_eq!(recovery.failed, 0); + assert_eq!(recovery.retained, 0); + assert_eq!(recovery.recovered, recovery.scanned); + let empty = recover_transition_transaction_records(ecstore.clone(), 100, None) + .await + .expect("successful reconciliation must remove every transition transaction"); + assert_eq!(empty.scanned, 0); } CleanupCase::RetryPersisted => { assert!( !err.to_string().contains("journal retry error"), - "a successful journal retry must preserve the original version-constraint error" + "the transaction path must not surface the removed journal fallback error" ); assert_eq!(backend.object_count().await, 1); - let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let retained = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("the retried journal should be recoverable"); - assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1)); + .expect("the retained transaction should be recoverable"); + assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1)); backend.set_remove_failure(false); - let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("recovery should delete the exact retried candidate"); - assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0)); + .expect("recovery should delete the exact transaction candidate"); + assert_eq!( + (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed), + (1, 1, 0, 0) + ); assert_eq!(backend.remove_versions().await, backend.put_versions().await); assert_eq!(backend.object_count().await, 0); - let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let empty = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("successful recovery must remove the retried journal"); + .expect("successful recovery must remove the retained transaction"); assert_eq!(empty.scanned, 0); } CleanupCase::FullyFailed => { let message = err.to_string(); - assert!(message.contains("initial journal error"), "{message}"); assert!(message.contains("cleanup error"), "{message}"); - assert!(message.contains("journal retry error"), "{message}"); assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible"); assert!(backend.remove_versions().await.is_empty()); - let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None) + let retained = recover_transition_transaction_records(ecstore.clone(), 100, None) .await - .expect("failed journal writes must not create partial recovery entries"); - assert_eq!(recovery.scanned, 0); + .expect("the pre-upload transaction must retain ownership after cleanup failure"); + assert_eq!(retained.scanned, 1, "the failed cleanup must keep one durable transaction owner"); + assert_eq!(retained.recovered, 0); + assert_eq!(retained.retained + retained.failed, 1); + backend.set_remove_failure(false); + let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None) + .await + .expect("recovery should delete the candidate after the backend becomes available"); + assert_eq!( + (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed), + (1, 1, 0, 0) + ); + assert_eq!(backend.remove_versions().await, backend.put_versions().await); + assert_eq!(backend.object_count().await, 0); + let empty = recover_transition_transaction_records(ecstore.clone(), 100, None) + .await + .expect("successful recovery must remove the failed cleanup transaction"); + assert_eq!(empty.scanned, 0); } } assert_eq!( @@ -1062,20 +1132,7 @@ mod serial_tests { #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] fn test_transition_and_restore_flows() { - std::thread::Builder::new() - .name("scanner-transition-restore-flows".to_string()) - .stack_size(32 * 1024 * 1024) - .spawn(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("transition and restore test runtime should build"); - - runtime.block_on(test_transition_and_restore_flows_inner()); - }) - .expect("transition and restore test thread should spawn") - .join() - .expect("transition and restore test thread should finish"); + run_large_stack_async_test("scanner-transition-restore-flows", test_transition_and_restore_flows_inner); } async fn test_transition_and_restore_flows_inner() { diff --git a/crates/scanner/tests/storage_api/mod.rs b/crates/scanner/tests/storage_api/mod.rs index aa066fcfb..61ace3902 100644 --- a/crates/scanner/tests/storage_api/mod.rs +++ b/crates/scanner/tests/storage_api/mod.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::transition_transaction::recover_transition_transaction_records; pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{ bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry}, @@ -50,7 +50,7 @@ pub(crate) mod lifecycle { TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent, enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found, - is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries, + is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records, register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence, }; } diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index dfaffb92f..7e4af6c47 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -39,6 +39,8 @@ for later deletion. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. - `not-initialized-error-code-v1` typed control-plane not-initialized wire code: control-plane RPC responses historically signaled an uninitialized peer only through the literal error_info string "errServerNotInitialized" (one drift site says "storage layer not initialized"). Responses now dual-carry a typed ControlPlaneErrorCode beside the legacy string, and clients prefer the code; the string stays populated and the client substring fallback (is_err_not_initialized, control_plane_failure) stays in place so mixed-version clusters keep classifying older peers' responses. Remove the substring fallback (and stop populating error_info for this case) after the minimum supported RustFS peer version always sends error_code. +- `backlog-2097-tier-mutation-v4-error-text` tier-mutation Prepare rejection classification: a v3 server rejects a v4 request before store/runtime dispatch with the FailedPrecondition status and an authenticated, byte-exact unsupported-version message. A v4 coordinator recognizes only that exact code/message/requested-version tuple as definitely not persisted and fails the mutation without sending that peer an incompatible Abort; Unimplemented, near-text, missing/unknown failure classes, timeouts, and every other transport outcome remain ambiguous and stay in identity-bound Abort fanout. There is no automatic v3 retry. New servers retain v3 request/proof decoding for older coordinators, while operators must pause tier edit/remove/clear during a mixed v3/v4 rollout. Remove the text classifier after the minimum supported RustFS peer version returns the signed v4 PreDispatchRejected failure class. +- `backlog-2097-tier-delete-journal-v6` tier-delete sole-owner recovery: v6 distinguishes transactions that may replace the xl.meta free-version owner and therefore require the all-pool live-source proof. v5-and-older readers reject and retain v6 records during rolling upgrades instead of performing an unsafe remote delete. Keep v1-v5 readers for upgrade recovery and keep the downgrade prohibition while any v6 record exists; retire the fence only after every supported rollback release understands and enforces v6 proof semantics. - `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor. ## Review Checklist diff --git a/docs/architecture/decommission-compatibility.md b/docs/architecture/decommission-compatibility.md index 1d6fa95ea..3c243ac05 100644 --- a/docs/architecture/decommission-compatibility.md +++ b/docs/architecture/decommission-compatibility.md @@ -165,16 +165,33 @@ and all listing/GET paths exclude them. ### Lifecycle And Consumers -Creation: any local delete that removes a version whose transition status is -`complete` appends the record via `MetaObject::delete_version` → -`init_free_version` (skipped only when `skip_tier_free_version` is set, as on -data-movement copies). The same deletes also persist a durable tier-journal -entry on every user-facing path: S3 single deletes (`execute_delete_object` → -`delete_object_with_tier_delete_journal`), S3 batch deletes, lifecycle expiry, -and lifecycle delete-all all prepare and commit a journal entry around the -delete. A journal entry is omitted when the removed version's transition state -decodes as `TransitionVersionState::Unknown`, or on internal journal-less -delete paths that never touch transitioned user objects. +Creation: a local delete that removes a version whose transition status is +`complete` normally appends the record via `MetaObject::delete_version` → +`init_free_version`. User-facing single and batch deletes always retain that +historical owner when they actually remove a transitioned source; they do not +create a tier journal, probe a fleet capability, or issue a peer mutation RPC. +`TransitionVersionState::Unknown` and incomplete destination identities remain +on the same conservative free-version path. Delete-marker creation on an Enabled +bucket remains unchanged and does not schedule remote deletion. + +Recursive prefix/delete-all cannot preserve per-object markers across its +physical directory purge, so it requires a v6 recoverable journal for every +transitioned visible source plus a durable dispatch manifest for the complete +operation. It fails closed before mutation on legacy metadata or any existing +hidden tier free-version under the prefix. Its internal streaming walk +discovers logical keys, then exact-loads every key from its authoritative set in +every pool, including free versions; the S3 listing merge is never treated as a +complete physical-owner inventory. Tier-operation leases remain held from that +preflight through journal prepare and physical deletion. Once physical deletion +starts, any error is mutation-ambiguous: authorized/dispatched journals remain +for recovery to commit owners only after all physical sets prove both the source +and exact free-version identity absent; uncertain owners are retained. +If a retry discovers a later transitioned source after the manifest reached +`DispatchAuthorized`, it replays only the manifest's immutable predecessor set, +completes that operation, and leaves the newcomer for a successor dispatch. +Operators may retry after the legacy free-version worker has durably completed +remote and local cleanup. Journal-less internal deletes and older nodes retain +their established marker behavior. Consumption while the record exists: the background recovery loop started by `init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`, @@ -209,16 +226,93 @@ only when its free-version identity matches; a conflicting ordinary version or different free record is an overwrite error. This makes retries idempotent and prevents a free record from replacing a user-visible version. +### Remote-Tuple Publication Fence + +Cross-pool capability v3 includes a commit-late publication contract for every +path that can copy an existing transition tuple to a new physical owner. This +capability version is independent of the tier-mutation RPC protocol version. +A mixed fleet whose minimum cross-pool capability is below v3 cannot authorize +journal-v6 remote deletion. + +Data movement captures a non-cloneable, process-local source capability before +copying, but it does not hold a namespace write lock or tier-operation lease +while reading a large body or uploading multipart parts. `NewMultipartUpload` +and `UploadPart` are staging only. Immediately before single-PUT rename, +Multipart Complete, or a pure-remote/free-version metadata quorum write, the +final consumer acquires the exact tier generation (when a remote tuple exists), +then fixed/source/target write domains in stable order. The fixed domain is used +only for a real remote-tuple decommission publisher; an ordinary local object +keeps the lighter source/target commit scope. + +While that owned scope is held, the publisher re-reads the exact source pool and +compares version, data directory, modification time, ETag, checksums, transition +tuple, transition-version state, and destination identity. A missing or changed +source, changed/revoked tier generation, bucket incarnation change, or lost lock +fails before target rename. The scope remains owned through rename quorum and +the existing rename-tail guard handoff. Consequently, recovery-first ordering +cannot delete the remote object and then have a stale restored-transitioned +rebalance recreate its tuple; publisher-first ordering makes recovery wait and +rescan the newly committed owner. + +Full cross-key S3 Copy is not an ownership-sharing operation: it materializes +local data and strips transition, destination, transaction, and free-version +keys. Same-key metadata/version-only updates preserve the existing protected +state. Admin heal keeps the legacy `nolock` request field for wire compatibility +but ignores it as lock authority; final heal writes enter the normal locked +path. Restore similarly ignores ambient `ObjectOptions.no_lock`, acquires its +own commit-late PUT/Complete lock, validates the restore operation id, and keeps +an exact tier generation lease through the local commit. + ### Reference-Audit Result After migration, user-facing GET/list/transition/replication/restore paths still exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal -continue to see it when they request free versions, so an unresolved remote -delete remains actionable on the target pool. The committed tier journal remains -an independent retry source where one exists; it is not used as a reason to drop -the xl.meta record. In particular, `Unknown` transition state records are -migrated unchanged rather than discarded: the lifecycle worker retains them if -remote identity validation cannot make a delete request. +continue to see a legacy/fallback record when they request free versions, so an +unresolved remote delete remains actionable on the target pool. Only an +authorized recursive prefix/delete-all v6 transaction may instead use a +per-source journal as the sole retry source; ordinary single/batch deletes never +take that path. A journal discovered +alongside an older or fallback free-version does not authorize dropping the +record. In particular, `Unknown` transition state records are migrated unchanged +rather than discarded: the lifecycle worker retains them if remote identity +validation cannot make a delete request. + +Tier edit/remove/clear reference proof uses the internal walk with +`include_free_versions = true`, in addition to persisted journal and transition +transaction checks. Protocol v3 peer Prepare blocks new reference creators and +drains existing tier-operation leases before this proof; protocol v4 preserves +that state machine and adds a signed failure classification. Abort carries the +canonical Prepare intent, so a peer can create an identity-bound `Aborted` +tombstone even when Abort overtakes Prepare. A delayed matching Prepare then +converges on `Aborted` instead of reinstalling the block; a conflicting intent +with the same mutation id fails closed. The tombstone remains durable until the +intent expiry plus the configured clock-skew allowance, including across reload +and coordinator-record cleanup. After +expiry, a missing-record replay of the original signed Prepare is rejected and +cannot recreate a peer-only runtime fence. Abort checks an existing same-identity +terminal record before consulting mutable current-config proof, and recovery +reconstructs the original Prepared revision for Abort fanout. + +A new server accepts both v3 and v4 requests and selects the matching canonical +response proof. During a mixed rollout, an older v3 server rejects a v4 request +with an authenticated, byte-exact unsupported-version status before dispatch; +the v4 coordinator treats only that exact rejection as definitely not installed, +fails the admin mutation, and does not send the peer an incompatible Abort. +There is deliberately no automatic v3 retry. `Unimplemented`, near-text, +timeouts, missing/unknown failure classes, and other ambiguous outcomes still +receive Abort and retain the coordinator retry record if Abort cannot be proven. +Operators must pause and drain tier edit/remove/clear operations before starting +the rolling upgrade, leave them disabled while any v3-only peer remains, and +resume only after every topology member advertises the v4-capable release. +Ordinary object I/O and free-version cleanup remain available; xl.meta is +unchanged by the rejected mutation. + +Sole-owner transactions use journal v6: v5-and-older readers reject and retain +those records, so an old recovery worker cannot bypass the all-pool proof. Older +nodes may continue to create fallback free-versions until the rollout is +homogeneous. A deployment must not downgrade every v6-aware recovery worker +while any v6 record remains; drain the journal first or keep at least one v6-aware +worker until cleanup converges. Each migrated record emits `state = "free_version_migrated"` with reason `tier_free_version_migrated`. A record consumed before migration emits diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index db65eb35d..cbfdaf2fd 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -1109,7 +1109,10 @@ fn build_heal_channel_request(hip: &HealInitParams) -> HealChannelRequest { heal_request.update_parity = Some(hip.hs.update_parity); heal_request.recursive = Some(recursive || root_cluster_target); heal_request.dry_run = Some(hip.hs.dry_run); - heal_request.no_lock = Some(hip.hs.no_lock); + // `nolock` is retained in the admin wire shape for compatibility, but it + // is not transferable lock authority. Admin heals must enter the storage + // layer through its normal object-lock path. + heal_request.no_lock = Some(false); heal_request.source = HealRequestSource::Admin; heal_request } @@ -2588,7 +2591,7 @@ mod tests { } #[test] - fn test_build_heal_channel_request_preserves_client_options() { + fn test_build_heal_channel_request_preserves_safe_client_options() { let hip = HealInitParams { bucket: "bucket-a".to_string(), obj_prefix: "prefix-a".to_string(), @@ -2623,7 +2626,7 @@ mod tests { assert_eq!(request.update_parity, Some(false)); assert_eq!(request.recursive, Some(true)); assert_eq!(request.dry_run, Some(true)); - assert_eq!(request.no_lock, Some(true)); + assert_eq!(request.no_lock, Some(false)); let envelope = rustfs_protos::heal_control::Envelope::start( request, @@ -2639,7 +2642,7 @@ mod tests { else { panic!("expected admin heal start command"); }; - assert_eq!(request.no_lock, Some(true)); + assert_eq!(request.no_lock, Some(false)); } #[test] diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 6e393b31b..53c6fc5f1 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -29,7 +29,7 @@ use super::storage_api::test::object_utils::to_s3s_etag; use super::storage_api::test::runtime::{MockWarmBackend, MockWarmOp, register_mock_tier as register_mock_tier_util}; use super::storage_api::test::{ ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo, - StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, + StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, install_all_v6_fleet_capability_proof, }; use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; use crate::app::bucket_usecase::DefaultBucketUsecase; @@ -79,6 +79,12 @@ async fn setup_test_env() -> (Vec, Arc) { return (paths.clone(), ecstore.clone()); } + // Production startup installs this proof through the notification-system + // fleet probe. This isolated single-node harness constructs ECStore + // directly, so model that completed all-v6 probe before any journal permit + // or background expiry worker can exist. + install_all_v6_fleet_capability_proof(); + let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4()); let temp_dir = PathBuf::from(&test_base_dir); if temp_dir.exists() { diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 8eb19d5d4..a1a682f6f 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1270,6 +1270,7 @@ pub(crate) mod test { pub(crate) mod data_usage { pub(crate) use super::super::data_usage::*; } + pub(crate) use crate::storage::storage_api::ecstore_bucket::install_all_v6_fleet_capability_proof; pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata}; pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 83ea962e7..71dd5c643 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -83,6 +83,9 @@ const TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE: i32 = 0; const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1; const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2; const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3; +const TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE: i32 = 0; +const TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE: i32 = 1; +const TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE: i32 = 2; fn signal_service_response(success: bool, error_info: Option) -> Response { Response::new(SignalServiceResponse { @@ -178,7 +181,10 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2; +// v3 additionally promises the v6 tier-delete dispatch-manifest policy. The +// existing periodic topology probe carries both capabilities so normal object +// operations do not add another peer RPC. +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 3; fn admit_heal_control_replay( replay_cache: &mut HashMap>, @@ -723,9 +729,28 @@ impl TierMutationControlRpcService { .map_err(|_| Status::invalid_argument("tier mutation request length cannot be represented"))?; verify_tonic_canonical_body_digest(request, &body) .map_err(|err| Status::permission_denied(format!("tier mutation authentication failed: {err}")))?; - let store = self - .resolve_object_store() - .ok_or_else(|| Status::failed_precondition("tier mutation object store is not initialized"))?; + if !tier_mutation_protocol_version_is_supported(version) { + return Err(Status::failed_precondition(format!( + "unsupported tier mutation peer protocol version: {version}" + ))); + } + let store = match self.resolve_object_store() { + Some(store) => store, + None if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION => { + return tier_mutation_control_response(TierMutationControlResponseInput { + version, + phase, + mutation_id, + canonical_payload, + success: false, + state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE, + applied: false, + error_info: Some("tier mutation object store is not initialized".to_string()), + failure_class: TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE, + }); + } + None => return Err(Status::failed_precondition("tier mutation object store is not initialized")), + }; match tier_mutation_peer::handle_tier_mutation_peer_request(store, version, phase, mutation_id, canonical_payload).await { Ok(outcome) => tier_mutation_control_response(TierMutationControlResponseInput { @@ -737,6 +762,7 @@ impl TierMutationControlRpcService { state: tier_mutation_peer_state_to_proto_wire(outcome.state), applied: outcome.applied, error_info: None, + failure_class: TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE, }), Err(err) => tier_mutation_control_response(TierMutationControlResponseInput { version, @@ -746,7 +772,12 @@ impl TierMutationControlRpcService { success: false, state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE, applied: false, - error_info: Some(err.to_string()), + error_info: Some(bounded_tier_mutation_error_info(err.to_string())), + failure_class: if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION { + TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE + } else { + TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE + }, }), } } @@ -811,15 +842,35 @@ fn parse_tier_mutation_id(mutation_id: &str) -> Result { Ok(parsed) } +fn tier_mutation_protocol_version_is_supported(version: u32) -> bool { + matches!( + version, + rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION + ) +} + +fn bounded_tier_mutation_error_info(mut error_info: String) -> String { + let limit = rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE; + if error_info.len() <= limit { + return error_info; + } + let mut boundary = limit; + while !error_info.is_char_boundary(boundary) { + boundary -= 1; + } + error_info.truncate(boundary); + error_info +} + fn validate_tier_mutation_payload_size(phase: rustfs_protos::TierMutationRpcPhase, payload_len: usize) -> Result<(), Status> { let limit = match phase { rustfs_protos::TierMutationRpcPhase::Prepare => rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE, rustfs_protos::TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE, rustfs_protos::TierMutationRpcPhase::Abort => { - if payload_len != 0 { - return Err(Status::invalid_argument("tier mutation abort payload must be empty")); + if payload_len == 0 { + return Err(Status::invalid_argument("tier mutation abort payload is empty")); } - return Ok(()); + rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE } _ => return Err(Status::invalid_argument("tier mutation rpc phase is unsupported")), }; @@ -838,6 +889,7 @@ struct TierMutationControlResponseInput<'a> { state: i32, applied: bool, error_info: Option, + failure_class: i32, } fn tier_mutation_control_response( @@ -853,6 +905,7 @@ fn tier_mutation_control_response( state: input.state, applied: input.applied, error_info: input.error_info.as_deref(), + failure_class: input.failure_class, }) .map_err(|_| Status::internal("tier mutation response length cannot be represented"))?; let response_proof = sign_tonic_rpc_response_proof(&canonical_response) @@ -863,6 +916,7 @@ fn tier_mutation_control_response( applied: input.applied, error_info: input.error_info, response_proof: response_proof.into(), + failure_class: input.failure_class, })) } @@ -2469,10 +2523,10 @@ impl Node for NodeService { #[allow(unused_imports)] mod tests { use super::{ - CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _, - NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, - SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, + CROSS_POOL_FENCE_SUPPORTED_VERSION, CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, + KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _, NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context, @@ -2514,9 +2568,9 @@ mod tests { RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, - StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest, - UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - WriteRequest, + StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, + TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, + VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -3286,6 +3340,7 @@ mod tests { #[tokio::test] async fn tier_mutation_control_requires_body_bound_auth_before_store_lookup() { + let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-auth-test-secret".to_string()); let service = make_tier_mutation_control_server_for_context(None); let mutation_id = uuid::Uuid::new_v4(); let unsigned = service @@ -3317,8 +3372,26 @@ mod tests { let unavailable = service .prepare_tier_mutation(signed) .await - .expect_err("authenticated request still requires initialized object store"); - assert_eq!(unavailable.code(), tonic::Code::FailedPrecondition); + .expect("authenticated v4 pre-dispatch rejection should be signed") + .into_inner(); + assert!(!unavailable.success); + assert_eq!(unavailable.failure_class, TierMutationFailureClass::PreDispatchRejected as i32); + assert_eq!(unavailable.error_info.as_deref(), Some("tier mutation object store is not initialized")); + let canonical = + rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput { + version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, + phase: rustfs_protos::TierMutationRpcPhase::Prepare, + mutation_id, + canonical_payload: b"intent", + success: false, + state: TierMutationPeerState::Unspecified as i32, + applied: false, + error_info: Some("tier mutation object store is not initialized"), + failure_class: TierMutationFailureClass::PreDispatchRejected as i32, + }) + .expect("small response should encode"); + crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &unavailable.response_proof) + .expect("v4 pre-dispatch rejection must authenticate its failure class"); } #[tokio::test] @@ -3337,12 +3410,13 @@ mod tests { } #[tokio::test] - async fn tier_mutation_control_rejects_old_protocol_version_before_store_lookup() { + async fn tier_mutation_control_rejects_unsupported_protocol_version_before_store_lookup() { let service = make_tier_mutation_control_server_for_context(None); let mutation_id = uuid::Uuid::new_v4(); let payload = Bytes::from_static(b"intent"); + let unsupported_version = rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION - 1; let mut request = Request::new(TierMutationPrepareRequest { - version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION - 1, + version: unsupported_version, mutation_id: mutation_id.to_string(), canonical_payload: payload, }); @@ -3361,6 +3435,38 @@ mod tests { .await .expect_err("old tier mutation protocol version must fail closed"); assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!( + error.message(), + format!("unsupported tier mutation peer protocol version: {}", unsupported_version) + ); + } + + #[tokio::test] + async fn tier_mutation_control_accepts_v3_before_store_lookup() { + let service = make_tier_mutation_control_server_for_context(None); + let mutation_id = uuid::Uuid::new_v4(); + let payload = Bytes::from_static(b"intent"); + let mut request = Request::new(TierMutationPrepareRequest { + version: rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, + mutation_id: mutation_id.to_string(), + canonical_payload: payload, + }); + let body = rustfs_protos::canonical_tier_mutation_rpc_body( + request.get_ref().version, + rustfs_protos::TierMutationRpcPhase::Prepare, + mutation_id, + &request.get_ref().canonical_payload, + ) + .expect("v3 request should encode"); + set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode"); + mark_v2_authenticated(&mut request); + + let error = service + .prepare_tier_mutation(request) + .await + .expect_err("accepted v3 request should reach the missing-store check"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!(error.message(), "tier mutation object store is not initialized"); } #[tokio::test] @@ -3379,6 +3485,32 @@ mod tests { assert_eq!(error.code(), tonic::Code::InvalidArgument); } + #[tokio::test] + async fn tier_mutation_control_rejects_invalid_abort_payload_before_auth_and_store_lookup() { + let service = make_tier_mutation_control_server_for_context(None); + let mutation_id = uuid::Uuid::new_v4(); + let empty = service + .abort_tier_mutation(Request::new(TierMutationAbortRequest { + version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, + mutation_id: mutation_id.to_string(), + canonical_payload: Bytes::new(), + })) + .await + .expect_err("empty abort must fail before digest construction"); + assert_eq!(empty.code(), tonic::Code::InvalidArgument); + + let oversized = Bytes::from(vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1]); + let error = service + .abort_tier_mutation(Request::new(TierMutationAbortRequest { + version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, + mutation_id: mutation_id.to_string(), + canonical_payload: oversized, + })) + .await + .expect_err("oversized abort must fail before digest construction"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + #[test] fn tier_mutation_peer_state_wire_constants_match_generated_proto() { assert_eq!( @@ -3390,6 +3522,19 @@ mod tests { assert_eq!(super::TIER_MUTATION_PEER_STATE_ABORTED_WIRE, TierMutationPeerState::Aborted as i32); } + #[test] + fn tier_mutation_error_info_bound_preserves_utf8_and_byte_limit() { + let ascii = "a".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1); + let bounded = super::bounded_tier_mutation_error_info(ascii); + assert_eq!(bounded.len(), rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE); + + let unicode = "界".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE); + let bounded = super::bounded_tier_mutation_error_info(unicode); + assert!(bounded.len() <= rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE); + assert!(bounded.is_char_boundary(bounded.len())); + assert!(bounded.chars().all(|character| character == '界')); + } + #[test] fn tier_mutation_control_response_proof_binds_request_and_result() { let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-response-proof-test-secret".to_string()); @@ -3404,6 +3549,7 @@ mod tests { state: TierMutationPeerState::Unspecified as i32, applied: false, error_info: Some("store failed".to_string()), + failure_class: TierMutationFailureClass::Ambiguous as i32, }) .expect("response proof should be signed") .into_inner(); @@ -3417,6 +3563,7 @@ mod tests { state: TierMutationPeerState::Unspecified as i32, applied: false, error_info: Some("store failed"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }) .expect("small mutation response should encode"); crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof) @@ -3432,6 +3579,7 @@ mod tests { state: TierMutationPeerState::Unspecified as i32, applied: false, error_info: Some("store failed"), + failure_class: TierMutationFailureClass::Ambiguous as i32, }) .expect("small mutation response should encode"); let error = crate::storage::storage_api::verify_tonic_rpc_response_proof(&tampered, &response.response_proof) @@ -3599,7 +3747,7 @@ mod tests { } #[tokio::test] - async fn cross_pool_fence_probe_authenticates_supported_v2_state() { + async fn cross_pool_fence_probe_authenticates_supported_v3_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); assert!( @@ -3664,7 +3812,7 @@ mod tests { assert!(response.success); assert_eq!(response.error_info, None); - assert_eq!(&response.result[..4], &2_u32.to_be_bytes()); + assert_eq!(&response.result[..4], &CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes()); let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..]) .expect("capability identity should decode"); assert_eq!(topology_member, "node-a:9000"); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index f82adf0ce..258a1737e 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -404,6 +404,8 @@ pub(crate) mod ecstore_admin { } pub(crate) mod ecstore_bucket { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof; pub(crate) use rustfs_ecstore::api::bucket::{ bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication, tagging, target, utils, diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index bfed7fc0e..e41af7bfd 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -21,12 +21,12 @@ 2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs 3|crates/ecstore/src/cluster/rpc/http_auth.rs 2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs -11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs +10|crates/ecstore/src/cluster/rpc/peer_rest_client.rs 10|crates/ecstore/src/cluster/rpc/peer_s3_client.rs 10|crates/ecstore/src/cluster/rpc/remote_disk.rs 6|crates/ecstore/src/config/com.rs 14|crates/ecstore/src/config/storageclass.rs -183|crates/ecstore/src/core/pools.rs +182|crates/ecstore/src/core/pools.rs 8|crates/ecstore/src/data_movement/mod.rs 2|crates/ecstore/src/data_usage/local_snapshot.rs 12|crates/ecstore/src/data_usage/mod.rs @@ -70,5 +70,5 @@ 12|crates/ecstore/src/store/init.rs 2|crates/ecstore/src/store/init_format.rs 3|crates/ecstore/src/store/multipart.rs -8|crates/ecstore/src/store/object.rs +7|crates/ecstore/src/store/object.rs 5|crates/ecstore/src/store/rebalance/support.rs