diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs index a8b6262fb..d596ef35f 100644 --- a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -182,6 +182,16 @@ pub(crate) enum DurableIlmRecordCheckpoint { identity_sha256: String, state: tier_delete_journal::TierDeleteDispatchManifestState, }, + TierDeleteDispatchParent { + content_sha256: String, + identity_sha256: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + active_chunk_identity_sha256: Option, + completed: bool, + }, TransitionTransaction { content_sha256: String, identity_sha256: String, @@ -220,6 +230,7 @@ impl DurableIlmRecordCheckpoint { match self { Self::TierDeleteJournal { content_sha256, .. } | Self::TierDeleteDispatchManifest { content_sha256, .. } + | Self::TierDeleteDispatchParent { content_sha256, .. } | Self::TransitionTransaction { content_sha256, .. } | Self::ManualTransitionJob { content_sha256, .. } | Self::ManualTransitionScope { content_sha256, .. } @@ -341,6 +352,52 @@ impl DurableIlmRecordCheckpoint { (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed) ) } + ( + Self::TierDeleteDispatchParent { + identity_sha256: previous_identity, + revision: previous_revision, + next_chunk_sequence: previous_sequence, + completed_journal_count: previous_completed_journals, + active_chunk_identity_sha256: previous_active, + completed: previous_completed, + .. + }, + Self::TierDeleteDispatchParent { + identity_sha256: next_identity, + revision: next_revision, + next_chunk_sequence: next_sequence, + completed_journal_count: next_completed_journals, + active_chunk_identity_sha256: next_active, + completed: next_completed, + .. + }, + ) => { + let Some((sequence_delta, completed_journal_delta)) = tier_delete_dispatch_parent_progress_delta( + *previous_sequence, + *previous_completed_journals, + *next_sequence, + *next_completed_journals, + ) else { + return Err(Error::other("durable ILM record generation is not a monotonic successor")); + }; + let same_position_transition = sequence_delta == 0 + && completed_journal_delta == 0 + && matches!( + (previous_active.as_ref(), next_active.as_ref(), previous_completed, next_completed), + (None, Some(_), false, false) | (Some(_), None, false, false) | (None, None, false, true) + ); + let progress_transition = sequence_delta > 0 + && completed_journal_delta > 0 + && !matches!( + (previous_active.as_ref(), next_active.as_ref()), + (Some(previous), Some(next)) if previous == next + ); + previous_identity == next_identity + && !previous_completed + && next_revision > previous_revision + && (!next_completed || next_active.is_none()) + && (same_position_transition || progress_transition) + } ( Self::TransitionTransaction { identity_sha256: previous_identity, @@ -475,11 +532,58 @@ impl DurableIlmRecordCheckpoint { .. }, ) => previous_identity == terminal_identity, + ( + Self::TierDeleteDispatchParent { + identity_sha256: previous_identity, + revision: previous_revision, + next_chunk_sequence: previous_sequence, + completed_journal_count: previous_completed_journals, + active_chunk_identity_sha256: previous_active, + completed: false, + .. + }, + Self::TierDeleteDispatchParent { + identity_sha256: terminal_identity, + revision: terminal_revision, + next_chunk_sequence: terminal_sequence, + completed_journal_count: terminal_completed_journals, + active_chunk_identity_sha256: None, + completed: true, + .. + }, + ) => { + previous_identity == terminal_identity + && terminal_revision > previous_revision + && tier_delete_dispatch_parent_progress_delta( + *previous_sequence, + *previous_completed_journals, + *terminal_sequence, + *terminal_completed_journals, + ) + .is_some_and(|(sequence_delta, completed_journal_delta)| { + if sequence_delta == 0 && completed_journal_delta == 0 { + previous_active.is_none() + } else { + sequence_delta > 0 && completed_journal_delta > 0 + } + }) + } _ => false, } } } +fn tier_delete_dispatch_parent_progress_delta( + previous_sequence: u64, + previous_completed_journals: u64, + next_sequence: u64, + next_completed_journals: u64, +) -> Option<(u64, u64)> { + let sequence_delta = next_sequence.checked_sub(previous_sequence)?; + let completed_journal_delta = next_completed_journals.checked_sub(previous_completed_journals)?; + (sequence_delta <= completed_journal_delta).then_some((sequence_delta, completed_journal_delta)) +} + fn transition_state_distance( from: transition_transaction::TransitionTransactionState, to: transition_transaction::TransitionTransactionState, @@ -913,17 +1017,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { - let (operation_id, identity_sha256, state) = - tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?; - ( - "operation_id", - hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), - DurableIlmRecordCheckpoint::TierDeleteDispatchManifest { - content_sha256, + match tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)? { + tier_delete_journal::TierDeleteDispatchDurableRecord::Manifest { + operation_id, identity_sha256, state, - }, - ) + } => ( + "operation_id", + hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), + DurableIlmRecordCheckpoint::TierDeleteDispatchManifest { + content_sha256, + identity_sha256, + state, + }, + ), + tier_delete_journal::TierDeleteDispatchDurableRecord::Parent { + operation_id, + identity_sha256, + revision, + next_chunk_sequence, + completed_journal_count, + active_chunk_identity_sha256, + completed, + } => ( + "operation_id", + hex_sha256(operation_id.as_bytes(), ToOwned::to_owned), + DurableIlmRecordCheckpoint::TierDeleteDispatchParent { + content_sha256, + identity_sha256, + revision, + next_chunk_sequence, + completed_journal_count, + active_chunk_identity_sha256, + completed, + }, + ), + } } DurableIlmRecordKind::TransitionTransaction => { let transaction = transition_transaction::decode_transition_transaction_record(path, data) @@ -1143,6 +1272,85 @@ mod tests { assert!(aborted.validate_successor(&preparing).is_err()); } + #[test] + fn tier_delete_dispatch_parent_checkpoint_is_monotonic_across_chunks() { + let identity = "a".repeat(64); + let checkpoint = |revision, sequence, completed_journals, active: Option<&str>, completed| { + DurableIlmRecordCheckpoint::TierDeleteDispatchParent { + content_sha256: format!("{revision:064x}"), + identity_sha256: identity.clone(), + revision, + next_chunk_sequence: sequence, + completed_journal_count: completed_journals, + active_chunk_identity_sha256: active.map(ToOwned::to_owned), + completed, + } + }; + let idle = checkpoint(0, 0, 0, None, false); + let first_child = "b".repeat(64); + let second_child = "c".repeat(64); + let bound = checkpoint(1, 0, 0, Some(&first_child), false); + let advanced = checkpoint(2, 1, 2, None, false); + let next_bound = checkpoint(3, 1, 2, Some(&second_child), false); + let completed = checkpoint(4, 2, 3, None, true); + let terminal_after_more_chunks = checkpoint(6, 4, 7, None, true); + + idle.validate_successor(&bound).expect("an idle parent may bind one child"); + bound + .validate_successor(&advanced) + .expect("a completed child may advance the parent sequence"); + advanced + .validate_successor(&next_bound) + .expect("the next sequence may bind a new immutable child"); + next_bound + .validate_successor(&completed) + .expect("receipt progress may skip directly to a later terminal checkpoint"); + assert!( + bound.is_predecessor_of_terminal(&terminal_after_more_chunks), + "terminal cleanup may still recognize a valid multi-chunk predecessor" + ); + assert!( + advanced.is_predecessor_of_terminal(&terminal_after_more_chunks), + "terminal cleanup may still skip over later valid parent generations" + ); + assert!( + idle.validate_successor(&checkpoint(1, 0, 1, Some(&first_child), false)) + .is_err() + ); + assert!(bound.validate_successor(&checkpoint(2, 1, 0, None, false)).is_err()); + assert!( + bound.validate_successor(&checkpoint(2, 2, 1, None, false)).is_err(), + "sequence cannot advance beyond completed journal evidence" + ); + assert!( + advanced.validate_successor(&checkpoint(3, 1, 3, None, false)).is_err(), + "completed journal count cannot grow without a completed child sequence" + ); + assert!( + bound.validate_successor(&checkpoint(2, 0, 0, None, true)).is_err(), + "an active child cannot be marked completed without completion evidence" + ); + assert!( + bound + .validate_successor(&checkpoint(2, 0, 0, Some(&second_child), false)) + .is_err(), + "an active child cannot be replaced at the same parent sequence" + ); + assert!( + bound + .validate_successor(&checkpoint(2, 1, 1, Some(&first_child), false)) + .is_err(), + "sequence growth cannot retain the same active child identity" + ); + assert!( + !bound.is_predecessor_of_terminal(&checkpoint(2, 0, 0, None, true)), + "terminal cleanup must not treat an active child as completed without count evidence" + ); + assert!(completed.validate_successor(&checkpoint(5, 3, 4, None, true)).is_err()); + assert!(completed.validate_successor(&advanced).is_err()); + assert!(advanced.validate_successor(&idle).is_err()); + } + #[test] fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() { use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared}; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index d31295f5b..ed502e570 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -93,9 +93,127 @@ pub(crate) const TIER_DELETE_JOURNAL_LEGACY_PREFIX: &str = TIER_DELETE_JOURNAL_N pub(crate) const TIER_DELETE_JOURNAL_V6_PREFIX: &str = TIER_DELETE_JOURNAL_V6_NAMESPACE.prefix; pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_PREFIX: &str = "ilm/tier-delete-dispatch-manifests/"; const TIER_DELETE_DISPATCH_MANIFEST_VERSION: u8 = 1; +// RUSTFS_COMPAT_TODO(backlog-2133-tier-delete-chunk-parent): retain the v1 single-manifest reader and fail-closed root sentinel while supported rollback releases do not understand chunk parents. Remove after every supported rollback release validates the parent/child protocol and no retained v1 manifest remains. +const TIER_DELETE_DISPATCH_PARENT_VERSION: u8 = 1; +const TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE: &str = "chunked_parent"; +const TIER_DELETE_DISPATCH_CHUNK_PATH: &str = "chunks"; pub(crate) const MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE: usize = 32 * 1024 * 1024; const MAX_TIER_DELETE_DISPATCH_JOURNALS: usize = 200_000; +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST: AtomicUsize = AtomicUsize::new(0); + +pub(crate) fn tier_delete_dispatch_batch_limit() -> usize { + #[cfg(all(test, feature = "test-util"))] + { + let configured = TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST.load(Ordering::Acquire); + if configured != 0 { + return configured; + } + } + MAX_TIER_DELETE_DISPATCH_JOURNALS +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteDispatchBatchLimitGuard; + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteDispatchBatchLimitGuard { + pub(crate) fn install(limit: usize) -> Self { + assert!(limit > 0 && limit <= MAX_TIER_DELETE_DISPATCH_JOURNALS); + TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST + .compare_exchange(0, limit, Ordering::AcqRel, Ordering::Acquire) + .expect("tier delete dispatch batch limit test override must be exclusive"); + Self + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteDispatchBatchLimitGuard { + fn drop(&mut self) { + TIER_DELETE_DISPATCH_BATCH_LIMIT_FOR_TEST.store(0, Ordering::Release); + } +} + +#[cfg(all(test, feature = "test-util"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TierDeleteChunkTestStage { + ParentPersisted, + ChildManifestPersisted, + ParentBound, + DispatchAuthorized, + LocalReplayCompleted, + ChildCompleted, + ParentProgressed, + FinalLocalDeletionCompleted, + ParentCompleted, +} + +#[cfg(all(test, feature = "test-util"))] +struct TierDeleteChunkTestBarrierState { + stage: TierDeleteChunkTestStage, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TierDeleteChunkTestBarrier { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static TIER_DELETE_CHUNK_TEST_BARRIER: OnceLock>>> = OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl TierDeleteChunkTestBarrier { + pub(crate) fn install(stage: TierDeleteChunkTestStage) -> Self { + let state = Arc::new(TierDeleteChunkTestBarrierState { + stage, + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = TIER_DELETE_CHUNK_TEST_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete chunk test barrier should not poison"); + assert!(slot.is_none(), "tier delete chunk test barrier must not already be installed"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + self.state.arrived.notified().await; + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TierDeleteChunkTestBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + if let Some(slot) = TIER_DELETE_CHUNK_TEST_BARRIER.get() { + let mut slot = slot.lock().expect("tier delete chunk test barrier should not poison"); + if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) { + *slot = None; + } + } + } +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn tier_delete_chunk_test_pause(stage: TierDeleteChunkTestStage) { + let state = TIER_DELETE_CHUNK_TEST_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("tier delete chunk test barrier should not poison") + .as_ref() + .filter(|state| state.stage == stage) + .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + } +} + fn valid_tier_delete_topology_generation(generation: &str) -> bool { generation.len() == 64 && generation.bytes().all(|byte| byte.is_ascii_hexdigit()) } @@ -145,13 +263,14 @@ impl TierDeleteDispatchManifest { { return Err(Error::other("tier delete dispatch manifest is invalid")); } + let expected_journal_prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", self.operation_id.simple()); if !self.journal_names.windows(2).all(|pair| pair[0] < pair[1]) - || self.journal_names.iter().any(|name| { - let expected_prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", self.operation_id.simple()); - !name.starts_with(&expected_prefix) || !name.ends_with(".json") - }) + || self + .journal_names + .iter() + .any(|name| !name.starts_with(&expected_journal_prefix) || !name.ends_with(".json")) || tier_delete_dispatch_journal_set_digest(&self.journal_names) != self.journal_set_sha256 - || tier_delete_dispatch_manifest_object_name(&self.bucket, self.bucket_incarnation, &self.prefix) != object_name + || !tier_delete_dispatch_manifest_path_matches(self, object_name) { return Err(Error::other("tier delete dispatch manifest binding is invalid")); } @@ -159,6 +278,91 @@ impl TierDeleteDispatchManifest { } } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum TierDeleteDispatchParentState { + Active, + Completed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct TierDeleteDispatchChunkBinding { + sequence: u64, + operation_id: uuid::Uuid, + manifest_object: String, + journal_set_sha256: String, + journal_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct TierDeleteDispatchParent { + version: u8, + record_type: String, + operation_id: uuid::Uuid, + bucket: String, + bucket_incarnation: uuid::Uuid, + prefix: String, + topology_generation: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + active_chunk: Option, + state: TierDeleteDispatchParentState, +} + +impl TierDeleteDispatchParent { + fn validate(&self, object_name: &str) -> Result<()> { + let max_journal_count = u64::try_from(MAX_TIER_DELETE_DISPATCH_JOURNALS) + .map_err(|_| Error::other("tier delete dispatch journal limit is not representable"))?; + if self.version != TIER_DELETE_DISPATCH_PARENT_VERSION + || self.record_type != TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE + || self.operation_id.is_nil() + || self.bucket.is_empty() + || self.bucket_incarnation.is_nil() + || !valid_tier_delete_topology_generation(&self.topology_generation) + || tier_delete_dispatch_manifest_object_name(&self.bucket, self.bucket_incarnation, &self.prefix) != object_name + || (self.state == TierDeleteDispatchParentState::Completed && self.active_chunk.is_some()) + || self.next_chunk_sequence > self.revision + || self.completed_journal_count < self.next_chunk_sequence + || ((self.next_chunk_sequence == 0) != (self.completed_journal_count == 0)) + { + return Err(Error::other("tier delete dispatch parent binding is invalid")); + } + if let Some(chunk) = &self.active_chunk + && (self.state != TierDeleteDispatchParentState::Active + || chunk.sequence != self.next_chunk_sequence + || chunk.operation_id.is_nil() + || chunk.journal_count == 0 + || chunk.journal_count > max_journal_count + || chunk.manifest_object + != tier_delete_dispatch_chunk_manifest_object_name( + &self.bucket, + self.bucket_incarnation, + &self.prefix, + chunk.operation_id, + ) + || !rustfs_utils::crypto::is_sha256_checksum(&chunk.journal_set_sha256)) + { + return Err(Error::other("tier delete dispatch parent chunk binding is invalid")); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct TierDeleteDispatchParentAdvance { + parent_manifest_object: String, + parent_operation_id: uuid::Uuid, + chunk: TierDeleteDispatchChunkBinding, +} + +enum TierDeleteDispatchRecord { + Manifest(TierDeleteDispatchManifest), + Parent(TierDeleteDispatchParent), +} + fn tier_delete_dispatch_journal_set_digest(names: &[String]) -> String { let mut hasher = Sha256::new(); for name in names { @@ -168,19 +372,46 @@ fn tier_delete_dispatch_journal_set_digest(names: &[String]) -> String { rustfs_utils::crypto::hex(hasher.finalize().as_slice()) } -fn tier_delete_dispatch_manifest_object_name(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { +fn tier_delete_dispatch_manifest_digest(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { let mut hasher = Sha256::new(); hasher.update(bucket.as_bytes()); hasher.update([0]); hasher.update(incarnation.as_bytes()); hasher.update([0]); hasher.update(prefix.as_bytes()); + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +fn tier_delete_dispatch_manifest_object_name(bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> String { format!( "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{}.json", - rustfs_utils::crypto::hex(hasher.finalize().as_slice()) + tier_delete_dispatch_manifest_digest(bucket, incarnation, prefix) ) } +fn tier_delete_dispatch_chunk_manifest_object_name( + bucket: &str, + incarnation: uuid::Uuid, + prefix: &str, + operation_id: uuid::Uuid, +) -> String { + format!( + "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{TIER_DELETE_DISPATCH_CHUNK_PATH}/{}/{}.json", + tier_delete_dispatch_manifest_digest(bucket, incarnation, prefix), + operation_id.simple() + ) +} + +fn tier_delete_dispatch_manifest_path_matches(manifest: &TierDeleteDispatchManifest, object_name: &str) -> bool { + let parent_digest = tier_delete_dispatch_manifest_digest(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix); + object_name == format!("{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{parent_digest}.json") + || object_name + == format!( + "{TIER_DELETE_DISPATCH_MANIFEST_PREFIX}{TIER_DELETE_DISPATCH_CHUNK_PATH}/{parent_digest}/{}.json", + manifest.operation_id.simple() + ) +} + fn tier_delete_dispatch_operation_lock_name(manifest_object: &str) -> String { format!("{manifest_object}.operation-lock") } @@ -216,6 +447,15 @@ fn encode_tier_delete_dispatch_manifest(manifest: &TierDeleteDispatchManifest) - Ok(data) } +fn encode_tier_delete_dispatch_parent(parent: &TierDeleteDispatchParent) -> Result> { + let data = + serde_json::to_vec(parent).map_err(|err| Error::other_with_context("encode tier delete dispatch parent failed", err))?; + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch parent is too large")); + } + Ok(data) +} + fn decode_tier_delete_dispatch_manifest(data: &[u8], object_name: &str) -> Result { if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { return Err(Error::other("tier delete dispatch manifest is too large")); @@ -226,28 +466,94 @@ fn decode_tier_delete_dispatch_manifest(data: &[u8], object_name: &str) -> Resul Ok(manifest) } +fn decode_tier_delete_dispatch_record(data: &[u8], object_name: &str) -> Result { + if data.len() > MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE { + return Err(Error::other("tier delete dispatch record is too large")); + } + match decode_tier_delete_dispatch_manifest(data, object_name) { + Ok(manifest) => Ok(TierDeleteDispatchRecord::Manifest(manifest)), + Err(manifest_error) => { + let parent: TierDeleteDispatchParent = serde_json::from_slice(data).map_err(|parent_error| { + Error::other_with_context( + "decode tier delete dispatch record failed", + format!("manifest: {manifest_error}; parent: {parent_error}"), + ) + })?; + parent.validate(object_name)?; + Ok(TierDeleteDispatchRecord::Parent(parent)) + } + } +} + +pub(crate) enum TierDeleteDispatchDurableRecord { + Manifest { + operation_id: uuid::Uuid, + identity_sha256: String, + state: TierDeleteDispatchManifestState, + }, + Parent { + operation_id: uuid::Uuid, + identity_sha256: String, + revision: u64, + next_chunk_sequence: u64, + completed_journal_count: u64, + active_chunk_identity_sha256: Option, + completed: bool, + }, +} + pub(crate) fn validate_tier_delete_dispatch_manifest_record( object_name: &str, data: &[u8], -) -> Result<(uuid::Uuid, String, TierDeleteDispatchManifestState)> { - let manifest = decode_tier_delete_dispatch_manifest(data, object_name)?; - let identity = serde_json::to_vec(&( - manifest.version, - manifest.operation_id, - &manifest.bucket, - manifest.bucket_incarnation, - &manifest.prefix, - &manifest.journal_names, - &manifest.journal_set_sha256, - manifest.journal_count, - &manifest.topology_generation, - )) - .map_err(Error::other)?; - Ok(( - manifest.operation_id, - rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), - manifest.state, - )) +) -> Result { + match decode_tier_delete_dispatch_record(data, object_name)? { + TierDeleteDispatchRecord::Manifest(manifest) => { + let identity = serde_json::to_vec(&( + manifest.version, + manifest.operation_id, + &manifest.bucket, + manifest.bucket_incarnation, + &manifest.prefix, + &manifest.journal_names, + &manifest.journal_set_sha256, + manifest.journal_count, + &manifest.topology_generation, + )) + .map_err(Error::other)?; + Ok(TierDeleteDispatchDurableRecord::Manifest { + operation_id: manifest.operation_id, + identity_sha256: rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), + state: manifest.state, + }) + } + TierDeleteDispatchRecord::Parent(parent) => { + let identity = serde_json::to_vec(&( + parent.version, + &parent.record_type, + parent.operation_id, + &parent.bucket, + parent.bucket_incarnation, + &parent.prefix, + &parent.topology_generation, + )) + .map_err(Error::other)?; + let active_chunk_identity_sha256 = parent + .active_chunk + .as_ref() + .map(|chunk| serde_json::to_vec(chunk).map(|data| rustfs_utils::crypto::hex_sha256(&data, ToOwned::to_owned))) + .transpose() + .map_err(Error::other)?; + Ok(TierDeleteDispatchDurableRecord::Parent { + operation_id: parent.operation_id, + identity_sha256: rustfs_utils::crypto::hex_sha256(&identity, ToOwned::to_owned), + revision: parent.revision, + next_chunk_sequence: parent.next_chunk_sequence, + completed_journal_count: parent.completed_journal_count, + active_chunk_identity_sha256, + completed: parent.state == TierDeleteDispatchParentState::Completed, + }) + } + } } /// Return the fleet generation durably bound to a v6 manifest or journal. @@ -255,7 +561,12 @@ pub(crate) fn validate_tier_delete_dispatch_manifest_record( /// cleanup compatibility behavior. pub(crate) fn durable_ilm_v6_topology_generation(object_name: &str, data: &[u8]) -> Result> { if object_name.starts_with(TIER_DELETE_DISPATCH_MANIFEST_PREFIX) { - return decode_tier_delete_dispatch_manifest(data, object_name).map(|manifest| Some(manifest.topology_generation)); + return decode_tier_delete_dispatch_record(data, object_name).map(|record| { + Some(match record { + TierDeleteDispatchRecord::Manifest(manifest) => manifest.topology_generation, + TierDeleteDispatchRecord::Parent(parent) => parent.topology_generation, + }) + }); } if object_name.starts_with(TIER_DELETE_JOURNAL_V6_PREFIX) { let entry = decode_tier_delete_journal_entry(data)?; @@ -302,10 +613,12 @@ pub(crate) fn test_tier_delete_dispatch_manifest_record( } struct DispatchedJournalPermit { + manifest_object: String, manifest: TierDeleteDispatchManifest, authorized_etag: String, entries: Vec, fleet_proof: TierDeleteJournalFleetProofToken, + parent_advance: Option, } struct TierDeleteDispatchAuthorizationInner { @@ -423,31 +736,68 @@ pub(crate) struct PreparedTierDeleteDispatch { predecessor_replay_required: bool, } +pub(crate) enum TierDeleteChunkParentInspection { + NoParent, + LegacyManifest, + Ready(String), + Resume(Box), + RetryRequired, +} + pub(crate) struct ActiveTierDeleteDispatch { + manifest_object: String, manifest: TierDeleteDispatchManifest, authorized_etag: String, entries: Arc<[Jentry]>, authorization: TierDeleteDispatchAuthorization, predecessor_replay_required: bool, + parent_advance: Option, } impl PreparedTierDeleteDispatch { + pub(crate) fn entries(&self) -> Result<&[Jentry]> { + self.permit + .as_ref() + .map(|permit| permit.entries.as_slice()) + .ok_or_else(|| Error::other("tier delete dispatch permit was already consumed")) + } + + pub(crate) fn require_exact_predecessor_replay(&mut self) { + self.predecessor_replay_required = true; + } + pub(crate) fn consume(mut self, bucket: &str, incarnation: uuid::Uuid, prefix: &str) -> Result { let permit = self .permit .take() .ok_or_else(|| Error::other("tier delete dispatch permit was already consumed"))?; - let manifest_object = tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix); if permit.manifest.state != TierDeleteDispatchManifestState::DispatchAuthorized || permit.manifest.bucket != bucket || permit.manifest.bucket_incarnation != incarnation || permit.manifest.prefix != prefix - || permit.entries.iter().map(tier_delete_journal_object_name).collect::>() != permit.manifest.journal_names + || permit.manifest.validate(&permit.manifest_object).is_err() + || permit.entries.len() != permit.manifest.journal_names.len() + || !permit + .entries + .iter() + .zip(&permit.manifest.journal_names) + .all(|(entry, name)| tier_delete_journal_object_name(entry) == name.as_str()) || !tier_delete_journal_fleet_proof_matches(&permit.fleet_proof) || tier_delete_journal_topology_generation(&permit.fleet_proof) != permit.manifest.topology_generation + || permit.parent_advance.as_ref().is_some_and(|parent| { + parent.chunk.manifest_object != permit.manifest_object + || parent.chunk.operation_id != permit.manifest.operation_id + || parent.chunk.journal_set_sha256 != permit.manifest.journal_set_sha256 + || parent.chunk.journal_count != permit.manifest.journal_count + }) { return Err(Error::other("tier delete dispatch permit validation failed")); } + if permit.parent_advance.is_none() + && permit.manifest_object != tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix) + { + return Err(Error::other("unbound tier delete child dispatch cannot authorize local mutation")); + } let entries = Arc::<[Jentry]>::from(permit.entries); let journal_entry_indexes = permit .manifest @@ -458,7 +808,7 @@ impl PreparedTierDeleteDispatch { .map(|(index, name)| (name, index)) .collect(); let authorization = TierDeleteDispatchAuthorization(Arc::new(TierDeleteDispatchAuthorizationInner { - manifest_object, + manifest_object: permit.manifest_object.clone(), operation_id: permit.manifest.operation_id, bucket: permit.manifest.bucket.clone(), bucket_incarnation: permit.manifest.bucket_incarnation, @@ -471,11 +821,13 @@ impl PreparedTierDeleteDispatch { mutation_started: AtomicBool::new(false), })); Ok(ActiveTierDeleteDispatch { + manifest_object: permit.manifest_object, manifest: permit.manifest, authorized_etag: permit.authorized_etag, entries, authorization, predecessor_replay_required: self.predecessor_replay_required, + parent_advance: permit.parent_advance, }) } } @@ -492,6 +844,10 @@ impl ActiveTierDeleteDispatch { pub(crate) fn predecessor_replay_required(&self) -> bool { self.predecessor_replay_required } + + pub(crate) fn is_chunked(&self) -> bool { + self.parent_advance.is_some() + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -829,6 +1185,22 @@ async fn read_tier_delete_dispatch_manifest( } } +async fn read_tier_delete_dispatch_record( + api: Arc, + object_name: &str, +) -> Result> { + match config_boundary::read_config_with_metadata(api, object_name, &ObjectOptions::default()).await { + Ok((data, metadata)) => { + let etag = metadata + .etag + .ok_or_else(|| Error::other("tier delete dispatch record has no entity tag"))?; + Ok(Some((decode_tier_delete_dispatch_record(&data, object_name)?, etag))) + } + Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => Ok(None), + Err(err) => Err(err), + } +} + async fn read_tier_delete_journal_with_etag(api: Arc, name: &str) -> Result> { match config_boundary::read_config_with_metadata(api, name, &ObjectOptions::default()).await { Ok((data, metadata)) => { @@ -1638,6 +2010,15 @@ async fn record_tier_delete_dispatch_manifest_progress_fenced( record_durable_config_progress_fenced(api, name, &encode_tier_delete_dispatch_manifest(manifest)?, fences_current).await } +async fn record_tier_delete_dispatch_parent_progress_fenced( + api: Arc, + name: &str, + parent: &TierDeleteDispatchParent, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + record_durable_config_progress_fenced(api, name, &encode_tier_delete_dispatch_parent(parent)?, fences_current).await +} + async fn delete_durable_config_if_match( api: Arc, name: &str, @@ -1835,7 +2216,7 @@ fn bind_dispatch_entries( topology_generation: &str, ) -> Result> { let mut entries = entries; - entries.sort_by_key(|entry| tier_delete_journal_v6_object_name(entry, operation_id)); + entries.sort_by_cached_key(|entry| tier_delete_journal_v6_object_name(entry, operation_id)); let mut bound = Vec::with_capacity(entries.len()); let mut last_name: Option = None; for mut entry in entries { @@ -1887,20 +2268,24 @@ fn tier_delete_dispatch_desired_names(entries: &[Jentry], operation_id: uuid::Uu Ok(names) } -fn validate_bound_journal(manifest: &TierDeleteDispatchManifest, name: &str, entry: &Jentry) -> Result<()> { +fn validate_bound_journal( + manifest: &TierDeleteDispatchManifest, + manifest_object: &str, + name: &str, + entry: &Jentry, +) -> Result<()> { + // Callers pass a manifest already validated at its decode/create boundary. + // Keep this member check O(1); rehashing the whole manifest here would be + // quadratic across a maximum-sized batch. + let dispatch_matches = entry.dispatch.as_ref().is_some_and(|binding| { + binding.operation_id == manifest.operation_id + && binding.manifest_object == manifest_object + && binding.journal_set_sha256 == manifest.journal_set_sha256 + && binding.topology_generation == manifest.topology_generation + }); if entry.persisted_version != TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION || tier_delete_journal_object_name(entry) != name - || entry.dispatch.as_ref() - != Some(&TierDeleteDispatchBinding { - operation_id: manifest.operation_id, - manifest_object: tier_delete_dispatch_manifest_object_name( - &manifest.bucket, - manifest.bucket_incarnation, - &manifest.prefix, - ), - journal_set_sha256: manifest.journal_set_sha256.clone(), - topology_generation: manifest.topology_generation.clone(), - }) + || !dispatch_matches { return Err(Error::other("tier delete journal does not match its dispatch manifest")); } @@ -1909,6 +2294,7 @@ fn validate_bound_journal(manifest: &TierDeleteDispatchManifest, name: &str, ent async fn load_complete_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, allowed_states: &[TierDeleteJournalState], fences_current: &impl Fn() -> bool, @@ -1920,7 +2306,7 @@ async fn load_complete_dispatch_journal_set( let (entry, _) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("tier delete dispatch manifest references a missing journal"))?; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !allowed_states.contains(&entry.state) { return Err(Error::other("tier delete dispatch journal has an invalid state")); } @@ -1949,6 +2335,7 @@ async fn load_complete_dispatch_journal_set( async fn persist_prepared_dispatch_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, entry: &Jentry, fences_current: &impl Fn() -> bool, @@ -1956,7 +2343,7 @@ async fn persist_prepared_dispatch_journal( let name = tier_delete_journal_object_name(entry); for _ in 0..4 { if let Some((current, _)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? { - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, entry) { return Err(Error::other("tier delete journal key is occupied by another cleanup identity")); } @@ -1978,6 +2365,7 @@ async fn persist_prepared_dispatch_journal( async fn dispatch_prepared_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, expected: &Jentry, fences_current: &impl Fn() -> bool, @@ -1987,7 +2375,7 @@ async fn dispatch_prepared_journal( let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("prepared tier delete journal disappeared before dispatch"))?; - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, expected) { return Err(Error::other("prepared tier delete journal changed identity before dispatch")); } @@ -2023,6 +2411,7 @@ fn ensure_tier_delete_dispatch_member_scan_fence(fences_current: &impl Fn() -> b async fn validate_staged_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result<()> { @@ -2046,7 +2435,7 @@ async fn validate_staged_dispatch_journal_set( let Some((entry, _)) = observed else { return Ok(()); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); } @@ -2090,6 +2479,7 @@ async fn validate_staged_dispatch_journal_set( async fn delete_staged_dispatch_journal_set( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &F, ) -> Result<()> @@ -2099,7 +2489,7 @@ where // Validate the complete immutable set before deleting its first member so // a corrupt binding or impossible Committed state quarantines the whole // operation rather than producing a partial rollback. - validate_staged_dispatch_journal_set(api.clone(), manifest, fences_current).await?; + validate_staged_dispatch_journal_set(api.clone(), manifest_object, manifest, fences_current).await?; // The validation barrier above must drain before this stream is created. // Once deletion starts, stop admitting useful work after the first error @@ -2123,7 +2513,7 @@ where let Some((entry, etag)) = read_tier_delete_journal_with_etag(api.clone(), &name).await? else { return Ok(()); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if !matches!(entry.state, TierDeleteJournalState::Prepared | TierDeleteJournalState::Dispatched) { return Err(Error::other("an uncommitted tier delete dispatch contains a committed journal")); } @@ -2206,19 +2596,18 @@ where async fn seal_and_rollback_preparing_dispatch( api: Arc, + manifest_name: &str, expected: &TierDeleteDispatchManifest, fleet_proof: &TierDeleteJournalFleetProofToken, bucket_fence: &crate::object_api::NamespaceLockFence, operation_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result<()> { - let manifest_name = - tier_delete_dispatch_manifest_object_name(&expected.bucket, expected.bucket_incarnation, &expected.prefix); let fences_current = || dispatch_write_fences_current(bucket_fence, operation_guard, fleet_proof, &expected.topology_generation); if !fences_current() { return Err(Error::other("tier delete dispatch rollback fence changed")); } - let Some((mut current, etag)) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? else { + let Some((mut current, etag)) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await? else { return Ok(()); }; if current.operation_id != expected.operation_id @@ -2235,26 +2624,26 @@ async fn seal_and_rollback_preparing_dispatch( } save_config_if_match_fenced( api.clone(), - &manifest_name, + manifest_name, encode_tier_delete_dispatch_manifest(¤t)?, &etag, &fences_current, ) .await?; - let (mut sealed, sealed_etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + let (mut sealed, sealed_etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await? .ok_or_else(|| Error::other("sealed tier delete dispatch manifest disappeared"))?; if sealed.operation_id != expected.operation_id || sealed.state != TierDeleteDispatchManifestState::Aborting { return Err(Error::other("tier delete dispatch rollback seal changed")); } - delete_staged_dispatch_journal_set(api.clone(), &sealed, &fences_current).await?; + delete_staged_dispatch_journal_set(api.clone(), manifest_name, &sealed, &fences_current).await?; sealed.state = TierDeleteDispatchManifestState::Aborted; if !fences_current() { return Err(Error::other("tier delete dispatch rollback fence changed")); } save_config_if_match_fenced( api, - &manifest_name, + manifest_name, encode_tier_delete_dispatch_manifest(&sealed)?, &sealed_etag, &fences_current, @@ -2269,6 +2658,19 @@ async fn authorized_dispatch_permit( bucket_fence: &crate::object_api::NamespaceLockFence, operation_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result { + authorized_dispatch_permit_with_parent(api, manifest_name, fleet_proof, bucket_fence, operation_guard, None, None).await +} + +async fn authorized_dispatch_permit_with_parent( + api: Arc, + manifest_name: &str, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + parent_operation_guard: Option<&rustfs_lock::NamespaceLockGuard>, + parent_advance: Option, +) -> Result { + let is_chunked = parent_advance.is_some(); let (manifest, authorized_etag) = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await? .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared"))?; @@ -2279,11 +2681,14 @@ async fn authorized_dispatch_permit( return Err(Error::other("tier delete dispatch authorization is stale or unconfirmed")); } let entries = { - let fences_current = - || dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation); + let fences_current = || { + dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation) + && parent_operation_guard.is_none_or(|guard| !guard.is_lock_lost()) + }; record_tier_delete_dispatch_manifest_progress_fenced(api.clone(), manifest_name, &manifest, &fences_current).await?; load_complete_dispatch_journal_set( api, + manifest_name, &manifest, &[TierDeleteJournalState::Dispatched, TierDeleteJournalState::Committed], &fences_current, @@ -2292,15 +2697,444 @@ async fn authorized_dispatch_permit( }; Ok(PreparedTierDeleteDispatch { permit: Some(DispatchedJournalPermit { + manifest_object: manifest_name.to_string(), manifest, authorized_etag, entries, fleet_proof, + parent_advance, }), - predecessor_replay_required: false, + predecessor_replay_required: is_chunked, }) } +fn tier_delete_dispatch_child_matches_parent( + parent: &TierDeleteDispatchParent, + binding: &TierDeleteDispatchChunkBinding, + child: &TierDeleteDispatchManifest, +) -> bool { + // Every caller supplies a child decoded against `binding.manifest_object`, + // which already verifies the path, sorted member set, and full digest. + // Keep the parent comparison O(1) for a maximum-sized child. + child.operation_id == binding.operation_id + && child.bucket == parent.bucket + && child.bucket_incarnation == parent.bucket_incarnation + && child.prefix == parent.prefix + && child.topology_generation == parent.topology_generation + && child.journal_set_sha256 == binding.journal_set_sha256 + && child.journal_count == binding.journal_count +} + +fn tier_delete_dispatch_parent_fences_current( + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + parent: &TierDeleteDispatchParent, +) -> bool { + dispatch_write_fences_current(bucket_fence, operation_guard, fleet_proof, &parent.topology_generation) +} + +async fn cas_tier_delete_dispatch_parent( + api: Arc, + object_name: &str, + expected: &TierDeleteDispatchParent, + expected_etag: &str, + next: &TierDeleteDispatchParent, + fences_current: &impl Fn() -> bool, +) -> Result<(TierDeleteDispatchParent, String)> { + expected.validate(object_name)?; + next.validate(object_name)?; + if expected.operation_id != next.operation_id || !fences_current() { + return Err(Error::other("tier delete dispatch parent fence changed before progress")); + } + let write = save_config_if_match_fenced( + api.clone(), + object_name, + encode_tier_delete_dispatch_parent(next)?, + expected_etag, + fences_current, + ) + .await; + if write.as_ref().is_err_and(is_decommission_checkpoint_targets_incomplete) { + return Err(write.expect_err("decommission checkpoint result should remain an error")); + } + let observed = read_tier_delete_dispatch_record(api, object_name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch parent fence changed during progress")); + } + match observed { + Some((TierDeleteDispatchRecord::Parent(observed), etag)) if observed == *next => Ok((observed, etag)), + _ => match write { + Ok(()) | Err(Error::PreconditionFailed) => Err(Error::other("tier delete dispatch parent changed during progress")), + Err(err) => Err(err), + }, + } +} + +async fn tier_delete_dispatch_chunk_journal_namespace_empty(api: Arc, operation_id: uuid::Uuid) -> Result { + let prefix = format!("{TIER_DELETE_JOURNAL_V6_PREFIX}{}/", operation_id.simple()); + let list = api + .list_objects_v2(RUSTFS_META_BUCKET, &prefix, None, None, 1, false, None, false) + .await?; + Ok(list.objects.is_empty()) +} + +pub(crate) async fn inspect_tier_delete_chunk_parent( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let Some((record, _)) = read_tier_delete_dispatch_record(api.clone(), &parent_name).await? else { + return Ok(TierDeleteChunkParentInspection::NoParent); + }; + let TierDeleteDispatchRecord::Parent(observed_parent) = record else { + return Ok(TierDeleteChunkParentInspection::LegacyManifest); + }; + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if observed_parent.bucket != bucket + || observed_parent.bucket_incarnation != bucket_incarnation + || observed_parent.prefix != prefix + || tier_delete_journal_topology_generation(&fleet_proof) != observed_parent.topology_generation + { + return Err(Error::other("an incompatible tier delete chunk parent already owns this prefix")); + } + // Lock order: caller-held bucket lifecycle WRITE -> parent operation -> + // child operation (only while reconstructing an Authorized child permit). + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = match read_tier_delete_dispatch_record(api.clone(), &parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("tier delete chunk parent was replaced by a legacy manifest")); + } + None => return Ok(TierDeleteChunkParentInspection::RetryRequired), + }; + if parent != observed_parent + || !tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent) + { + return Err(Error::other("tier delete chunk parent changed during inspection")); + } + { + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &parent_name, &parent, &fences_current).await?; + } + if parent.state == TierDeleteDispatchParentState::Completed { + return Ok(TierDeleteChunkParentInspection::RetryRequired); + } + let Some(binding) = parent.active_chunk.clone() else { + return Ok(TierDeleteChunkParentInspection::Ready(parent.topology_generation.clone())); + }; + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + let child = read_tier_delete_dispatch_manifest(api.clone(), &binding.manifest_object).await?; + let Some((child, _)) = child else { + if !tier_delete_dispatch_chunk_journal_namespace_empty(api.clone(), binding.operation_id).await? { + return Err(Error::other("tier delete chunk parent references a missing child with retained journals")); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &parent_etag, &next, &fences_current).await?; + return Ok(TierDeleteChunkParentInspection::RetryRequired); + }; + if !tier_delete_dispatch_child_matches_parent(&parent, &binding, &child) { + return Err(Error::other("tier delete chunk parent child binding changed")); + } + match child.state { + TierDeleteDispatchManifestState::DispatchAuthorized => { + let child_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&binding.manifest_object)) + .await?; + let child_guard = child_lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let advance = TierDeleteDispatchParentAdvance { + parent_manifest_object: parent_name, + parent_operation_id: parent.operation_id, + chunk: binding, + }; + let child_manifest_object = advance.chunk.manifest_object.clone(); + let prepared = authorized_dispatch_permit_with_parent( + api, + &child_manifest_object, + fleet_proof, + bucket_fence, + &child_guard, + Some(&parent_guard), + Some(advance), + ) + .await?; + Ok(TierDeleteChunkParentInspection::Resume(Box::new(prepared))) + } + TierDeleteDispatchManifestState::Completed => { + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(binding.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &parent_etag, &next, &fences_current).await?; + Ok(TierDeleteChunkParentInspection::RetryRequired) + } + TierDeleteDispatchManifestState::Preparing + | TierDeleteDispatchManifestState::Aborting + | TierDeleteDispatchManifestState::Aborted => Ok(TierDeleteChunkParentInspection::RetryRequired), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn prepare_tier_delete_chunk_dispatch( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + create_parent_if_missing: bool, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + if entries.is_empty() + || entries.len() > tier_delete_dispatch_batch_limit() + || bucket_incarnation.is_nil() + || bucket_fence.is_lock_lost() + || !tier_delete_journal_fleet_proof_matches(&fleet_proof) + { + return Err(Error::other("tier delete chunk dispatch input is invalid or stale")); + } + let topology_generation = tier_delete_journal_topology_generation(&fleet_proof); + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + // Lock order: caller-held bucket lifecycle WRITE -> parent operation -> + // newly created child operation. Completion releases the child lock before + // reacquiring the parent lock, so no child -> parent nesting exists. + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = loop { + let observed = read_tier_delete_dispatch_record(api.clone(), &parent_name).await?; + match observed { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => break (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("a legacy tier delete dispatch already owns this prefix")); + } + None if !create_parent_if_missing => { + return Err(Error::other("tier delete chunk parent disappeared before batch creation")); + } + None => { + let parent = TierDeleteDispatchParent { + version: TIER_DELETE_DISPATCH_PARENT_VERSION, + record_type: TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE.to_string(), + operation_id: uuid::Uuid::new_v4(), + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + topology_generation: topology_generation.clone(), + revision: 0, + next_chunk_sequence: 0, + completed_journal_count: 0, + active_chunk: None, + state: TierDeleteDispatchParentState::Active, + }; + parent.validate(&parent_name)?; + let fences_current = + || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent); + match save_config_if_none_fenced( + api.clone(), + &parent_name, + encode_tier_delete_dispatch_parent(&parent)?, + &fences_current, + ) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => continue, + Err(err) => return Err(err), + } + } + } + }; + if parent.bucket != bucket + || parent.bucket_incarnation != bucket_incarnation + || parent.prefix != prefix + || parent.topology_generation != topology_generation + || parent.state != TierDeleteDispatchParentState::Active + || parent.active_chunk.is_some() + { + return Err(Error::other("tier delete chunk parent is not ready for a successor batch")); + } + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentPersisted).await; + let child_operation_id = uuid::Uuid::new_v4(); + let journal_names = tier_delete_dispatch_desired_names(&entries, child_operation_id)?; + let journal_set_sha256 = tier_delete_dispatch_journal_set_digest(&journal_names); + let child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, bucket_incarnation, prefix, child_operation_id); + let child = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id: child_operation_id, + bucket: bucket.to_string(), + bucket_incarnation, + prefix: prefix.to_string(), + journal_count: journal_names + .len() + .try_into() + .map_err(|_| Error::other("tier delete chunk journal count is not representable"))?, + journal_names, + journal_set_sha256: journal_set_sha256.clone(), + topology_generation, + state: TierDeleteDispatchManifestState::Preparing, + }; + let child_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&child_name)) + .await?; + let child_guard = child_lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let (child_etag, binding) = { + let write_fences_current = || { + tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, &fleet_proof, &parent) + && !child_guard.is_lock_lost() + }; + match save_config_if_none_fenced( + api.clone(), + &child_name, + encode_tier_delete_dispatch_manifest(&child)?, + &write_fences_current, + ) + .await + { + Ok(()) | Err(Error::PreconditionFailed) => {} + Err(err) => return Err(err), + } + let (observed_child, child_etag) = read_tier_delete_dispatch_manifest(api.clone(), &child_name) + .await? + .ok_or_else(|| Error::other("tier delete chunk manifest disappeared after creation"))?; + if observed_child != child || !write_fences_current() { + return Err(Error::other("tier delete chunk manifest changed during creation")); + } + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ChildManifestPersisted).await; + let binding = TierDeleteDispatchChunkBinding { + sequence: parent.next_chunk_sequence, + operation_id: child_operation_id, + manifest_object: child_name.clone(), + journal_set_sha256, + journal_count: child.journal_count, + }; + let mut bound_parent = parent.clone(); + bound_parent.revision = bound_parent + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + bound_parent.active_chunk = Some(binding.clone()); + if let Err(err) = cas_tier_delete_dispatch_parent( + api.clone(), + &parent_name, + &parent, + &parent_etag, + &bound_parent, + &write_fences_current, + ) + .await + { + let _ = + seal_and_rollback_preparing_dispatch(api, &child_name, &child, &fleet_proof, bucket_fence, &child_guard).await; + return Err(err); + } + (child_etag, binding) + }; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentBound).await; + let parent_advance = TierDeleteDispatchParentAdvance { + parent_manifest_object: parent_name, + parent_operation_id: parent.operation_id, + chunk: binding, + }; + let prepared = finish_preparing_tier_delete_dispatch( + api, + &child_name, + child, + child_etag, + entries, + fleet_proof, + bucket_fence, + &child_guard, + Some(&parent_guard), + Some(parent_advance), + ) + .await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::DispatchAuthorized).await; + Ok(prepared) +} + +pub(crate) async fn complete_tier_delete_chunk_parent( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + bucket_fence: &crate::object_api::NamespaceLockFence, + fleet_proof: &TierDeleteJournalFleetProofToken, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, bucket_incarnation, prefix); + let Some((record, _)) = read_tier_delete_dispatch_record(api.clone(), &parent_name).await? else { + return Ok(false); + }; + if matches!(record, TierDeleteDispatchRecord::Manifest(_)) { + return Ok(false); + } + let parent_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&parent_name)) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, etag) = match read_tier_delete_dispatch_record(api.clone(), &parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + _ => return Err(Error::other("tier delete chunk parent changed before final completion")), + }; + if parent.bucket != bucket + || parent.bucket_incarnation != bucket_incarnation + || parent.prefix != prefix + || parent.active_chunk.is_some() + || tier_delete_journal_topology_generation(fleet_proof) != parent.topology_generation + { + return Err(Error::other("tier delete chunk parent is not ready for final completion")); + } + if !tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, fleet_proof, &parent) { + return Err(Error::other("tier delete chunk parent completion fence changed")); + } + if parent.state == TierDeleteDispatchParentState::Completed { + return Ok(true); + } + let fences_current = || tier_delete_dispatch_parent_fences_current(bucket_fence, &parent_guard, fleet_proof, &parent); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &parent_name, &parent, &fences_current).await?; + let mut completed = parent.clone(); + completed.revision = completed + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + completed.state = TierDeleteDispatchParentState::Completed; + cas_tier_delete_dispatch_parent(api, &parent_name, &parent, &etag, &completed, &fences_current).await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentCompleted).await; + Ok(true) +} + pub(crate) async fn prepare_tier_delete_dispatch( api: Arc, bucket: &str, @@ -2318,10 +3152,34 @@ pub(crate) async fn prepare_tier_delete_dispatch( entries, fleet_proof, bucket_fence, + true, )) .await } +pub(crate) async fn resume_tier_delete_dispatch( + api: Arc, + bucket: &str, + bucket_incarnation: uuid::Uuid, + prefix: &str, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result { + Box::pin(prepare_tier_delete_dispatch_inner( + api, + bucket, + bucket_incarnation, + prefix, + entries, + fleet_proof, + bucket_fence, + false, + )) + .await +} + +#[allow(clippy::too_many_arguments)] async fn prepare_tier_delete_dispatch_inner( api: Arc, bucket: &str, @@ -2330,6 +3188,7 @@ async fn prepare_tier_delete_dispatch_inner( entries: Vec, fleet_proof: TierDeleteJournalFleetProofToken, bucket_fence: &crate::object_api::NamespaceLockFence, + create_if_missing: bool, ) -> Result { if bucket_incarnation.is_nil() || bucket_fence.is_lock_lost() || !tier_delete_journal_fleet_proof_matches(&fleet_proof) { return Err(Error::other("tier delete journal v6 fleet capability is unavailable")); @@ -2344,7 +3203,7 @@ async fn prepare_tier_delete_dispatch_inner( .await?; ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; - let (mut manifest, manifest_etag) = loop { + let (manifest, manifest_etag) = loop { ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await? { Some((existing, etag)) => { @@ -2394,6 +3253,11 @@ async fn prepare_tier_delete_dispatch_inner( } } None => { + if !create_if_missing { + return Err(Error::other( + "legacy tier delete dispatch disappeared; retry to establish a chunk parent if still required", + )); + } let operation_id = uuid::Uuid::new_v4(); let desired_names = tier_delete_dispatch_desired_names(&entries, operation_id)?; let desired_digest = tier_delete_dispatch_journal_set_digest(&desired_names); @@ -2426,18 +3290,48 @@ async fn prepare_tier_delete_dispatch_inner( } }; + finish_preparing_tier_delete_dispatch( + api, + &manifest_name, + manifest, + manifest_etag, + entries, + fleet_proof, + bucket_fence, + &operation_guard, + None, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn finish_preparing_tier_delete_dispatch( + api: Arc, + manifest_name: &str, + mut manifest: TierDeleteDispatchManifest, + manifest_etag: String, + entries: Vec, + fleet_proof: TierDeleteJournalFleetProofToken, + bucket_fence: &crate::object_api::NamespaceLockFence, + operation_guard: &rustfs_lock::NamespaceLockGuard, + parent_operation_guard: Option<&rustfs_lock::NamespaceLockGuard>, + parent_advance: Option, +) -> Result { let bound = bind_dispatch_entries( entries, manifest.operation_id, - &manifest_name, + manifest_name, &manifest.journal_set_sha256, &manifest.topology_generation, )?; let attempt = async { - let fences_current = - || dispatch_write_fences_current(bucket_fence, &operation_guard, &fleet_proof, &manifest.topology_generation); + let fences_current = || { + dispatch_write_fences_current(bucket_fence, operation_guard, &fleet_proof, &manifest.topology_generation) + && parent_operation_guard.is_none_or(|guard| !guard.is_lock_lost()) + }; let manifest_ref = &manifest; - let operation_guard_ref = &operation_guard; + let operation_guard_ref = operation_guard; let prepare_stopped = Arc::new(AtomicBool::new(false)); let mut prepare_writes = futures::stream::iter((0..bound.len()).map(|index| { let api = api.clone(); @@ -2451,7 +3345,7 @@ async fn prepare_tier_delete_dispatch_inner( return Ok(()); } let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { - Ok(()) => persist_prepared_dispatch_journal(api, manifest, &entry, fences_current).await, + Ok(()) => persist_prepared_dispatch_journal(api, manifest_name, manifest, &entry, fences_current).await, Err(err) => Err(err), }; if result.is_err() { @@ -2491,7 +3385,7 @@ async fn prepare_tier_delete_dispatch_inner( return Ok(()); } let result = match ensure_dispatch_lock_current(bucket_fence, operation_guard) { - Ok(()) => dispatch_prepared_journal(api, manifest, &entry, fences_current) + Ok(()) => dispatch_prepared_journal(api, manifest_name, manifest, &entry, fences_current) .await .map(|_| ()), Err(err) => Err(err), @@ -2515,7 +3409,7 @@ async fn prepare_tier_delete_dispatch_inner( if let Some(err) = dispatch_error { return Err(err); } - ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; + ensure_dispatch_lock_current(bucket_fence, operation_guard)?; if !tier_delete_journal_fleet_proof_matches(&fleet_proof) || tier_delete_journal_topology_generation(&fleet_proof) != manifest.topology_generation { @@ -2523,7 +3417,7 @@ async fn prepare_tier_delete_dispatch_inner( } manifest.state = TierDeleteDispatchManifestState::DispatchAuthorized; let authorized_data = encode_tier_delete_dispatch_manifest(&manifest)?; - match save_config_if_match_fenced(api.clone(), &manifest_name, authorized_data, &manifest_etag, &fences_current).await { + match save_config_if_match_fenced(api.clone(), manifest_name, authorized_data, &manifest_etag, &fences_current).await { Ok(()) => {} Err(Error::PreconditionFailed) => { return Err(Error::other("tier delete dispatch manifest changed before authorization")); @@ -2533,7 +3427,7 @@ async fn prepare_tier_delete_dispatch_inner( // The write may have reached quorum even if the client saw a // timeout. Only a strong read confirming Authorized permits // mutation; every other outcome is retained for recovery. - match read_tier_delete_dispatch_manifest(api.clone(), &manifest_name).await { + match read_tier_delete_dispatch_manifest(api.clone(), manifest_name).await { Ok(Some((observed, _))) if observed.operation_id == manifest.operation_id && observed.state == TierDeleteDispatchManifestState::DispatchAuthorized => {} @@ -2546,7 +3440,7 @@ async fn prepare_tier_delete_dispatch_inner( .await; if let Err(err) = attempt { - let authorized = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + let authorized = read_tier_delete_dispatch_manifest(api.clone(), manifest_name) .await .ok() .flatten() @@ -2555,8 +3449,15 @@ async fn prepare_tier_delete_dispatch_inner( && current.state == TierDeleteDispatchManifestState::DispatchAuthorized }); if !authorized - && let Err(rollback_err) = - seal_and_rollback_preparing_dispatch(api.clone(), &manifest, &fleet_proof, bucket_fence, &operation_guard).await + && let Err(rollback_err) = seal_and_rollback_preparing_dispatch( + api.clone(), + manifest_name, + &manifest, + &fleet_proof, + bucket_fence, + operation_guard, + ) + .await { warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, @@ -2572,8 +3473,17 @@ async fn prepare_tier_delete_dispatch_inner( // Re-read the exact Authorized manifest and every bound journal before // constructing the private one-shot permit. - ensure_dispatch_lock_current(bucket_fence, &operation_guard)?; - authorized_dispatch_permit(api, &manifest_name, fleet_proof, bucket_fence, &operation_guard).await + ensure_dispatch_lock_current(bucket_fence, operation_guard)?; + authorized_dispatch_permit_with_parent( + api, + manifest_name, + fleet_proof, + bucket_fence, + operation_guard, + parent_operation_guard, + parent_advance, + ) + .await } async fn commit_dispatched_journal( @@ -2592,7 +3502,7 @@ async fn commit_dispatched_journal( let (mut current, etag) = read_tier_delete_journal_with_etag(api.clone(), &name) .await? .ok_or_else(|| Error::other("dispatched tier delete journal disappeared before commit"))?; - validate_bound_journal(manifest, &name, ¤t)?; + validate_bound_journal(manifest, &authorization.0.manifest_object, &name, ¤t)?; if !same_tier_delete_journal_identity(¤t, expected) { return Err(Error::other("dispatched tier delete journal changed identity before commit")); } @@ -2621,6 +3531,75 @@ async fn commit_dispatched_journal( Err(Error::other("tier delete journal changed repeatedly during commit")) } +async fn advance_tier_delete_chunk_parent( + api: Arc, + active: &ActiveTierDeleteDispatch, + bucket_fence: &crate::object_api::NamespaceLockFence, +) -> Result<()> { + let Some(advance) = active.parent_advance.as_ref() else { + return Ok(()); + }; + let parent_lock = api + .new_ns_lock( + RUSTFS_META_BUCKET, + &tier_delete_dispatch_operation_lock_name(&advance.parent_manifest_object), + ) + .await?; + let parent_guard = parent_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, parent_etag) = match read_tier_delete_dispatch_record(api.clone(), &advance.parent_manifest_object).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + _ => return Err(Error::other("tier delete chunk parent disappeared before batch progress")), + }; + if parent.operation_id != advance.parent_operation_id + || parent.state != TierDeleteDispatchParentState::Active + || parent.active_chunk.as_ref() != Some(&advance.chunk) + || parent.bucket != active.manifest.bucket + || parent.bucket_incarnation != active.manifest.bucket_incarnation + || parent.prefix != active.manifest.prefix + || parent.topology_generation != active.manifest.topology_generation + { + return Err(Error::other("tier delete chunk parent changed before batch progress")); + } + let (child, _) = read_tier_delete_dispatch_manifest(api.clone(), &advance.chunk.manifest_object) + .await? + .ok_or_else(|| Error::other("completed tier delete child manifest disappeared before parent progress"))?; + if child.state != TierDeleteDispatchManifestState::Completed + || !tier_delete_dispatch_child_matches_parent(&parent, &advance.chunk, &child) + { + return Err(Error::other("tier delete child is not durably completed before parent progress")); + } + let fences_current = || { + !bucket_fence.is_lock_lost() + && !parent_guard.is_lock_lost() + && active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) + .is_ok() + }; + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), &advance.parent_manifest_object, &parent, &fences_current) + .await?; + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(advance.chunk.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, &advance.parent_manifest_object, &parent, &parent_etag, &next, &fences_current).await?; + #[cfg(all(test, feature = "test-util"))] + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ParentProgressed).await; + Ok(()) +} + pub(crate) async fn complete_tier_delete_dispatch( api: Arc, active: &ActiveTierDeleteDispatch, @@ -2632,113 +3611,127 @@ pub(crate) async fn complete_tier_delete_dispatch( if !active.authorization.mutation_started() { return Err(Error::other("tier delete dispatch cannot commit before its local mutation starts")); } - let manifest_name = tier_delete_dispatch_manifest_object_name( - &active.manifest.bucket, - active.manifest.bucket_incarnation, - &active.manifest.prefix, - ); - let operation_lock = api - .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) - .await?; - let operation_guard = operation_lock - .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) - .await?; - let fences_current = || { - !bucket_fence.is_lock_lost() - && !operation_guard.is_lock_lost() - && active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) - .is_ok() - }; - ensure_durable_write_fence(&fences_current, "before tier delete dispatch completion")?; - let make_commit = |index: usize| { - let api = api.clone(); - let fences_current = &fences_current; - async move { - active.authorization.ensure_current( - &active.manifest.bucket, - active.manifest.bucket_incarnation, - &active.manifest.prefix, - )?; - commit_dispatched_journal(api, &active.manifest, &active.entries[index], &active.authorization, fences_current) - .await - .map(|entry| (index, entry)) - } - }; - let mut next = 0; - let mut commits = futures::stream::FuturesUnordered::new(); - while next < active.entries.len() && commits.len() < TIER_DELETE_DISPATCH_CAS_CONCURRENCY { - commits.push(make_commit(next)); - next += 1; - } - let mut committed_entries = vec![None; active.entries.len()]; - let mut first_error = None; - while let Some(result) = commits.next().await { - match result { - Ok((index, entry)) => committed_entries[index] = Some(entry), - Err(err) if first_error.is_none() => first_error = Some(err), - Err(_) => {} - } - // A failed member CAS leaves the Authorized manifest and every - // already-committed member recoverable. Stop admitting tail work, but - // drain the bounded in-flight set before releasing the operation lock. - if first_error.is_none() && next < active.entries.len() { + // The caller holds bucket lifecycle WRITE. Scope the child operation lock + // so it is released before `advance_tier_delete_chunk_parent` takes the + // parent operation lock. + { + let manifest_name = active.manifest_object.clone(); + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(&manifest_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let fences_current = || { + !bucket_fence.is_lock_lost() + && !operation_guard.is_lock_lost() + && active + .authorization + .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix) + .is_ok() + }; + ensure_durable_write_fence(&fences_current, "before tier delete dispatch completion")?; + let make_commit = |index: usize| { + let api = api.clone(); + let fences_current = &fences_current; + async move { + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + commit_dispatched_journal(api, &active.manifest, &active.entries[index], &active.authorization, fences_current) + .await + .map(|entry| (index, entry)) + } + }; + let mut next = 0; + let mut commits = futures::stream::FuturesUnordered::new(); + while next < active.entries.len() && commits.len() < TIER_DELETE_DISPATCH_CAS_CONCURRENCY { commits.push(make_commit(next)); next += 1; } - } - if let Some(err) = first_error { - return Err(err); - } - let committed_entries = committed_entries - .into_iter() - .collect::>>() - .ok_or_else(|| Error::other("tier delete dispatch completion omitted a journal"))?; + let mut committed_entries = vec![None; active.entries.len()]; + let mut first_error = None; + while let Some(result) = commits.next().await { + match result { + Ok((index, entry)) => committed_entries[index] = Some(entry), + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + // A failed member CAS leaves the Authorized manifest and every + // already-committed member recoverable. Stop admitting tail work, + // but drain the bounded in-flight set before releasing the operation lock. + if first_error.is_none() && next < active.entries.len() { + commits.push(make_commit(next)); + next += 1; + } + } + if let Some(err) = first_error { + return Err(err); + } + let committed_entries = committed_entries + .into_iter() + .collect::>>() + .ok_or_else(|| Error::other("tier delete dispatch completion omitted a journal"))?; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - let (mut current, etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) - .await? - .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared before completion"))?; - if current.operation_id != active.manifest.operation_id - || current.journal_set_sha256 != active.manifest.journal_set_sha256 - || current.state != TierDeleteDispatchManifestState::DispatchAuthorized - || (etag != active.authorized_etag && current != active.manifest) - { - return Err(Error::other("tier delete dispatch manifest changed before completion")); - } - current.state = TierDeleteDispatchManifestState::Completed; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - let completion = save_config_if_match_fenced( - api.clone(), - &manifest_name, - encode_tier_delete_dispatch_manifest(¤t)?, - &etag, - &fences_current, - ) - .await; - active - .authorization - .ensure_current(&active.manifest.bucket, active.manifest.bucket_incarnation, &active.manifest.prefix)?; - completion?; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + let (mut current, etag) = read_tier_delete_dispatch_manifest(api.clone(), &manifest_name) + .await? + .ok_or_else(|| Error::other("authorized tier delete dispatch manifest disappeared before completion"))?; + if current.operation_id != active.manifest.operation_id + || current.journal_set_sha256 != active.manifest.journal_set_sha256 + || current.state != TierDeleteDispatchManifestState::DispatchAuthorized + || (etag != active.authorized_etag && current != active.manifest) + { + return Err(Error::other("tier delete dispatch manifest changed before completion")); + } + current.state = TierDeleteDispatchManifestState::Completed; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + let completion = save_config_if_match_fenced( + api.clone(), + &manifest_name, + encode_tier_delete_dispatch_manifest(¤t)?, + &etag, + &fences_current, + ) + .await; + active.authorization.ensure_current( + &active.manifest.bucket, + active.manifest.bucket_incarnation, + &active.manifest.prefix, + )?; + completion?; - for entry in committed_entries { - if let Err(err) = enqueue_committed_tier_delete_journal_entry(&entry).await { - debug!( - event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - operation_id = %current.operation_id, - error = ?err, - "Committed tier delete dispatch will be picked up by periodic recovery" - ); + #[cfg(all(test, feature = "test-util"))] + if active.is_chunked() { + tier_delete_chunk_test_pause(TierDeleteChunkTestStage::ChildCompleted).await; + } + + for entry in committed_entries { + if let Err(err) = enqueue_committed_tier_delete_journal_entry(&entry).await { + debug!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + operation_id = %current.operation_id, + error = ?err, + "Committed tier delete dispatch will be picked up by periodic recovery" + ); + } } } + advance_tier_delete_chunk_parent(api, active, bucket_fence).await?; + Ok(()) } @@ -3338,8 +4331,12 @@ async fn load_manifest_for_journal( let (manifest, _) = read_tier_delete_dispatch_manifest(api, &binding.manifest_object) .await? .ok_or_else(|| Error::other("tier delete dispatch manifest is missing"))?; - validate_bound_journal(&manifest, journal_name, journal)?; - if manifest.journal_names.binary_search(&journal_name.to_string()).is_err() { + validate_bound_journal(&manifest, &binding.manifest_object, journal_name, journal)?; + if manifest + .journal_names + .binary_search_by(|name| name.as_str().cmp(journal_name)) + .is_err() + { return Err(Error::other("tier delete dispatch manifest does not contain its journal")); } Ok(manifest) @@ -3582,6 +4579,138 @@ enum TierDeleteDispatchManifestRecoveryOutcome { Retained, } +fn parent_recovery_fences_current( + bucket_guard: &rustfs_lock::NamespaceLockGuard, + operation_guard: &rustfs_lock::NamespaceLockGuard, + fleet_proof: &TierDeleteJournalFleetProofToken, + parent: &TierDeleteDispatchParent, + cancel_token: Option<&CancellationToken>, +) -> bool { + cancel_token.is_none_or(|token| !token.is_cancelled()) + && !bucket_guard.is_lock_lost() + && !operation_guard.is_lock_lost() + && tier_delete_journal_fleet_proof_matches(fleet_proof) + && tier_delete_journal_topology_generation(fleet_proof) == parent.topology_generation +} + +async fn delete_tier_delete_dispatch_parent_if_match_confirmed( + api: Arc, + parent_name: &str, + parent: &TierDeleteDispatchParent, + etag: &str, + fences_current: &impl Fn() -> bool, +) -> Result<()> { + let data = encode_tier_delete_dispatch_parent(parent)?; + let delete = delete_durable_config_if_match(api.clone(), parent_name, &data, etag, fences_current).await; + let observed = read_tier_delete_dispatch_record(api.clone(), parent_name).await?; + if !fences_current() { + return Err(Error::other("tier delete dispatch parent recovery fence changed during deletion")); + } + match observed { + None => Ok(()), + Some((TierDeleteDispatchRecord::Parent(observed), _)) => { + let observed_data = encode_tier_delete_dispatch_parent(&observed)?; + if api + .durable_ilm_terminal_receipt_covers_active_source(parent_name, &observed_data) + .await? + { + Ok(()) + } else { + Err(Error::other_with_context( + "tier delete dispatch parent changed during deletion", + format!("observed {:?}, delete result {:?}", observed.state, delete.as_ref().err()), + )) + } + } + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + Err(Error::other("tier delete dispatch parent was replaced during deletion")) + } + } +} + +async fn process_tier_delete_dispatch_parent( + api: Arc, + parent_name: &str, + observed_before_lock: &TierDeleteDispatchParent, + cancel_token: Option<&CancellationToken>, +) -> Result { + if cancel_token.is_some_and(CancellationToken::is_cancelled) { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if tier_delete_journal_topology_generation(&fleet_proof) != observed_before_lock.topology_generation { + return Err(Error::other("tier delete chunk parent topology generation changed")); + } + // Background lock order matches the request path: bucket lifecycle WRITE + // precedes the parent operation lock. This path never takes a child lock. + let bucket_guard = api.acquire_bucket_lifecycle_write_lock(&observed_before_lock.bucket).await?; + let operation_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &tier_delete_dispatch_operation_lock_name(parent_name)) + .await?; + let operation_guard = operation_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let (parent, etag) = match read_tier_delete_dispatch_record(api.clone(), parent_name).await? { + Some((TierDeleteDispatchRecord::Parent(parent), etag)) => (parent, etag), + Some((TierDeleteDispatchRecord::Manifest(_), _)) => { + return Err(Error::other("tier delete chunk parent was replaced by a legacy manifest")); + } + None => return Err(Error::ConfigNotFound), + }; + if parent.operation_id != observed_before_lock.operation_id + || parent.bucket != observed_before_lock.bucket + || !parent_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, &parent, cancel_token) + { + return Err(Error::other("tier delete chunk parent recovery fence changed")); + } + let fences_current = || parent_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, &parent, cancel_token); + record_tier_delete_dispatch_parent_progress_fenced(api.clone(), parent_name, &parent, &fences_current).await?; + if parent.state == TierDeleteDispatchParentState::Completed { + delete_tier_delete_dispatch_parent_if_match_confirmed(api, parent_name, &parent, &etag, &fences_current).await?; + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Deleted); + } + let Some(binding) = parent.active_chunk.clone() else { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + }; + let child = read_tier_delete_dispatch_manifest(api.clone(), &binding.manifest_object).await?; + let Some((child, _)) = child else { + if !tier_delete_dispatch_chunk_journal_namespace_empty(api.clone(), binding.operation_id).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, parent_name, &parent, &etag, &next, &fences_current).await?; + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced); + }; + if !tier_delete_dispatch_child_matches_parent(&parent, &binding, &child) { + return Err(Error::other("tier delete chunk parent child binding changed during recovery")); + } + if child.state != TierDeleteDispatchManifestState::Completed { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + let mut next = parent.clone(); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent revision overflow"))?; + next.next_chunk_sequence = next + .next_chunk_sequence + .checked_add(1) + .ok_or_else(|| Error::other("tier delete chunk parent sequence overflow"))?; + next.completed_journal_count = next + .completed_journal_count + .checked_add(binding.journal_count) + .ok_or_else(|| Error::other("tier delete chunk parent journal count overflow"))?; + next.active_chunk = None; + cas_tier_delete_dispatch_parent(api, parent_name, &parent, &etag, &next, &fences_current).await?; + Ok(TierDeleteDispatchManifestRecoveryOutcome::Advanced) +} + fn manifest_recovery_fences_current( bucket_guard: &rustfs_lock::NamespaceLockGuard, operation_guard: &rustfs_lock::NamespaceLockGuard, @@ -3674,8 +4803,52 @@ async fn delete_tier_delete_dispatch_manifest_if_match_confirmed( } } +/// Child manifests share the dispatch-record namespace with the legacy/root +/// manifest and are therefore returned by the same recovery listing. A child +/// that is still bound by an active chunk parent must remain available until +/// the parent records the child completion and advances its sequence/count. +/// The bucket lifecycle WRITE lock held by the caller serializes this check +/// with request-side parent inspection/progress, so no parent lock nesting is +/// needed here. +async fn child_dispatch_manifest_is_bound_to_active_parent( + api: Arc, + manifest_name: &str, + manifest: &TierDeleteDispatchManifest, +) -> Result { + let parent_name = tier_delete_dispatch_manifest_object_name(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix); + if parent_name == manifest_name { + return Ok(false); + } + let Some((record, _)) = read_tier_delete_dispatch_record(api, &parent_name).await? else { + return Ok(false); + }; + let TierDeleteDispatchRecord::Parent(parent) = record else { + return Err(Error::other("tier delete child exists beside a legacy root manifest")); + }; + if parent.bucket != manifest.bucket + || parent.bucket_incarnation != manifest.bucket_incarnation + || parent.prefix != manifest.prefix + || parent.topology_generation != manifest.topology_generation + { + return Err(Error::other("tier delete child parent identity changed during recovery")); + } + let Some(binding) = parent.active_chunk.as_ref() else { + return Ok(false); + }; + if binding.manifest_object != manifest_name { + return Ok(false); + } + if parent.state != TierDeleteDispatchParentState::Active + || !tier_delete_dispatch_child_matches_parent(&parent, binding, manifest) + { + return Err(Error::other("tier delete child binding is inconsistent during recovery")); + } + Ok(true) +} + async fn authorized_dispatch_all_committed( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result { @@ -3698,7 +4871,7 @@ async fn authorized_dispatch_all_committed( ensure_tier_delete_dispatch_member_scan_fence(fences_current)?; let (entry, _) = observed .ok_or_else(|| Error::other("authorized tier delete dispatch manifest references a missing journal"))?; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; #[cfg(all(test, feature = "test-util"))] tier_delete_dispatch_authorized_progress_test_observed(); record_tier_delete_journal_progress_fenced(api, &name, &entry, fences_current).await?; @@ -3752,6 +4925,7 @@ async fn authorized_dispatch_all_committed( async fn completed_dispatch_has_present_journal( api: Arc, + manifest_object: &str, manifest: &TierDeleteDispatchManifest, fences_current: &impl Fn() -> bool, ) -> Result { @@ -3776,7 +4950,7 @@ async fn completed_dispatch_has_present_journal( let Some((entry, _)) = observed else { return Ok(None); }; - validate_bound_journal(manifest, &name, &entry)?; + validate_bound_journal(manifest, manifest_object, &name, &entry)?; if entry.state != TierDeleteJournalState::Committed { return Err(Error::other("completed tier delete dispatch contains an uncommitted journal")); } @@ -3845,6 +5019,8 @@ async fn process_tier_delete_dispatch_manifest( } return Err(Error::other("tier delete dispatch manifest recovery fence changed")); } + let child_bound_to_active_parent = + child_dispatch_manifest_is_bound_to_active_parent(api.clone(), manifest_name, ¤t).await?; { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); @@ -3883,7 +5059,7 @@ async fn process_tier_delete_dispatch_manifest( { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - delete_staged_dispatch_journal_set(api.clone(), ¤t, &fences_current).await?; + delete_staged_dispatch_journal_set(api.clone(), manifest_name, ¤t, &fences_current).await?; } if current.state == TierDeleteDispatchManifestState::Aborting { let next = { @@ -3907,7 +5083,7 @@ async fn process_tier_delete_dispatch_manifest( } return Err(Error::other("tier delete dispatch manifest recovery fence changed")); } - validate_staged_dispatch_journal_set(api.clone(), ¤t, &|| { + validate_staged_dispatch_journal_set(api.clone(), manifest_name, ¤t, &|| { manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) }) .await?; @@ -3920,7 +5096,7 @@ async fn process_tier_delete_dispatch_manifest( if current.state == TierDeleteDispatchManifestState::DispatchAuthorized { let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - if !authorized_dispatch_all_committed(api.clone(), ¤t, &fences_current).await? { + if !authorized_dispatch_all_committed(api.clone(), manifest_name, ¤t, &fences_current).await? { return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); } let next = { @@ -3947,7 +5123,14 @@ async fn process_tier_delete_dispatch_manifest( } let fences_current = || manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token); - if completed_dispatch_has_present_journal(api.clone(), ¤t, &fences_current).await? { + if completed_dispatch_has_present_journal(api.clone(), manifest_name, ¤t, &fences_current).await? { + return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); + } + if child_bound_to_active_parent { + // The parent must observe the completed child and durably add its + // journal count before the child record can be removed. Otherwise a + // concurrent child recovery could erase the only evidence needed to + // advance the parent's sequence. return Ok(TierDeleteDispatchManifestRecoveryOutcome::Retained); } if !manifest_recovery_fences_current(&bucket_guard, &operation_guard, &fleet_proof, ¤t, cancel_token) { @@ -4059,8 +5242,8 @@ async fn recover_tier_delete_dispatch_manifest_object( return TierDeleteDispatchManifestScanOutcome::Failed; } }; - let manifest = match decode_tier_delete_dispatch_manifest(&data, &object_name) { - Ok(manifest) => manifest, + let record = match decode_tier_delete_dispatch_record(&data, &object_name) { + Ok(record) => record, Err(err) => { warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, @@ -4068,11 +5251,15 @@ async fn recover_tier_delete_dispatch_manifest_object( subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, error = ?err, - "Invalid tier delete dispatch manifest is quarantined" + "Invalid tier delete dispatch record is quarantined" ); return TierDeleteDispatchManifestScanOutcome::Failed; } }; + let operation_id = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => manifest.operation_id, + TierDeleteDispatchRecord::Parent(parent) => parent.operation_id, + }; match api .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data) .await @@ -4090,14 +5277,25 @@ async fn recover_tier_delete_dispatch_manifest_object( component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, - operation_id = %manifest.operation_id, + operation_id = %operation_id, error = ?err, "Tier delete dispatch terminal source proof will retry later" ); return TierDeleteDispatchManifestScanOutcome::Failed; } } - let result = process_tier_delete_dispatch_manifest(api, &object_name, &manifest, cancel_token.as_ref()).await; + let state = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => format!("{:?}", manifest.state), + TierDeleteDispatchRecord::Parent(parent) => format!("parent::{:?}", parent.state), + }; + let result = match &record { + TierDeleteDispatchRecord::Manifest(manifest) => { + process_tier_delete_dispatch_manifest(api, &object_name, manifest, cancel_token.as_ref()).await + } + TierDeleteDispatchRecord::Parent(parent) => { + process_tier_delete_dispatch_parent(api, &object_name, parent, cancel_token.as_ref()).await + } + }; if cancel_token.as_ref().is_some_and(CancellationToken::is_cancelled) { // Cancellation is cooperative: every admitted mutation above has // already returned while the operation/fleet guards and registry @@ -4116,8 +5314,8 @@ async fn recover_tier_delete_dispatch_manifest_object( component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, manifest_object = %object_name, - operation_id = %manifest.operation_id, - state = ?manifest.state, + operation_id = %operation_id, + state = %state, error = ?err, "Tier delete dispatch manifest recovery will retry later" ); @@ -4574,11 +5772,17 @@ where #[cfg(test)] mod tests { use super::{ + TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX, - await_tier_delete_journal_recovery, decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, - object_info_references_tier_delete, record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, - same_tier_delete_journal_identity, tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, + TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent, + TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery, + decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest, + encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete, + record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity, + tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name, + tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name, + tier_delete_source_matches_dispatch_scope, }; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -4603,6 +5807,118 @@ mod tests { } } + #[test] + fn chunk_parent_and_child_preserve_the_legacy_fail_closed_fence() { + let legacy_v1_manifest_path_accepts = |data: &[u8], object_name: &str| { + serde_json::from_slice::(data).is_ok_and(|manifest| { + tier_delete_dispatch_manifest_object_name(&manifest.bucket, manifest.bucket_incarnation, &manifest.prefix) + == object_name + }) + }; + let bucket = "chunked-bucket"; + let incarnation = uuid::Uuid::new_v4(); + let prefix = "large/"; + let parent_name = tier_delete_dispatch_manifest_object_name(bucket, incarnation, prefix); + let parent = TierDeleteDispatchParent { + version: TIER_DELETE_DISPATCH_PARENT_VERSION, + record_type: TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE.to_string(), + operation_id: uuid::Uuid::new_v4(), + bucket: bucket.to_string(), + bucket_incarnation: incarnation, + prefix: prefix.to_string(), + topology_generation: "a".repeat(64), + revision: 0, + next_chunk_sequence: 0, + completed_journal_count: 0, + active_chunk: None, + state: TierDeleteDispatchParentState::Active, + }; + let parent_data = encode_tier_delete_dispatch_parent(&parent).expect("chunk parent should encode"); + assert!( + serde_json::from_slice::(&parent_data).is_err(), + "the legacy v1 manifest codec must reject a chunk-parent root sentinel" + ); + assert!(matches!( + decode_tier_delete_dispatch_record(&parent_data, &parent_name).expect("current codec should accept the parent"), + TierDeleteDispatchRecord::Parent(_) + )); + assert!( + decode_tier_delete_dispatch_record(&parent_data, &format!("{parent_name}.other")).is_err(), + "a chunk parent must remain bound to the deterministic legacy root" + ); + let mut impossible_progress = parent.clone(); + impossible_progress.revision = 2; + impossible_progress.next_chunk_sequence = 2; + impossible_progress.completed_journal_count = 1; + let impossible_progress_data = + encode_tier_delete_dispatch_parent(&impossible_progress).expect("invalid test parent should still serialize"); + assert!( + decode_tier_delete_dispatch_record(&impossible_progress_data, &parent_name).is_err(), + "a parent cannot complete fewer journals than its completed child count" + ); + + let child_operation_id = uuid::Uuid::new_v4(); + let child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, incarnation, prefix, child_operation_id); + let journal_names = vec![format!( + "{TIER_DELETE_JOURNAL_V6_PREFIX}{}/{}.json", + child_operation_id.simple(), + "b".repeat(64) + )]; + let journal_set_sha256 = tier_delete_dispatch_journal_set_digest(&journal_names); + let child = TierDeleteDispatchManifest { + version: TIER_DELETE_DISPATCH_MANIFEST_VERSION, + operation_id: child_operation_id, + bucket: bucket.to_string(), + bucket_incarnation: incarnation, + prefix: prefix.to_string(), + journal_count: 1, + journal_names, + journal_set_sha256: journal_set_sha256.clone(), + topology_generation: "a".repeat(64), + state: TierDeleteDispatchManifestState::Completed, + }; + let child_data = encode_tier_delete_dispatch_manifest(&child).expect("chunk child should encode"); + assert_ne!(child_name, parent_name); + assert!( + legacy_v1_manifest_path_accepts(&child_data, &parent_name), + "the unchanged child payload must remain byte-compatible at the legacy root" + ); + assert!( + !legacy_v1_manifest_path_accepts(&child_data, &child_name), + "the legacy root-only path validator must reject an operation-scoped child" + ); + assert!(matches!( + decode_tier_delete_dispatch_record(&child_data, &child_name).expect("current codec should accept the child"), + TierDeleteDispatchRecord::Manifest(_) + )); + let wrong_child_name = tier_delete_dispatch_chunk_manifest_object_name(bucket, incarnation, prefix, uuid::Uuid::new_v4()); + assert!( + decode_tier_delete_dispatch_record(&child_data, &wrong_child_name).is_err(), + "a child manifest must remain bound to its exact operation-scoped path" + ); + + let binding = TierDeleteDispatchChunkBinding { + sequence: 0, + operation_id: child_operation_id, + manifest_object: child_name, + journal_set_sha256, + journal_count: 1, + }; + let mut bound_parent = parent; + bound_parent.revision = 1; + bound_parent.active_chunk = Some(binding.clone()); + bound_parent + .validate(&parent_name) + .expect("the exact child binding should produce a valid active parent"); + assert!(tier_delete_dispatch_child_matches_parent(&bound_parent, &binding, &child)); + let mut mismatched_child = child; + mismatched_child.topology_generation = "c".repeat(64); + assert!( + !tier_delete_dispatch_child_matches_parent(&bound_parent, &binding, &mismatched_child), + "a valid child path and payload cannot bypass the exact parent topology binding" + ); + } + fn bound_v6_journal_entry(state: TierDeleteJournalState) -> Jentry { let operation_id = uuid::Uuid::new_v4(); Jentry { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 859415707..00a74906e 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -816,6 +816,7 @@ mod tests { }, tier_delete_journal::{ DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, + TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard, TierDeleteDispatchManifestState, TierDeleteDispatchMemberReadTestHook, TierDeleteDispatchMemberReadTestStage, TierDeleteDispatchRollbackTestHook, complete_tier_delete_dispatch, encode_tier_delete_journal_entry, install_test_tier_delete_dispatch_fixture, persist_tier_delete_journal_entry, prepare_tier_delete_dispatch, @@ -11624,6 +11625,612 @@ mod tests { assert_eq!(backend.remove_versions().await.len(), objects.len()); } + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_prefix_limit_and_multi_chunk_batches_converge() { + run_large_stack_async_test( + "tier-delete-prefix-limit-and-multi-chunk", + tier_delete_prefix_limit_and_multi_chunk_batches_converge_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_prefix_limit_and_multi_chunk_batches_converge_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(2); + let temp_dir = tempfile::tempdir().expect("create chunked prefix-delete store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunked-prefix-delete", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "CHUNKED-PREFIX-DELETE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let bucket = "chunked-prefix-delete-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("chunked prefix source bucket should be created"); + + for (prefix, count) in [("at-limit", 2), ("limit-plus-one", 3), ("multi-chunk", 5)] { + for index in 0..count { + let object = format!("{prefix}/object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("chunked prefix source should be written"); + store + .transition_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("chunked prefix source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("chunked prefix source should transition"); + } + } + backend.set_remove_failure(true); + + store + .delete_object_with_tier_delete_journal( + bucket, + "at-limit/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("exactly one batch must retain the v1 one-shot path"); + + let limit_plus_one_first = store + .delete_object_with_tier_delete_journal( + bucket, + "limit-plus-one/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("limit plus one must start a bounded parent transaction"); + assert!( + limit_plus_one_first.to_string().contains("retry the next durable batch"), + "unexpected first limit-plus-one result: {limit_plus_one_first}" + ); + let active_limit_plus_one_records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 10, + false, + None, + false, + ) + .await + .expect("active limit-plus-one records should be listable"); + let mut active_limit_plus_one_parent_seen = false; + for record in active_limit_plus_one_records + .objects + .iter() + .filter(|record| !record.name.contains("/chunks/")) + { + let data = com::read_config(store.clone(), &record.name) + .await + .expect("an active dispatch root should be readable"); + let value: serde_json::Value = serde_json::from_slice(&data).expect("an active dispatch root should contain JSON"); + if value["prefix"] == "limit-plus-one/" { + assert_eq!(value["record_type"], "chunked_parent"); + active_limit_plus_one_parent_seen = true; + } + } + assert!( + active_limit_plus_one_parent_seen, + "limit plus one must establish the fail-closed parent at the legacy root" + ); + + let mut limit_plus_one_completed = false; + let mut limit_plus_one_retries = 1; + for _ in 0..3 { + match store + .delete_object_with_tier_delete_journal( + bucket, + "limit-plus-one/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + limit_plus_one_completed = true; + break; + } + Err(err) if err.to_string().contains("retry the next durable batch") => { + limit_plus_one_retries += 1; + } + Err(err) => panic!("limit-plus-one delete returned an unexpected error: {err}"), + } + } + assert!(limit_plus_one_completed, "limit plus one must converge through two bounded children"); + assert_eq!(limit_plus_one_retries, 2, "limit plus one must require exactly two child batches"); + + let first_batch = store + .delete_object_with_tier_delete_journal( + bucket, + "multi-chunk/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("the first bounded child must request a successor batch"); + assert!( + first_batch.to_string().contains("retry the next durable batch"), + "unexpected first child result: {first_batch}" + ); + + let newcomer = "multi-chunk/zzz-newcomer.bin"; + let mut newcomer_reader = PutObjReader::from_vec(vec![b'n'; 1024 * 1024]); + let newcomer_source = store + .put_object(bucket, newcomer, &mut newcomer_reader, &ObjectOptions::default()) + .await + .expect("a source created between chunks should be written"); + store + .transition_object( + bucket, + newcomer, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: newcomer_source + .etag + .clone() + .expect("the between-chunks source should have an etag"), + ..Default::default() + }, + mod_time: newcomer_source.mod_time, + ..Default::default() + }, + ) + .await + .expect("a source created between chunks should transition"); + + let mut completed = false; + let mut durable_batch_retries = 1; + for _ in 0..7 { + match store + .delete_object_with_tier_delete_journal( + bucket, + "multi-chunk/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + completed = true; + break; + } + Err(err) + if err.to_string().contains("retry the next durable batch") + || err.to_string().contains("retry the next batch") => + { + durable_batch_retries += 1; + } + Err(err) => panic!("chunked prefix delete returned an unexpected error: {err}"), + } + } + assert!(completed, "six entries must converge through three bounded child batches"); + assert_eq!(durable_batch_retries, 3, "limit two should require exactly three child batches"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 11); + assert_eq!( + backend.object_count().await, + 11, + "remote cleanup must remain durable while the tier is unavailable" + ); + let dispatch_records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 20, + false, + None, + false, + ) + .await + .expect("bounded dispatch records should be listable"); + let mut child_batches = 0; + let mut legacy_at_limit_seen = false; + for record in &dispatch_records.objects { + let data = com::read_config(store.clone(), &record.name) + .await + .expect("a retained dispatch record should be readable"); + let value: serde_json::Value = serde_json::from_slice(&data).expect("a retained dispatch record should contain JSON"); + if record.name.contains("/chunks/") { + let journal_count = value["journal_count"] + .as_u64() + .expect("a retained child should declare its journal count"); + assert!(journal_count <= 2, "a child batch exceeded the configured resource bound"); + child_batches += 1; + } else if value["prefix"] == "at-limit/" { + assert!( + value.get("record_type").is_none(), + "the exact-limit root must remain a legacy v1 manifest" + ); + assert_eq!(value["journal_count"], 2); + legacy_at_limit_seen = true; + } + } + assert!( + legacy_at_limit_seen, + "the exact-limit dispatch must retain its byte-compatible root shape" + ); + assert_eq!(child_batches, 5, "nine chunked sources should persist exactly five bounded children"); + + backend.set_remove_failure(false); + drive_tier_delete_dispatch_restart_to_convergence(store.clone()).await; + assert_eq!(tier_delete_journal_count(store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!( + backend.remove_versions().await.len(), + 11, + "each remote version must be removed exactly once" + ); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources() { + run_large_stack_async_test( + "tier-delete-chunk-crash-boundaries", + tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_chunk_crash_boundaries_resume_without_skipping_sources_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); + let stages = [ + TierDeleteChunkTestStage::ParentPersisted, + TierDeleteChunkTestStage::ChildManifestPersisted, + TierDeleteChunkTestStage::ParentBound, + TierDeleteChunkTestStage::DispatchAuthorized, + TierDeleteChunkTestStage::LocalReplayCompleted, + TierDeleteChunkTestStage::ChildCompleted, + TierDeleteChunkTestStage::ParentProgressed, + TierDeleteChunkTestStage::FinalLocalDeletionCompleted, + TierDeleteChunkTestStage::ParentCompleted, + ]; + for (case, stage) in stages.into_iter().enumerate() { + let temp_dir = tempfile::tempdir().expect("create chunk crash-boundary store dir"); + let initial_name = format!("chunk-crash-boundary-{case}"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), &initial_name, &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = format!("CHUNK-CRASH-BOUNDARY-{case}"); + let backend = register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await; + backend.set_remove_failure(true); + let bucket = format!("chunk-crash-boundary-{case}-bucket"); + let prefix = "prefix/"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("chunk crash-boundary bucket should be created"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'a' + index as u8; 1024 * 1024]); + let source = store + .put_object(&bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("chunk crash-boundary source should be written"); + store + .transition_object( + &bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: source.etag.clone().expect("chunk crash-boundary source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("chunk crash-boundary source should transition"); + } + let local_only = format!("{prefix}local-only.bin"); + let mut local_reader = PutObjReader::from_vec(vec![b'l'; 1024 * 1024]); + store + .put_object(&bucket, &local_only, &mut local_reader, &ObjectOptions::default()) + .await + .expect("chunk crash-boundary local-only object should be written"); + let tier_config = ctx + .tier_config_mgr() + .read() + .await + .tiers + .get(&tier_name) + .expect("chunk crash-boundary tier config should remain available for restart") + .clone_with_credentials(); + + if matches!( + stage, + TierDeleteChunkTestStage::FinalLocalDeletionCompleted | TierDeleteChunkTestStage::ParentCompleted + ) { + for _ in 0..2 { + let retry = store + .delete_object_with_tier_delete_journal( + &bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("each bounded child must complete before the final crash boundary"); + assert!( + retry.to_string().contains("retry the next durable batch"), + "unexpected bounded-child result before {stage:?}: {retry}" + ); + } + } + + let barrier = TierDeleteChunkTestBarrier::install(stage); + let worker_store = store.clone(); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .delete_object_with_tier_delete_journal( + &worker_bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}")); + worker.abort(); + let _ = worker.await; + drop(barrier); + let released_bucket_guard = + tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket)) + .await + .unwrap_or_else(|_| panic!("canceling at {stage:?} did not release the bucket lifecycle lock")) + .unwrap_or_else(|err| { + panic!("bucket lifecycle lock reacquire failed after cancellation at {stage:?}: {err}") + }); + drop(released_bucket_guard); + shutdown.cancel(); + drop(store); + drop(ctx); + + let restarted_name = format!("chunk-crash-boundary-{case}-restart"); + let (restarted_ctx, restarted_store, restarted_shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), &restarted_name, &[4])).await; + { + let tier_config_mgr = restarted_ctx.tier_config_mgr(); + let mut manager = tier_config_mgr.write().await; + manager.tiers.insert(tier_name.clone(), tier_config); + manager + .install_test_driver(&tier_name, Box::new(backend.clone())) + .expect("the exact chunk crash-boundary tier driver should reinstall after restart"); + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(restarted_store.clone(), Vec::new()).await; + + let mut completed = false; + for _ in 0..20 { + let recovery = recover_tier_delete_dispatch_manifests(restarted_store.clone(), 100, None) + .await + .expect("chunk crash-boundary manifest recovery should remain readable"); + assert_eq!(recovery.failed, 0, "recovery must not quarantine a valid chunk boundary"); + match restarted_store + .delete_object_with_tier_delete_journal( + &bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => { + completed = true; + break; + } + Err(err) + if err.to_string().contains("retry") + || err.to_string().contains("rollback") + || err.to_string().contains("durable cleanup") => {} + Err(err) => panic!("chunk crash-boundary retry returned an unexpected error at {stage:?}: {err}"), + } + } + assert!(completed, "chunk state must converge after cancellation at {stage:?}"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + assert!( + restarted_store.pools[0] + .get_disks_by_key(&object) + .load_file_info_versions_exact(&bucket, &object) + .await + .expect("chunk crash-boundary source lookup should succeed") + .is_none(), + "source {object} must not be skipped after cancellation at {stage:?}" + ); + } + assert!( + restarted_store.pools[0] + .get_disks_by_key(&local_only) + .load_file_info_versions_exact(&bucket, &local_only) + .await + .expect("chunk crash-boundary local-only source lookup should succeed") + .is_none(), + "the final raw delete must remove the local-only source after cancellation at {stage:?}" + ); + assert_eq!(backend.object_count().await, 2); + backend.set_remove_failure(false); + drive_tier_delete_dispatch_restart_to_convergence(restarted_store.clone()).await; + assert_eq!(tier_delete_journal_count(restarted_store.clone()).await, 0); + assert_eq!(tier_delete_dispatch_manifest_count(restarted_store.clone()).await, 0); + assert_eq!(backend.object_count().await, 0); + assert_eq!( + backend.remove_versions().await.len(), + 2, + "each crash case must retain exactly one cleanup owner per remote version" + ); + restarted_shutdown.cancel(); + } + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn tier_delete_chunk_missing_child_with_journals_fails_closed() { + run_large_stack_async_test( + "tier-delete-chunk-missing-child", + tier_delete_chunk_missing_child_with_journals_fails_closed_case, + ); + } + + #[cfg(feature = "test-util")] + async fn tier_delete_chunk_missing_child_with_journals_fails_closed_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); + let temp_dir = tempfile::tempdir().expect("create missing-child store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "chunk-missing-child", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "CHUNK-MISSING-CHILD"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_remove_failure(true); + let bucket = "chunk-missing-child-bucket"; + let prefix = "prefix/"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("missing-child bucket should be created"); + for index in 0..2 { + let object = format!("{prefix}object-{index}.bin"); + let mut reader = PutObjReader::from_vec(vec![b'm' + index as u8; 1024 * 1024]); + let source = store + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("missing-child source should be written"); + store + .transition_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("missing-child source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("missing-child source should transition"); + } + + let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized); + let worker_store = store.clone(); + let worker = tokio::spawn(async move { + worker_store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("chunk delete should persist its authorization before corruption injection"); + worker.abort(); + let _ = worker.await; + drop(barrier); + + let records = store + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + TIER_DELETE_DISPATCH_MANIFEST_PREFIX, + None, + None, + 10, + false, + None, + false, + ) + .await + .expect("parent and child records should be listable"); + let child = records + .objects + .iter() + .find(|object| object.name.contains("/chunks/")) + .expect("the bound child manifest should exist") + .name + .clone(); + com::delete_config(store.clone(), &child) + .await + .expect("the test should remove only the child manifest"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 1); + + let error = store + .delete_object_with_tier_delete_journal( + bucket, + prefix, + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a missing child with retained journals must quarantine the parent"); + assert!( + error.to_string().contains("missing child with retained journals"), + "unexpected missing-child result: {error}" + ); + assert_eq!(backend.object_count().await, 2, "fail-closed inspection must not remove a remote version"); + shutdown.cancel(); + } + #[cfg(feature = "test-util")] #[test] #[serial_test::serial(storage_class_env)] @@ -11647,6 +12254,7 @@ mod tests { #[cfg(feature = "test-util")] async fn authorized_prefix_retry_replays_predecessor_before_newcomer_case() { + let _batch_limit = TierDeleteDispatchBatchLimitGuard::install(1); let temp_dir = tempfile::tempdir().expect("create authorized predecessor replay store dir"); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "authorized-prefix-predecessor-replay", &[4])) @@ -11659,8 +12267,8 @@ mod tests { .expect("authorized replay tier lease should resolve"); let bucket = "authorized-prefix-predecessor-replay-bucket"; let prefix = "prefix/"; - let predecessor = "prefix/predecessor.bin"; - let newcomer = "prefix/newcomer.bin"; + let predecessor = "prefix/000-predecessor.bin"; + let newcomer = "prefix/zzz-newcomer.bin"; store .make_bucket(bucket, &MakeBucketOptions::default()) .await diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index c3e150964..f231e738a 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -18,7 +18,9 @@ use crate::bucket::lifecycle::{ get_expiry_configs, tier_delete_journal::{ ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE, - complete_tier_delete_dispatch, prepare_tier_delete_dispatch, record_tier_delete_journal_backend_identity, + TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch, + inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch, + record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit, tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope, }, tier_sweeper::{ @@ -41,7 +43,10 @@ use crate::object_api::{ NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState, TierFreeVersionReceiptSink, }; -use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof; +use crate::services::notification_sys::{ + TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, + tier_delete_journal_topology_generation, +}; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::{ SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, @@ -52,6 +57,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _}, }; +use futures::StreamExt as _; use parking_lot::Mutex as ParkingMutex; use rustfs_filemeta::ObjectPartInfo; use rustfs_io_metrics::{ @@ -73,6 +79,7 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000; const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2; const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks"; const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50); +const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16; fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option { if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix { @@ -147,14 +154,96 @@ async fn prepare_prefix_tier_delete_journal_entries( Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await } +type TierDeleteLeaseReference = (String, Option); + +fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool { + truncated && limit_cancellation && matches!(error, Error::OperationCanceled) +} + +fn combine_tier_delete_walk_results(results: impl IntoIterator>) -> Result<()> { + let mut cancelled = false; + for result in results { + match result { + Ok(()) => {} + Err(Error::OperationCanceled) => cancelled = true, + Err(err) => return Err(err), + } + } + if cancelled { Err(Error::OperationCanceled) } else { Ok(()) } +} + +async fn acquire_prefix_tier_delete_reference_leases( + api: &Arc, + tier_references: &std::collections::HashSet, +) -> Result> { + let mut tier_references = tier_references.iter().cloned().collect::>(); + tier_references.sort_unstable(); + let mut leases = Vec::with_capacity(tier_references.len()); + for (tier_name, backend_identity) in tier_references { + let lease = match backend_identity { + Some(backend_identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity) + .await + } + None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, + } + .map_err(Error::other)?; + leases.push(lease); + } + Ok(leases) +} + +async fn acquire_prefix_tier_delete_leases(api: &Arc, entries: &[Jentry]) -> Result> { + let tier_references = entries + .iter() + .map(|entry| (entry.tier_name.clone(), entry.backend_identity)) + .collect::>(); + acquire_prefix_tier_delete_reference_leases(api, &tier_references).await +} + async fn prepare_prefix_tier_delete_journal_entries_inner( api: &Arc, bucket: &str, prefix: &str, opts: &ObjectOptions, ) -> Result { + let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) { + (false, false, None) + } else { + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + match Box::pin(inspect_tier_delete_chunk_parent( + Arc::clone(api), + bucket, + bucket_incarnation, + prefix, + bucket_fence, + )) + .await? + { + TierDeleteChunkParentInspection::NoParent => (false, false, None), + TierDeleteChunkParentInspection::LegacyManifest => (false, true, None), + TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)), + TierDeleteChunkParentInspection::Resume(dispatch) => { + let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?; + return Ok(PreparedPrefixTierDelete { + dispatch: Some(*dispatch), + chunk_parent_active: true, + chunk_parent_fleet_proof: None, + _leases: leases, + }); + } + TierDeleteChunkParentInspection::RetryRequired => { + return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch")); + } + } + }; let mut tier_references = std::collections::HashSet::<(String, Option)>::new(); let mut entries_by_name = std::collections::BTreeMap::new(); + let batch_limit = tier_delete_dispatch_batch_limit(); let logical_prefix = decode_dir_object(prefix); let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone()); let physical_sets = api @@ -164,12 +253,11 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( .collect::>(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100); let cancellation = tokio_util::sync::CancellationToken::new(); + let limit_cancellation = Arc::new(AtomicBool::new(false)); let walk_cancel = cancellation.clone(); let bucket_owned = bucket.to_string(); let prefix_owned = prefix.to_string(); let walk = async move { - use futures::StreamExt as _; - let results = futures::stream::iter(physical_sets.into_iter().map(|set| { let tx = tx.clone(); let cancellation = walk_cancel.clone(); @@ -199,14 +287,28 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( .collect::>() .await; drop(tx); - results.into_iter().collect::>>().map(|_| ()) + combine_tier_delete_walk_results(results) }; + let collect_limit_cancellation = limit_cancellation.clone(); let collect = async { + let mut truncated = false; while let Some(result) = rx.recv().await { if let Some(err) = result.err { + // Once limit + 1 has been observed this request can authorize + // only the exact retained batch; it cannot infer prefix + // absence or run the raw delete. Drain only the explicit + // cancellation fallout; a real walker error must still fail + // the request even when another set reached the limit first. + if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err) + { + continue; + } cancellation.cancel(); return Err(err); } + if truncated { + continue; + } let Some(source) = result.item else { continue; }; @@ -223,62 +325,125 @@ async fn prepare_prefix_tier_delete_journal_entries_inner( "recursive prefix delete cannot discard an existing tier free-version cleanup obligation", )); } - if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE { + let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE { let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?; - tier_references.insert((source.transitioned_object.tier.clone(), backend_identity)); - } + Some((source.transitioned_object.tier.clone(), backend_identity)) + } else { + None + }; if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? { - entries_by_name - .entry(tier_delete_journal_object_name(&entry)) - .or_insert(entry); + let name = tier_delete_journal_object_name(&entry); + let at_limit = entries_by_name.len() == batch_limit; + match entries_by_name.entry(name) { + std::collections::btree_map::Entry::Occupied(_) => {} + std::collections::btree_map::Entry::Vacant(_) if at_limit => { + truncated = true; + collect_limit_cancellation.store(true, Ordering::Release); + cancellation.cancel(); + } + std::collections::btree_map::Entry::Vacant(slot) => { + if let Some(tier_reference) = tier_reference { + tier_references.insert(tier_reference); + } + slot.insert(entry); + } + } + } else if let Some(tier_reference) = tier_reference { + tier_references.insert(tier_reference); } } - Ok(()) + Ok(truncated) }; let (walk_result, collect_result) = tokio::join!(walk, collect); - collect_result?; - walk_result?; - let entries = entries_by_name.into_values().collect::>(); - - let mut tier_references = tier_references.into_iter().collect::>(); - tier_references.sort_unstable(); - let mut leases = Vec::with_capacity(tier_references.len()); - for (tier_name, backend_identity) in tier_references { - let lease = match backend_identity { - Some(backend_identity) => { - TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity) - .await - } - None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await, - } - .map_err(Error::other)?; - leases.push(lease); + let truncated = collect_result?; + // A truncated walk normally reports OperationCanceled from the physical + // walkers. That cancellation is expected; any other result is a genuine + // scan failure and cannot be hidden by the bounded batch. + if let Err(err) = walk_result + && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err) + { + return Err(err); } + let entries = entries_by_name.into_values().collect::>(); + let mut leased_tier_references = tier_references; + let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?; if entries.is_empty() { + let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() { + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?; + if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology { + return Err(Error::other("tier delete chunk parent topology changed during final source scan")); + } + Some(fleet_proof) + } else { + None + }; return Ok(PreparedPrefixTierDelete { dispatch: None, + chunk_parent_active, + chunk_parent_fleet_proof, _leases: leases, }); } let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; - let fleet_proof = acquire_tier_delete_journal_fleet_proof() - .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?; let bucket_fence = opts .bucket_lifecycle_lock_fence .as_ref() .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; - let dispatch = + let fleet_proof = acquire_tier_delete_journal_fleet_proof() + .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?; + if chunk_parent_topology_generation + .as_deref() + .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected) + { + return Err(Error::other("tier delete chunk parent topology changed during source scan")); + } + let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) { + Box::pin(prepare_tier_delete_chunk_dispatch( + Arc::clone(api), + bucket, + bucket_incarnation, + prefix, + entries, + truncated && !chunk_parent_active, + fleet_proof, + bucket_fence, + )) + .await? + } else if legacy_manifest_active && truncated { + resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence) + .await? + } else { prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence) - .await?; + .await? + }; + // A resumed legacy authorization may own predecessors that are absent + // from this bounded scan. Pin every backend generation in the actual + // permit before any local mutation, while avoiding duplicate leases for + // entries already covered by the scan. + let additional_tier_references = dispatch + .entries()? + .iter() + .map(|entry| (entry.tier_name.clone(), entry.backend_identity)) + .filter(|reference| leased_tier_references.insert(reference.clone())) + .collect::>(); + leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?); + if legacy_manifest_active && truncated { + dispatch.require_exact_predecessor_replay(); + } Ok(PreparedPrefixTierDelete { dispatch: Some(dispatch), + chunk_parent_active: chunk_parent_active || truncated, + chunk_parent_fleet_proof: None, _leases: leases, }) } struct PreparedPrefixTierDelete { dispatch: Option, + chunk_parent_active: bool, + chunk_parent_fleet_proof: Option, _leases: Vec, } @@ -365,12 +530,16 @@ async fn delete_prefix_with_tier_delete_journal( let Some(api) = tier_journal_api else { return store.delete_prefix(bucket, object, opts).await; }; - let PreparedPrefixTierDelete { dispatch, _leases } = - prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?; + let PreparedPrefixTierDelete { + dispatch, + chunk_parent_active, + chunk_parent_fleet_proof, + _leases, + } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?; let Some(dispatch) = dispatch else { - // There is no remote-cleanup candidate, so no v6 manifest or fleet - // proof is required. Keep any compatibility-path tier leases alive - // until the local delete has committed. + // There is no new remote-cleanup candidate. Keep compatibility-path + // tier leases and, for a chunked final pass, the matching parent fleet + // proof alive until local deletion and parent completion both commit. let _tier_leases = _leases; let mut operation_opts = opts.clone(); // `tier_delete_journal_api` means a v6 dispatch authorization must be @@ -379,7 +548,42 @@ async fn delete_prefix_with_tier_delete_journal( // transitioned metadata retains its FreeVersion fallback. operation_opts.tier_delete_journal_api = None; operation_opts.tier_delete_dispatch_authorization = None; - return store.delete_prefix(bucket, object, &operation_opts).await; + let parent_fleet_proof = if chunk_parent_active { + Some( + chunk_parent_fleet_proof + .as_ref() + .filter(|proof| tier_delete_journal_fleet_proof_matches(proof)) + .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?, + ) + } else { + None + }; + store.delete_prefix(bucket, object, &operation_opts).await?; + if let Some(parent_fleet_proof) = parent_fleet_proof { + #[cfg(all(test, feature = "test-util"))] + crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause( + crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted, + ) + .await; + let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; + let bucket_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?; + if !Box::pin(complete_tier_delete_chunk_parent( + Arc::clone(api), + bucket, + bucket_incarnation, + object, + bucket_fence, + parent_fleet_proof, + )) + .await? + { + return Err(Error::other("tier delete chunk parent disappeared after final local deletion")); + } + } + return Ok(()); }; let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; let bucket_fence = opts @@ -396,9 +600,19 @@ async fn delete_prefix_with_tier_delete_journal( // Keep every backend generation lease until the whole local operation has // either committed its journal set or returned an ambiguous mutation. let _tier_leases = _leases; - if active.predecessor_replay_required() { + if active.predecessor_replay_required() || active.is_chunked() { replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?; + #[cfg(all(test, feature = "test-util"))] + if active.is_chunked() { + crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause( + crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted, + ) + .await; + } complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?; + if active.is_chunked() { + return Err(Error::other("tier delete chunk completed; retry the next durable batch")); + } return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch")); } let result = store.delete_prefix(bucket, object, &operation_opts).await; @@ -443,7 +657,7 @@ async fn replay_authorized_tier_delete_sources( let authorization = active.authorization(); authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?; - let mut source_objects = std::collections::BTreeSet::new(); + let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len()); for entry in active.entries() { let source = entry .source @@ -453,61 +667,130 @@ async fn replay_authorized_tier_delete_sources( if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) { return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope")); } - source_objects.insert(source.object.clone()); + source_objects.insert(source.object.as_str()); } - let mut deleted = 0; - for object in source_objects { - if bucket_fence.is_lock_lost() { - return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + if let Some(scope) = publication_scope { + if scope.state() == ScannerPublicationCommitState::Admitted { + scope + .try_begin() + .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?; } - let encoded_object = encode_dir_object(&object); - let guards = if opts.delete_prefix_object { - store - .acquire_remaining_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) - .await? - } else { - store - .acquire_all_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object) - .await? - }; - authorization.ensure_current(bucket, bucket_incarnation, prefix)?; - if let Some(scope) = publication_scope { - if scope.state() == ScannerPublicationCommitState::Admitted { - scope - .try_begin() - .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?; - } - if !scope.can_commit() { - let _ = scope.mark_indeterminate(); - return Err(StorageError::OperationCanceled); - } - } - let mut replay_opts = opts.clone(); - replay_opts.no_lock = true; - replay_opts.delete_prefix = false; - replay_opts.delete_prefix_object = false; - for guard in &guards { - guard.add_namespace_lock_fence(&mut replay_opts); - } - for pool in &store.pools { - for set in &pool.disk_set { - authorization.ensure_current(bucket, bucket_incarnation, prefix)?; - deleted += set - .replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts) - .await?; - } - } - if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { - return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + if !scope.can_commit() { + let _ = scope.mark_indeterminate(); + return Err(StorageError::OperationCanceled); } } - if let Some(scope) = publication_scope { - let _ = scope.mark_committed(); + + let stopped = Arc::new(AtomicBool::new(false)); + // The caller holds bucket lifecycle WRITE. Each bounded future acquires + // only one logical object's physical lock set and releases it before + // completion; no future nests locks for two object keys. + let make_replay = |object: String| { + let stopped = stopped.clone(); + let authorization = authorization.clone(); + async move { + if stopped.load(Ordering::Acquire) { + return Ok::<_, Error>(0usize); + } + let result = async { + if bucket_fence.is_lock_lost() { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + } + let encoded_object = encode_dir_object(&object); + let guards = if opts.delete_prefix_object { + store + .acquire_remaining_physical_object_write_locks( + "tier_delete_dispatch_predecessor_replay", + bucket, + &encoded_object, + ) + .await? + } else { + store + .acquire_all_physical_object_write_locks( + "tier_delete_dispatch_predecessor_replay", + bucket, + &encoded_object, + ) + .await? + }; + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + if publication_scope.is_some_and(|scope| !scope.can_commit()) { + return Err(StorageError::OperationCanceled); + } + let mut replay_opts = opts.clone(); + replay_opts.no_lock = true; + replay_opts.delete_prefix = false; + replay_opts.delete_prefix_object = false; + for guard in &guards { + guard.add_namespace_lock_fence(&mut replay_opts); + } + let mut deleted = 0usize; + for pool in &store.pools { + for set in &pool.disk_set { + authorization.ensure_current(bucket, bucket_incarnation, prefix)?; + deleted = deleted + .checked_add( + set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts) + .await?, + ) + .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?; + } + } + if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { + return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay")); + } + Ok(deleted) + } + .await; + if result.is_err() { + stopped.store(true, Ordering::Release); + } + result + } + }; + let mut objects = source_objects.into_iter(); + let mut replays = futures::stream::FuturesUnordered::new(); + for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY { + let Some(object) = objects.next().map(ToOwned::to_owned) else { + break; + }; + replays.push(make_replay(object)); + } + let mut deleted = 0usize; + let mut first_error = None; + while let Some(result) = replays.next().await { + match result { + Ok(count) => { + deleted = deleted + .checked_add(count) + .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?; + } + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + if first_error.is_none() + && !stopped.load(Ordering::Acquire) + && let Some(object) = objects.next().map(ToOwned::to_owned) + { + replays.push(make_replay(object)); + } } if deleted > 0 { super::list_objects::observe_list_objects_mutation(store, bucket).await; } + if let Some(err) = first_error { + if publication_scope.is_some_and(|scope| !scope.can_commit()) + && let Some(scope) = publication_scope + { + let _ = scope.mark_indeterminate(); + } + return Err(err); + } + if let Some(scope) = publication_scope { + let _ = scope.mark_committed(); + } Ok(()) } @@ -6518,6 +6801,28 @@ mod tests { ); } + #[test] + fn tier_delete_walk_only_accepts_explicit_limit_cancellation() { + let cancelled = Error::OperationCanceled; + assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled)); + assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed"))); + } + + #[test] + fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() { + let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)]) + .expect_err("a real walk error must not be hidden by earlier cancellation"); + assert_eq!(err, StorageError::FileAccessDenied); + + assert_eq!( + combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)]) + .expect_err("cancellation must remain visible when there is no real error"), + Error::OperationCanceled + ); + } + impl Drop for BodyCacheHookGuard { fn drop(&mut self) { clear_get_object_body_cache_hook(); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 1e10a226c..6472481a0 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,7 @@ ## Open Items - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. +- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources. diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index a1b3c9fb3..ccf3d9995 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -30,7 +30,7 @@ These are approved-target invariants. A protocol's explicitly labeled current ex | Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence | | Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Current recovery finds the transaction's logical bucket/object/version and checks `TRANSITION_COMPLETE` plus the same remote object, tier, and remote version. It does not compare the recorded data directory, modification time, size, or ETag; the approved target adds that full source comparison | | An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version | -| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable dispatch manifest | Yes, but only after manifest completion and all-pool absence proof | `DispatchAuthorized`, local destructive mutation, every journal `Committed`, then manifest `Completed` | +| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion | | Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners | An old journal and a free-version can coexist during compatibility recovery. That coexistence is evidence of multiple possible owners, not permission to choose one: the journal path must retain its record until the version-specific recovery rule proves which owner is authoritative. @@ -50,11 +50,12 @@ All keys below are objects in the internal metadata bucket. The table gives the | Manual worker result | `rustfs-manual-transition-worker-result-v1` | `ilm/manual-transition/results///.json` | The worker persists it after an actual result; no current GC owner | Immutable job/task key and outcome/reason | Append-only create with `If-None-Match: *` and maximum parity | | Legacy tier-delete journal | Versions 1 through 5 | `ilm/tier-delete-journal/.json` | The deleting path creates it; version-specific journal recovery cleans it | Remote tuple; v2 adds backend identity, v3 exact version, v4 version state, v5 stable source and transaction state | v5 state changes use ETag CAS; v3/v4 recovery rereads and conditionally cleans. Initial legacy-compatible writes can still be unconditional | | Sole-owner tier-delete journal | Version 6 | `ilm/tier-delete-journal-v6//.json` | The manifest coordinator creates/dispatches it; the journal worker deletes the remote object and cleans the record | Exact remote/source/backend identity plus manifest/operation/topology binding; mutable state `Prepared`/`Dispatched`/`Committed` | Create-only and fenced ETag CAS. Record cleanup writes a terminal receipt first only while a decommission run is active; ordinary recovery without one conditionally deletes the exact ETag directly | -| Tier-delete dispatch manifest | Version 1 | `ilm/tier-delete-dispatch-manifests/.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback | +| Tier-delete dispatch manifest | Version 1 | Single dispatch: `ilm/tier-delete-dispatch-manifests/.json`; chunk child: `ilm/tier-delete-dispatch-manifests/chunks//.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback. A child cannot authorize local mutation without the exact active parent binding | +| Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion | | Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts////.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity | | Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal | -`durable_namespace.rs` registers exactly the two tier-journal namespaces, dispatch manifest, transaction, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object. +`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object. ## Durable fences and write primitives @@ -89,8 +90,8 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n | Path | Current acquisition order | Operations allowed while held | Operations forbidden while held | |---|---|---|---| | Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation | -| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; synthetic manifest-operation WRITE lock | Build and write the immutable journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes | -| v6 manifest recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; synthetic manifest-operation WRITE lock | Read/write manifest and journal metadata, verify exact set/digest, authorize, converge, or roll back local records | Remote tier DELETE; per-object worker cleanup; rollback after authorization | +| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition | +| v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery | | v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version | | v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version | | Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes | @@ -242,7 +243,9 @@ The lease interval is 60 seconds. CAS behavior is phase-specific: cancellation t | v5 | Adds stable source identity and transaction state. Recovery proves source/free-version presence across physical sets before deciding abort, retain, or commit | | v6 | Sole-owner record bound to immutable operation/manifest/topology. It is the only new journal format for destructive prefix dispatch | -The v1 manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are: +For a complete source set at or below 200,000 journals, the byte-compatible v1 single manifest remains at the deterministic scope-digest root. For a larger source set, that same root instead contains a strict `chunked_parent` sentinel, and each bounded child uses the unchanged v1 manifest payload at `chunks//.json`. A pre-chunking reader rejects the parent schema and the non-root child path, so it cannot start a competing single dispatch while chunking is active. + +A v1 single/child manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are: ```text Preparing -> DispatchAuthorized -> Completed @@ -251,6 +254,16 @@ Preparing -> DispatchAuthorized -> Completed The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinator owns the whole `Prepared` set and is the only actor that may roll it back or complete the manifest. A per-journal worker cannot remove one prepared member. +The chunk parent stores only monotonic O(1) progress and binds at most one child: + +```text +Active(no child) -> Active(bound Preparing child) +Active(bound Completed child) -> Active(no child, next sequence/count) +Active(no child, final empty rescan and local delete complete) -> Completed +``` + +Child creation is ordered `Preparing` child create, parent binding CAS, journal preparation/dispatch, then child `DispatchAuthorized`. One request exactly replays one bounded child under source-object locks, commits every child journal, marks the child `Completed`, advances the parent, and returns retry-required. A successor request rescans from the prefix start; no listing cursor crosses bucket-lock lifetimes. New or changed source identities are therefore admitted only by a fresh child. Final success requires an error-free, non-truncated scan with no v6 candidate, local prefix deletion under the same bucket fence, and the parent `Completed` CAS. + ### Journal recovery decisions | Record/state and evidence | Unique current owner | Current recovery decision | Remote DELETE admission | @@ -278,11 +291,24 @@ The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinato | `Completed` | Journal workers own member cleanup; coordinator owns final manifest cleanup | Wait for all member records to disappear, then conditionally delete manifest | Journal workers are the remote-delete owners | | Missing member, set/digest mismatch, wrong incarnation/topology, corrupt state, scan ambiguity, or cancellation | No actor acquires new destructive authority | Retain | Fail closed; an authorized operation never rolls back | -Manifest preparation is bounded by 200,000 journals and a 32 MiB record. Journal recovery scans bounded batches with per-entry timeouts and limited concurrency. Those are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely. +### Chunk-parent recovery decisions + +| Parent/child state and evidence | Unique current owner | Current recovery decision | Authority | +|---|---|---|---| +| Active parent with no child | A later prefix-delete retry under bucket WRITE | Retain the parent and rescan from the prefix start | No local or remote deletion | +| Active parent with exact bound `Preparing`/`Aborting`/`Aborted` child | Child manifest coordinator | Retain parent while child recovery rolls back and removes the child | Never authorize or advance that child | +| Active parent with exact bound `DispatchAuthorized` child | The bound child permit or journal recovery | Resume exact-source replay on request; otherwise retain until all journals become `Committed` and the child becomes `Completed` | Only the exact parent-bound child may authorize local replay | +| Active parent with exact bound `Completed` child | Parent coordinator | CAS the next sequence/count and clear the binding | Parent progress only; remote DELETE remains owned by committed child journals | +| Bound child missing and its exact operation journal namespace is non-empty or unreadable | No actor can prove safe abandonment | Retain and fail closed | Never clear the binding | +| Bound child missing and its exact operation journal namespace is proven empty | Parent coordinator | CAS-clear the stale binding and retry from the prefix start | No deletion; the fresh scan reconstructs any remaining source work | +| Completed parent with no active child | Parent recovery | Record terminal decommission evidence when applicable, then conditionally delete the exact parent ETag | Metadata cleanup only | +| Parent identity, child binding, topology, incarnation, sequence/count, CAS generation, or fence mismatches | No actor acquires progress authority | Retain | Fail closed | + +Single and child manifest preparation is bounded by 200,000 journals and a 32 MiB record. On the first unique candidate beyond the bound, the physical walks are cancelled and only the retained exact batch can proceed; cancellation fallout is not absence proof. The parent never accumulates child names, and exact local replay uses bounded concurrency. Journal and manifest recovery retain their existing bounded pages, per-entry timeouts, and concurrency. These are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely. ### Approved target and open design -- New destructive prefix paths use only v6 plus a manifest. No new v1-v5 sole-owner records may be created. +- New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created. - Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE. - Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous. - **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof. @@ -309,9 +335,9 @@ How historical objects without RustFS transition-version-state can be upgraded s ### Current contract -Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads. +Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads. -The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. +The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target. @@ -371,7 +397,7 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr | Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled | | Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof | | Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization | -| Journal v6 and dispatch manifest v1 | v6-aware writers/readers require immutable manifest membership and topology; v5-and-older readers reject and retain v6 | Gate v6 writers on fleet capability. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation | +| Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation | | Decommission receipt v2 and expected manifest v1 | Current decommission readers validate exact schema/checksum/path/checkpoint and fail completion on unknown input | No ignore path. Mixed-version decommission must not complete unless every participant preserves the registered durable namespace; broader downgrade negotiation is open | ## Reconcile, observability, and retention