diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index f1ead7972..9af74f2a3 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -197,6 +197,27 @@ pub const DEFAULT_POOL_META_V3_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_POOL_META_V3_WRITE); const _: () = assert!(!DEFAULT_POOL_META_V3_FLEET_CONFIRMED); +/// Maximum unpacked size accepted for one Snowball archive member. +/// +/// The value is expressed in bytes. Invalid values use the default, while +/// valid values are clamped to [`MAX_SNOWBALL_ENTRY_BYTES`]. +pub const ENV_SNOWBALL_MAX_ENTRY_BYTES: &str = "RUSTFS_SNOWBALL_MAX_ENTRY_BYTES"; +pub const DEFAULT_SNOWBALL_MAX_ENTRY_BYTES: u64 = 1024 * 1024 * 1024; +pub const MAX_SNOWBALL_ENTRY_BYTES: u64 = 1024 * DEFAULT_SNOWBALL_MAX_ENTRY_BYTES; + +/// Maximum cumulative unpacked object bytes accepted from one Snowball +/// archive request. +/// +/// This does not include tar headers or bounded PAX metadata. The value is +/// expressed in bytes and is clamped to +/// [`MAX_SNOWBALL_UNPACKED_BYTES`]. +pub const ENV_SNOWBALL_MAX_UNPACKED_BYTES: &str = "RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES"; +pub const DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES: u64 = 10 * 1024 * 1024 * 1024; +pub const MAX_SNOWBALL_UNPACKED_BYTES: u64 = 10 * 1024 * DEFAULT_SNOWBALL_MAX_ENTRY_BYTES; + +const _: () = assert!(DEFAULT_SNOWBALL_MAX_ENTRY_BYTES <= MAX_SNOWBALL_ENTRY_BYTES); +const _: () = assert!(DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES <= MAX_SNOWBALL_UNPACKED_BYTES); + // ============================================================================= // Concurrent Request Fix - Timeout and Backpressure Configuration // ============================================================================= @@ -820,4 +841,10 @@ mod remote_version_state_tests { assert_eq!(super::ENV_POOL_META_V3_WRITE, "RUSTFS_POOL_META_V3_WRITE"); assert_eq!(super::ENV_POOL_META_V3_FLEET_CONFIRMED, "RUSTFS_POOL_META_V3_FLEET_CONFIRMED"); } + + #[test] + fn snowball_limit_environment_names_are_stable() { + assert_eq!(super::ENV_SNOWBALL_MAX_ENTRY_BYTES, "RUSTFS_SNOWBALL_MAX_ENTRY_BYTES"); + assert_eq!(super::ENV_SNOWBALL_MAX_UNPACKED_BYTES, "RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES"); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 2f519ab55..3840199c2 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -1468,7 +1468,7 @@ async fn save_decommission_manifest_checkpoint_if_match( } let write_data = next_data.to_vec(); let write = api - .run_decommission_capacity_temporary_mutation_with_capacity_lease( + .run_decommission_capacity_non_growing_replacement_with_capacity_lease( target.target_pool_index, Some(target.capacity_owner), Some(next_data.len()), diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 46a98ef4b..8e2c54bee 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -75,7 +75,7 @@ use http::HeaderMap; #[cfg(test)] use rmp_serde::Deserializer; use rmp_serde::Serializer; -use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_heal_contracts::heal_channel::HealOpts; use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use rustfs_utils::path::{ @@ -106,6 +106,8 @@ const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket"; const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry"; const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof"; const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit"; +const DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED: &str = + "decommission target fence fleet capability proof expired before reservation commit"; const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object"; const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight"; const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup"; @@ -120,12 +122,19 @@ const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX]; -const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = 1; +const DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION: u16 = 1; +const DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION: u16 = 2; +#[cfg(test)] +const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION; const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1; const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10); const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled"; const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed"; const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed"; +pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target"; +const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250); +const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool "; +const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy"; const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total"; const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes"; const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes"; @@ -196,12 +205,16 @@ fn pool_meta_v3_writer_enabled() -> bool { ) } +fn decommission_capacity_writer_supported_for(version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool) -> bool { + matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled +} + fn ensure_decommission_ledger_persistence_supported_for( version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool, ) -> Result<()> { - if matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled { + if decommission_capacity_writer_supported_for(version, v2_writer_enabled, v3_writer_enabled) { return Ok(()); } @@ -209,9 +222,11 @@ fn ensure_decommission_ledger_persistence_supported_for( "decommission".to_string(), "pool-metadata-version".to_string(), format!( - "durable unresolved-entry recovery requires pool metadata V2 or V3; enable both {} and {} only after every reader and writer supports V2", + "durable unresolved-entry recovery requires pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format", rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, + rustfs_config::ENV_POOL_META_V3_WRITE, + rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, ), )) } @@ -908,7 +923,7 @@ fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> } fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool { - matches!(meta.version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || pool_meta_v2_writer_enabled() + decommission_capacity_writer_supported_for(meta.version, pool_meta_v2_writer_enabled(), pool_meta_v3_writer_enabled()) } fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> { @@ -916,9 +931,11 @@ fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> return Ok(()); } Err(Error::DecommissionCapacity(format!( - "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable both {} and {} only after every reader and writer supports V2", + "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format", rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, + rustfs_config::ENV_POOL_META_V3_WRITE, + rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, ))) } @@ -944,11 +961,105 @@ fn is_decommission_capacity_intent_conflict(err: &Error) -> bool { || err.to_string().contains("unresolved target capacity intent") } +fn decommission_capacity_target_gate_busy_index(err: &Error) -> Option { + if let Error::DecommissionCapacityBlocked { message } = err { + return message + .strip_prefix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX)? + .strip_suffix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX)? + .parse() + .ok(); + } + data_movement::data_movement_stage_source(err).and_then(decommission_capacity_target_gate_busy_index) +} + +fn is_decommission_capacity_target_gate_busy(err: &Error) -> bool { + decommission_capacity_target_gate_busy_index(err).is_some() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecommissionCapacityRetryKind { + IntentConflict, +} + +fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize) -> Option { + (intent_conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS + && is_decommission_capacity_intent_conflict(err)) + .then_some(DecommissionCapacityRetryKind::IntentConflict) +} + +fn ensure_decommission_capacity_target_fence( + guard: &rustfs_lock::NamespaceLockGuard, + target_pool_index: usize, + phase: &str, +) -> Result<()> { + if guard.is_lock_lost() { + return Err(Error::DecommissionCapacity(format!( + "target pool {target_pool_index} capacity mutation fence was lost during {phase}" + ))); + } + Ok(()) +} + +fn ensure_decommission_capacity_mutation_intent_current( + meta: &PoolMeta, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + expected_target_physical_bytes: usize, + mutation_id: uuid::Uuid, + temporary_release: bool, + model_version: u16, +) -> Result<()> { + let reservation = meta + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| { + // Admission validated the owner nonce before persisting this + // intent. The target lock is the mutation fence after that point, + // so a concurrent lease renewal may rotate the nonce without + // invalidating the already-durable intent. + reservation.active() + && reservation.source_pool_index == owner.source_pool_index + && reservation.operation_id == owner.operation_id + && reservation.generation == owner.generation + && reservation.model_version == model_version + }) + .ok_or_else(|| { + decommission_capacity_blocked_error("decommission target mutation owner changed before capacity finalize") + })?; + let target = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("decommission target allocation changed before capacity finalize"))?; + + if temporary_release { + // Cleanup is authorized by the persisted owner and target identity. + // Its exact temporary or pending record may already be absent after a + // prior successful finalize, so attribution is decided from the + // physical cleanup result below instead of rejecting the retry here. + return Ok(()); + } + + if expected_target_physical_bytes == 0 + || (target.pending_mutation_id == Some(mutation_id) && target.pending_physical_bytes >= expected_target_physical_bytes) + { + return Ok(()); + } + Err(decommission_capacity_blocked_error( + "pending capacity intent belongs to another mutation before capacity finalize", + )) +} + fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> { let Some(reservation) = reservation else { return Ok(()); }; - if reservation.model_version != DECOMMISSION_CAPACITY_MODEL_VERSION { + if !matches!( + reservation.model_version, + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ) { return Err(Error::DecommissionCapacity(format!( "pool metadata load failed: unsupported decommission capacity model version {}", reservation.model_version @@ -1009,7 +1120,7 @@ fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionC let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len()); let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| { if mutation.mutation_id.is_nil() - || mutation.physical_bytes == 0 + || (mutation.physical_bytes == 0 && reservation.model_version != DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) || !temporary_mutation_ids.insert(mutation.mutation_id) { return Err(Error::other( @@ -1076,6 +1187,70 @@ fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionC Ok(()) } +fn active_decommission_capacity_model(meta: &PoolMeta) -> Result> { + let mut active_model = None; + for reservation in meta + .pools + .iter() + .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref()) + .filter(|reservation| reservation.active()) + { + if !matches!( + reservation.model_version, + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ) { + return Err(Error::DecommissionCapacity(format!( + "pool metadata load failed: unsupported active decommission capacity model version {}", + reservation.model_version + ))); + } + match active_model { + Some(model_version) if model_version != reservation.model_version => { + return Err(Error::DecommissionCapacity( + "pool metadata load failed: active decommission capacity reservations use mixed lock models".to_string(), + )); + } + Some(_) => {} + None => active_model = Some(reservation.model_version), + } + } + Ok(active_model) +} + +fn validate_decommission_capacity_model_cohort(meta: &PoolMeta) -> Result<()> { + active_decommission_capacity_model(meta).map(|_| ()) +} + +fn select_decommission_capacity_model(meta: &PoolMeta, target_fence_proof_available: bool) -> Result { + match active_decommission_capacity_model(meta)? { + Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION), + Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) if target_fence_proof_available => { + Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) + } + Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => Err(Error::DecommissionCapacity( + "failed to start decommission: an active per-target capacity cohort requires a live all-v4 fleet proof".to_string(), + )), + Some(version) => Err(Error::DecommissionCapacity(format!( + "failed to start decommission: unsupported active capacity model version {version}" + ))), + None if target_fence_proof_available => Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION), + None => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION), + } +} + +fn ensure_decommission_target_fence_fleet_proof( + proof: Option<&crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>, + required: bool, +) -> Result<()> { + if !required { + return Ok(()); + } + if proof.is_some_and(crate::services::notification_sys::decommission_target_fence_fleet_proof_matches) { + return Ok(()); + } + Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED)) +} + fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool { let Some(reservation) = info.capacity_reservation.as_mut() else { return false; @@ -1135,6 +1310,7 @@ fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet { .collect() } +#[cfg(test)] fn build_decommission_capacity_reservation( source: DecommissionPoolCapacityInfo, target_layout: DecommissionErasureLayout, @@ -1142,11 +1318,37 @@ fn build_decommission_capacity_reservation( generation: u64, now: OffsetDateTime, ) -> Result { + build_decommission_capacity_reservation_with_model( + source, + target_layout, + operation_id, + generation, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) +} + +fn build_decommission_capacity_reservation_with_model( + source: DecommissionPoolCapacityInfo, + target_layout: DecommissionErasureLayout, + operation_id: uuid::Uuid, + generation: u64, + now: OffsetDateTime, + model_version: u16, +) -> Result { + if !matches!( + model_version, + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ) { + return Err(Error::DecommissionCapacity(format!( + "failed to build decommission capacity reservation: unsupported lock model {model_version}" + ))); + } let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?; let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?; let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES); Ok(DecommissionCapacityReservation { - model_version: DECOMMISSION_CAPACITY_MODEL_VERSION, + model_version, operation_id, generation, owner_nonce: uuid::Uuid::new_v4(), @@ -1182,7 +1384,9 @@ fn reserve_decommission_start_target_capacity( operation_id: uuid::Uuid, generation: u64, now: OffsetDateTime, + model_version: u16, ) -> Result<()> { + validate_decommission_capacity_model_cohort(meta)?; let active_sources = active_decommission_source_indices(meta); let targets = capacity_infos .iter() @@ -1221,7 +1425,14 @@ fn reserve_decommission_start_target_capacity( .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?; let new_reservation = requested.contains(&source_index); let mut reservation = if new_reservation { - build_decommission_capacity_reservation(source, target_layout, operation_id, generation, now)? + build_decommission_capacity_reservation_with_model( + source, + target_layout, + operation_id, + generation, + now, + model_version, + )? } else { info.capacity_reservation.clone().ok_or_else(|| { Error::DecommissionCapacity(format!( @@ -1334,6 +1545,7 @@ fn reserve_decommission_start_target_capacity( if meta.version != POOL_META_GENERATION_VERSION { meta.version = POOL_META_VERSION; } + validate_decommission_capacity_model_cohort(meta)?; Ok(()) } @@ -1341,6 +1553,7 @@ fn ensure_decommission_start_target_capacity( meta: &PoolMeta, indices: &[usize], capacity_infos: &[DecommissionPoolCapacityInfo], + target_fence_proof_available: bool, ) -> Result<()> { let generation = next_decommission_capacity_generation(meta)?; let mut projected = meta.clone(); @@ -1356,6 +1569,7 @@ fn ensure_decommission_start_target_capacity( projected.queue_decommission(idx, capacity.space)?; } } + let model_version = select_decommission_capacity_model(&projected, target_fence_proof_available)?; reserve_decommission_start_target_capacity( &mut projected, indices, @@ -1363,6 +1577,7 @@ fn ensure_decommission_start_target_capacity( uuid::Uuid::new_v4(), generation, OffsetDateTime::now_utc(), + model_version, ) } @@ -1370,6 +1585,7 @@ fn recover_decommission_capacity_reservations( meta: &mut PoolMeta, capacity_infos: &[DecommissionPoolCapacityInfo], now: OffsetDateTime, + target_fence_proof_available: bool, ) -> Result> { ensure_decommission_capacity_writer_supported(meta)?; let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::>(); @@ -1385,8 +1601,21 @@ fn recover_decommission_capacity_reservations( .is_none_or(|reservation| !reservation.active()) }) .collect::>(); + if missing_indices.is_empty() { + validate_decommission_capacity_model_cohort(meta)?; + return Ok(active_indices); + } let generation = next_decommission_capacity_generation(meta)?; - reserve_decommission_start_target_capacity(meta, &missing_indices, capacity_infos, uuid::Uuid::new_v4(), generation, now)?; + let model_version = select_decommission_capacity_model(meta, target_fence_proof_available)?; + reserve_decommission_start_target_capacity( + meta, + &missing_indices, + capacity_infos, + uuid::Uuid::new_v4(), + generation, + now, + model_version, + )?; for idx in missing_indices { if let Some(reservation) = meta .pools @@ -1692,21 +1921,57 @@ fn release_decommission_target_temporary_mutation( target: &mut DecommissionCapacityTarget, mutation_id: uuid::Uuid, maximum_physical_bytes: usize, -) -> usize { +) -> (usize, bool) { let Some(index) = target .temporary_mutations .iter() .position(|mutation| mutation.mutation_id == mutation_id) else { - return 0; + return (0, false); }; let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes); target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released); - if target.temporary_mutations[index].physical_bytes == 0 { + let removed = target.temporary_mutations[index].physical_bytes == 0; + if removed { target.temporary_mutations.remove(index); } target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); - released + (released, released > 0 || removed) +} + +fn settle_decommission_target_non_growing_replacement( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + mutation_id: uuid::Uuid, + now: OffsetDateTime, +) -> Result { + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("settle non-growing target replacement"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target replacement"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target replacement"))?; + let (released, changed) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX); + reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released); + if changed { + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + } + Ok(changed) } struct DecommissionTargetConsumption { @@ -1756,7 +2021,7 @@ fn record_decommission_target_consumption( target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed); target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes); let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty(); - let released_inflight = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX); + let (released_inflight, _) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX); let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations { let released = target.inflight_physical_bytes.min(consumed); target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); @@ -1790,7 +2055,7 @@ fn record_decommission_target_inflight( observed_physical_bytes: usize, mutation_id: uuid::Uuid, now: OffsetDateTime, -) -> Result<()> { +) -> Result { let pool_count = meta.pools.len(); let pool = meta .pools @@ -1810,20 +2075,25 @@ fn record_decommission_target_inflight( .iter_mut() .find(|target| target.pool_index == target_pool_index) .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?; - if observed_physical_bytes > 0 { - if let Some(mutation) = target - .temporary_mutations - .iter_mut() - .find(|mutation| mutation.mutation_id == mutation_id) - { - mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes); - } else { - target.temporary_mutations.push(DecommissionCapacityTemporaryMutation { - mutation_id, - physical_bytes: observed_physical_bytes, - }); - } - } + let ledger_changed = if let Some(mutation) = target + .temporary_mutations + .iter_mut() + .find(|mutation| mutation.mutation_id == mutation_id) + { + mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes); + observed_physical_bytes > 0 + } else if observed_physical_bytes > 0 || reservation.model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION { + // A v2 zero-byte record is a durable discovery marker. It keeps + // restart cleanup scoped to mutations that actually staged an MPU + // without charging capacity when statfs observed no physical delta. + target.temporary_mutations.push(DecommissionCapacityTemporaryMutation { + mutation_id, + physical_bytes: observed_physical_bytes, + }); + true + } else { + false + }; target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes); target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes); reservation.observed_target_physical_bytes = reservation @@ -1836,7 +2106,7 @@ fn record_decommission_target_inflight( signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes); renew_decommission_capacity_reservation(reservation, now, true); pool.last_update = now; - Ok(()) + Ok(ledger_changed) } fn record_decommission_target_observation( @@ -1882,9 +2152,13 @@ fn release_decommission_target_inflight( target_pool_index: usize, released_physical_bytes: usize, mutation_id: uuid::Uuid, - confirmed_absent: bool, + proof: DecommissionCapacityReleaseProof, now: OffsetDateTime, ) -> Result { + let DecommissionCapacityReleaseProof { + confirmed_absent, + clear_pending, + } = proof; let pool_count = meta.pools.len(); let pool = meta .pools @@ -1899,13 +2173,20 @@ fn release_decommission_target_inflight( .as_mut() .filter(|reservation| reservation.active()) .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?; + let model_version = reservation.model_version; let target = reservation .targets .iter_mut() .find(|target| target.pool_index == target_pool_index) .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?; - let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty(); - let released = release_decommission_target_temporary_mutation( + let has_temporary_mutations = !target.temporary_mutations.is_empty(); + let pending_belongs_to_mutation = target.pending_mutation_id == Some(mutation_id); + let zero_ledger_cleanup = confirmed_absent + && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + && !has_temporary_mutations + && target.pending_mutation_id.is_none() + && target.pending_physical_bytes == 0; + let (released, temporary_mutation_changed) = release_decommission_target_temporary_mutation( target, mutation_id, if confirmed_absent { @@ -1914,15 +2195,31 @@ fn release_decommission_target_inflight( released_physical_bytes }, ); - let released = if released == 0 && !has_scoped_temporary_mutations { + let released = if released == 0 && !has_temporary_mutations && model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION { let released = target.inflight_physical_bytes.min(released_physical_bytes); target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); released } else { released }; + let pending_matches = clear_pending && confirmed_absent && pending_belongs_to_mutation; + let published_pending_delayed_release = confirmed_absent + && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + && !clear_pending + && !has_temporary_mutations + && pending_belongs_to_mutation; + if released == 0 + && released_physical_bytes > 0 + && !zero_ledger_cleanup + && !published_pending_delayed_release + && (has_temporary_mutations || !pending_matches) + { + return Err(decommission_capacity_blocked_error( + "temporary target cleanup released bytes that cannot be attributed to its mutation", + )); + } reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released); - let cleared_pending = if confirmed_absent && target.pending_mutation_id == Some(mutation_id) { + let cleared_pending = if pending_matches { let cleared = target.pending_physical_bytes; target.pending_physical_bytes = 0; target.pending_mutation_id = None; @@ -1931,7 +2228,7 @@ fn release_decommission_target_inflight( } else { 0 }; - let changed = released > 0 || cleared_pending > 0; + let changed = released > 0 || temporary_mutation_changed || cleared_pending > 0; if changed { renew_decommission_capacity_reservation(reservation, now, true); pool.last_update = now; @@ -2986,6 +3283,7 @@ pub(crate) struct PoolRebalanceActivationFence { pool_meta_guard: rustfs_lock::NamespaceLockGuard, rebalance_meta_guard: rustfs_lock::NamespaceLockGuard, fleet_proof: Option, + decommission_target_fence_proof: Option, #[cfg(test)] forced_lost: Arc, } @@ -2998,6 +3296,13 @@ impl PoolRebalanceActivationFence { self.fleet_proof = fleet_proof; } + fn set_decommission_target_fence_proof( + &mut self, + fleet_proof: Option, + ) { + self.decommission_target_fence_proof = fleet_proof; + } + pub(crate) fn ensure_held(&self) -> Result<()> { #[cfg(test)] let forced_lost = self.forced_lost.load(Ordering::Acquire); @@ -3013,6 +3318,13 @@ impl PoolRebalanceActivationFence { { return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)); } + if self + .decommission_target_fence_proof + .as_ref() + .is_some_and(|proof| !crate::services::notification_sys::decommission_target_fence_fleet_proof_matches(proof)) + { + return Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED)); + } Ok(()) } @@ -3054,6 +3366,7 @@ where pool_meta_guard, rebalance_meta_guard, fleet_proof, + decommission_target_fence_proof: None, #[cfg(test)] forced_lost: Arc::new(AtomicBool::new(false)), }) @@ -3577,7 +3890,7 @@ enum DecommissionEntryAttemptOutcome { } #[cfg(test)] -pub(crate) type DecommissionTestFaultDecision = Arc bool + Send + Sync>; +pub(crate) type DecommissionTestFaultDecision = Arc bool + Send + Sync>; #[cfg(test)] static DECOMMISSION_TEST_FAULT_HOOK: std::sync::OnceLock>> = @@ -3626,7 +3939,8 @@ fn decommission_test_wrap_result( .lock() .expect("decommission test fault hook mutex should not poison") .clone(); - if result.is_ok() && decision.is_some_and(|decision| decision(stage, bucket, object, attempt)) { + let inject = decision.is_some_and(|decision| decision(stage, bucket, object, attempt, result.is_ok())); + if result.is_ok() && inject { return Err(Error::other(format!( "injected decommission test fault at {stage} attempt {attempt} for {bucket}/{object}" ))); @@ -4348,11 +4662,13 @@ fn pool_meta_from_v3_statuses(version: u16, pools: Vec) -> "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}" ))); } - Ok(PoolMeta { + let meta = PoolMeta { version, pools: pools.into_iter().map(TryInto::try_into).collect::>>()?, dont_save: false, - }) + }; + validate_decommission_capacity_model_cohort(&meta)?; + Ok(meta) } fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result { @@ -5477,11 +5793,13 @@ impl TryFrom for PoolMeta { fn try_from(value: PersistedPoolMeta) -> Result { ensure_pool_meta_payload_version(value.version, POOL_META_VERSION, "current")?; - Ok(Self { + let meta = Self { version: POOL_META_VERSION, pools: value.pools.into_iter().map(TryInto::try_into).collect::>>()?, dont_save: false, - }) + }; + validate_decommission_capacity_model_cohort(&meta)?; + Ok(meta) } } @@ -7107,6 +7425,29 @@ pub struct DecommissionCapacityReservation { pub release_reason: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecommissionCapacityMutationMode { + Durable, + Temporary, + NonGrowingReplacement, + TemporaryRelease, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DecommissionCapacityReleaseProof { + confirmed_absent: bool, + clear_pending: bool, +} + +impl DecommissionCapacityReleaseProof { + const fn confirmed_absence(clear_pending: bool) -> Self { + Self { + confirmed_absent: true, + clear_pending, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct DecommissionCapacityOwner { pub(crate) source_pool_index: usize, @@ -7116,6 +7457,172 @@ pub(crate) struct DecommissionCapacityOwner { pub(crate) mutation_id: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct DecommissionCapacityTargetPermitKey { + store_id: uuid::Uuid, + target_pool_index: usize, + source_pool_index: usize, + operation_id: uuid::Uuid, + generation: u64, + owner_nonce: uuid::Uuid, + mutation_id: uuid::Uuid, +} + +impl DecommissionCapacityTargetPermitKey { + fn new(store_id: uuid::Uuid, target_pool_index: usize, owner: DecommissionCapacityOwner) -> Option { + Some(Self { + store_id, + target_pool_index, + source_pool_index: owner.source_pool_index, + operation_id: owner.operation_id, + generation: owner.generation, + owner_nonce: owner.owner_nonce, + mutation_id: owner.mutation_id?, + }) + } + + fn owns_same_mutation(&self, store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> bool { + owner.mutation_id.is_some_and(|mutation_id| { + self.store_id == store_id + && self.source_pool_index == owner.source_pool_index + && self.operation_id == owner.operation_id + && self.generation == owner.generation + && self.owner_nonce == owner.owner_nonce + && self.mutation_id == mutation_id + }) + } +} + +struct DecommissionCapacityTargetPermit { + key: DecommissionCapacityTargetPermitKey, +} + +// Busy recovery crosses option rebuilding and a spawned migration task. Keep +// the distributed guard in one exact-mutation side table so target selection +// can pin that target and the formal capacity mutation can consume the guard. +static DECOMMISSION_CAPACITY_TARGET_PERMITS: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +fn decommission_capacity_target_permits() +-> &'static std::sync::Mutex> { + DECOMMISSION_CAPACITY_TARGET_PERMITS.get_or_init(|| std::sync::Mutex::new(HashMap::new())) +} + +fn install_decommission_capacity_target_permit( + store_id: uuid::Uuid, + target_pool_index: usize, + owner: DecommissionCapacityOwner, + guard: rustfs_lock::NamespaceLockGuard, +) -> Result { + let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner) + .ok_or_else(|| Error::other("decommission target permit is missing its mutation identity"))?; + let mut permits = decommission_capacity_target_permits() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if permits.keys().any(|candidate| candidate.owns_same_mutation(store_id, owner)) { + return Err(Error::other("decommission target permit already exists for this mutation")); + } + permits.insert(key, guard); + Ok(DecommissionCapacityTargetPermit { key }) +} + +fn take_decommission_capacity_target_permit( + store_id: uuid::Uuid, + target_pool_index: usize, + owner: DecommissionCapacityOwner, +) -> Option { + let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)?; + decommission_capacity_target_permits() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&key) +} + +fn decommission_capacity_target_permit_index(store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> Option { + decommission_capacity_target_permits() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .keys() + .find(|key| key.owns_same_mutation(store_id, owner)) + .map(|key| key.target_pool_index) +} + +fn discard_decommission_capacity_target_permit_except( + store_id: uuid::Uuid, + owner: DecommissionCapacityOwner, + retained_target_pool_index: Option, +) { + let guard = { + let mut permits = decommission_capacity_target_permits() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let key = permits + .keys() + .find(|key| key.owns_same_mutation(store_id, owner) && retained_target_pool_index != Some(key.target_pool_index)) + .copied(); + key.and_then(|key| permits.remove(&key)) + }; + drop(guard); +} + +impl Drop for DecommissionCapacityTargetPermit { + fn drop(&mut self) { + let guard = decommission_capacity_target_permits() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.key); + drop(guard); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DecommissionCapacityTerminalFencePlan { + model_version: u16, + operation_id: uuid::Uuid, + generation: u64, + owner_nonce: uuid::Uuid, + target_pool_indices: Vec, +} + +fn decommission_capacity_terminal_fence_plan( + meta: &PoolMeta, + source_pool_index: usize, +) -> Result> { + let Some(reservation) = meta + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.active()) + else { + return Ok(None); + }; + if reservation.source_pool_index != source_pool_index { + return Err(Error::DecommissionCapacity( + "decommission terminal transition found a mismatched capacity source".to_string(), + )); + } + if !matches!( + reservation.model_version, + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ) { + return Err(Error::DecommissionCapacity(format!( + "decommission terminal transition found unsupported capacity lock model {}", + reservation.model_version + ))); + } + let mut target_pool_indices = reservation.targets.iter().map(|target| target.pool_index).collect::>(); + target_pool_indices.sort_unstable(); + Ok(Some(DecommissionCapacityTerminalFencePlan { + model_version: reservation.model_version, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + target_pool_indices, + })) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct DecommissionDurableIlmCheckpointTarget { pub(crate) source_pool_index: usize, @@ -7655,6 +8162,7 @@ struct DecommissionCapacityLockOrderBarrierState { external_store_id: uuid::Uuid, owner_arrived: tokio::sync::Notify, owner_release: tokio::sync::Notify, + owner_pause_enabled: AtomicBool, external_capacity_released: tokio::sync::Notify, external_object_capacity_probe_acquired: tokio::sync::Notify, external_object_capacity_probe_release: tokio::sync::Notify, @@ -7664,6 +8172,15 @@ struct DecommissionCapacityLockOrderBarrierState { external_object_commit_phase_paused: AtomicBool, external_heal_operation_started: tokio::sync::Notify, external_heal_target_lock_attempted: tokio::sync::Notify, + target_gate_retry_entered: tokio::sync::Notify, + target_gate_retry_entries: AtomicUsize, + target_gate_exact_reloads: AtomicUsize, + target_gate_acquire_pause_target: AtomicUsize, + target_gate_acquire_entered: tokio::sync::Notify, + target_gate_acquire_release: tokio::sync::Notify, + cancel_before_start_entered: tokio::sync::Notify, + cancel_before_start_release: tokio::sync::Notify, + cancel_before_start_paused: AtomicBool, } #[cfg(test)] @@ -7684,6 +8201,7 @@ impl DecommissionCapacityLockOrderBarrier { external_store_id, owner_arrived: tokio::sync::Notify::new(), owner_release: tokio::sync::Notify::new(), + owner_pause_enabled: AtomicBool::new(true), external_capacity_released: tokio::sync::Notify::new(), external_object_capacity_probe_acquired: tokio::sync::Notify::new(), external_object_capacity_probe_release: tokio::sync::Notify::new(), @@ -7693,6 +8211,15 @@ impl DecommissionCapacityLockOrderBarrier { external_object_commit_phase_paused: AtomicBool::new(false), external_heal_operation_started: tokio::sync::Notify::new(), external_heal_target_lock_attempted: tokio::sync::Notify::new(), + target_gate_retry_entered: tokio::sync::Notify::new(), + target_gate_retry_entries: AtomicUsize::new(0), + target_gate_exact_reloads: AtomicUsize::new(0), + target_gate_acquire_pause_target: AtomicUsize::new(usize::MAX), + target_gate_acquire_entered: tokio::sync::Notify::new(), + target_gate_acquire_release: tokio::sync::Notify::new(), + cancel_before_start_entered: tokio::sync::Notify::new(), + cancel_before_start_release: tokio::sync::Notify::new(), + cancel_before_start_paused: AtomicBool::new(false), }); let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -7751,9 +8278,79 @@ impl DecommissionCapacityLockOrderBarrier { .expect("external heal should attempt the target namespace lock after capacity admission"); } + #[cfg(feature = "test-util")] + pub(crate) async fn wait_until_target_gate_retry(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_retry_entered.notified()) + .await + .expect("decommission entry should observe target gate contention"); + } + + #[cfg(feature = "test-util")] + pub(crate) async fn wait_until_target_gate_retries(&self, expected: usize) { + tokio::time::timeout(std::time::Duration::from_secs(30), async { + loop { + let notified = self.state.target_gate_retry_entered.notified(); + if self.state.target_gate_retry_entries.load(Ordering::Acquire) >= expected { + return; + } + notified.await; + } + }) + .await + .expect("decommission entries should observe target gate contention"); + } + + #[cfg(feature = "test-util")] + pub(crate) fn target_gate_exact_reloads(&self) -> usize { + self.state.target_gate_exact_reloads.load(Ordering::Acquire) + } + + #[cfg(feature = "test-util")] + pub(crate) fn pause_target_gate_acquire(&self, target_pool_index: usize) { + self.state + .target_gate_acquire_pause_target + .store(target_pool_index, Ordering::Release); + } + + #[cfg(feature = "test-util")] + pub(crate) async fn wait_until_target_gate_acquire_paused(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_acquire_entered.notified()) + .await + .expect("decommission capacity mutation should pause before acquiring its target gate"); + } + + #[cfg(feature = "test-util")] + pub(crate) fn release_target_gate_acquire(&self) { + self.state + .target_gate_acquire_pause_target + .store(usize::MAX, Ordering::Release); + self.state.target_gate_acquire_release.notify_one(); + } + + pub(crate) fn pause_cancel_before_start(&self) { + self.state.cancel_before_start_paused.store(true, Ordering::Release); + } + + pub(crate) async fn wait_until_cancel_before_start(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_before_start_entered.notified()) + .await + .expect("decommission cancel should pause before acquiring the start gate"); + } + + pub(crate) fn release_cancel_before_start(&self) { + self.state.cancel_before_start_release.notify_one(); + } + #[cfg(feature = "test-util")] pub(crate) fn release_owner(&self) { self.state.owner_release.notify_one(); + self.state.target_gate_acquire_release.notify_one(); + } + + #[cfg(feature = "test-util")] + pub(crate) fn disable_owner_pause(&self) { + self.state.owner_pause_enabled.store(false, Ordering::Release); + self.state.owner_release.notify_waiters(); } #[cfg(feature = "test-util")] @@ -7785,6 +8382,7 @@ impl Drop for DecommissionCapacityLockOrderBarrier { self.state.owner_release.notify_one(); self.state.external_object_capacity_probe_release.notify_one(); self.state.external_object_commit_phase_release.notify_one(); + self.state.cancel_before_start_release.notify_one(); let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() @@ -7802,7 +8400,7 @@ async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) { .lock() .expect("decommission capacity lock-order barrier should not be poisoned") .as_ref() - .filter(|state| state.owner_store_id == store_id) + .filter(|state| state.owner_store_id == store_id && state.owner_pause_enabled.load(Ordering::Acquire)) .cloned(); if let Some(barrier) = barrier { barrier.owner_arrived.notify_one(); @@ -7810,6 +8408,68 @@ async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) { } } +#[cfg(test)] +async fn pause_decommission_capacity_before_target_gate_acquire(store_id: uuid::Uuid, target_pool_index: usize) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| { + state.owner_store_id == store_id + && state.target_gate_acquire_pause_target.load(Ordering::Acquire) == target_pool_index + }) + .cloned(); + if let Some(barrier) = barrier { + barrier.target_gate_acquire_entered.notify_one(); + barrier.target_gate_acquire_release.notified().await; + } +} + +#[cfg(test)] +async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.owner_store_id == store_id && state.cancel_before_start_paused.load(Ordering::Acquire)) + .cloned(); + if let Some(barrier) = barrier { + barrier.cancel_before_start_entered.notify_one(); + barrier.cancel_before_start_release.notified().await; + } +} + +#[cfg(test)] +fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.owner_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel); + barrier.target_gate_retry_entered.notify_one(); + } +} + +#[cfg(test)] +fn notify_decommission_target_gate_exact_reload(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.owner_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.target_gate_exact_reloads.fetch_add(1, Ordering::AcqRel); + } +} + #[cfg(test)] pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) { let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER @@ -8326,6 +8986,131 @@ impl ECStore { Ok((pool_meta_guard, selection.meta)) } + async fn acquire_decommission_capacity_target_guard( + &self, + target_pool_index: usize, + ) -> Result { + // Some reconciliation callers already hold an object lock, while a + // target mutation acquires its object lock after this gate. A short, + // retryable acquisition bounds that inverse-order overlap. + let pool = self.pools.first().cloned().ok_or_else(|| { + Error::InvalidArgument( + "decommission-capacity".to_string(), + "storage-pools".to_string(), + "no storage pools available".to_string(), + ) + })?; + let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"); + let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + #[cfg(test)] + pause_decommission_capacity_before_target_gate_acquire(self.id, target_pool_index).await; + match target_lock + .get_write_lock_quiet(DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT) + .await + { + Ok(guard) => Ok(guard), + Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => { + Err(decommission_capacity_blocked_error(format!( + "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}{target_pool_index}{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}" + ))) + } + Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => Err(Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: RUSTFS_META_BUCKET.to_string(), + object, + required, + achieved, + }), + Err(err) => Err(Error::Lock(err)), + } + } + + async fn acquire_decommission_capacity_owner_target_guard( + &self, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + ) -> Result> { + let model_version = self + .pool_meta + .read() + .await + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| { + reservation.active() + && reservation.source_pool_index == owner.source_pool_index + && reservation.operation_id == owner.operation_id + && reservation.generation == owner.generation + }) + .map(|reservation| reservation.model_version) + .ok_or_else(|| decommission_capacity_blocked_error("decommission capacity owner is stale"))?; + match model_version { + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => Ok(None), + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => { + discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index)); + match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) { + Some(guard) => Ok(Some(guard)), + None => self + .acquire_decommission_capacity_target_guard(target_pool_index) + .await + .map(Some), + } + } + version => Err(Error::DecommissionCapacity(format!( + "decommission capacity owner uses unsupported lock model {version}" + ))), + } + } + + async fn acquire_decommission_capacity_terminal_guards( + &self, + plan: Option<&DecommissionCapacityTerminalFencePlan>, + ) -> Result> { + let Some(plan) = plan else { + return Ok(Vec::new()); + }; + match plan.model_version { + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => return Ok(Vec::new()), + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {} + version => { + return Err(Error::DecommissionCapacity(format!( + "decommission terminal transition found unsupported capacity lock model {version}" + ))); + } + } + // Terminal transitions hold no object or pool metadata lock here, so + // they can wait in target-index order without forming a lock cycle. + let pool = self.pools.first().cloned().ok_or_else(|| { + Error::InvalidArgument( + "decommission-capacity".to_string(), + "storage-pools".to_string(), + "no storage pools available".to_string(), + ) + })?; + let mut guards = Vec::with_capacity(plan.target_pool_indices.len()); + for &target_pool_index in &plan.target_pool_indices { + let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"); + let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + let guard = target_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| match err { + rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: RUSTFS_META_BUCKET.to_string(), + object, + required, + achieved, + }, + other => Error::Lock(other), + })?; + guards.push(guard); + } + Ok(guards) + } + pub(crate) async fn acquire_external_decommission_capacity_fence( &self, target_pool_indices: &[usize], @@ -8398,9 +9183,14 @@ impl ECStore { F: FnOnce() -> Fut, Fut: std::future::Future>, { - self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, false, false, |_| { - operation() - }) + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + expected_data_bytes, + DecommissionCapacityMutationMode::Durable, + |_| false, + |_| operation(), + ) .await } @@ -8415,8 +9205,15 @@ impl ECStore { F: FnOnce(Option>) -> Fut, Fut: std::future::Future>, { - self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, false, false, operation) - .await + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + expected_data_bytes, + DecommissionCapacityMutationMode::Durable, + |_| false, + operation, + ) + .await } pub(crate) async fn reconcile_decommission_capacity_before_exact_delete( @@ -8432,18 +9229,63 @@ impl ECStore { )); } - let reconciliations = { + let (reconciliations, model_version) = { let mut save_guard = self.pool_meta_save_gate.lock().await; let (_read_guard, snapshot) = self .acquire_pool_meta_read_guard(&mut save_guard, "exact delete capacity reconciliation failed") .await?; - plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? + let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?; + let model_version = active_decommission_capacity_model(&snapshot)?; + (reconciliations, model_version) }; if reconciliations.is_empty() { return Ok(()); } ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?; + let mut target_guards = Vec::new(); + let mut legacy_write_fence = None; + match model_version { + Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (write_guard, snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed") + .await?; + if active_decommission_capacity_model(&snapshot)? != model_version + || plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? != reconciliations + { + return Err(decommission_capacity_blocked_error( + "pending capacity changed before exact target evidence could be verified", + )); + } + legacy_write_fence = Some((save_guard, write_guard, snapshot)); + } + Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => { + let mut target_pool_indices = reconciliations + .iter() + .map(|reconciliation| reconciliation.target_pool_index) + .collect::>(); + target_pool_indices.sort_unstable(); + target_pool_indices.dedup(); + target_guards.reserve(target_pool_indices.len()); + for target_pool_index in target_pool_indices { + let guard = self.acquire_decommission_capacity_target_guard(target_pool_index).await?; + ensure_decommission_capacity_target_fence(&guard, target_pool_index, "exact delete evidence")?; + target_guards.push((target_pool_index, guard)); + } + } + Some(version) => { + return Err(Error::DecommissionCapacity(format!( + "exact delete capacity reconciliation found unsupported lock model {version}" + ))); + } + None => { + return Err(decommission_capacity_blocked_error( + "exact delete capacity reconciliation has no active reservation lock model", + )); + } + } + let target_lookup_options = ObjectOptions { versioned: opts.versioned, version_suspended: opts.version_suspended, @@ -8477,10 +9319,20 @@ impl ECStore { } ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?; - let mut save_guard = self.pool_meta_save_gate.lock().await; - let (write_guard, mut snapshot) = self - .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed") - .await?; + let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence { + fence + } else { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (write_guard, snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed") + .await?; + (save_guard, write_guard, snapshot) + }; + if active_decommission_capacity_model(&snapshot)? != model_version { + return Err(decommission_capacity_blocked_error( + "capacity lock model changed while exact target evidence was being verified", + )); + } let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?; if current_reconciliations != reconciliations { return Err(decommission_capacity_blocked_error( @@ -8488,6 +9340,9 @@ impl ECStore { )); } ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?; + for (target_pool_index, target_guard) in &target_guards { + ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity finalize")?; + } let now = OffsetDateTime::now_utc(); let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len()); @@ -8521,11 +9376,17 @@ impl ECStore { .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices) .await?; ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?; + for (target_pool_index, target_guard) in &target_guards { + ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity save")?; + } { let mut pool_meta = self.pool_meta.write().await; publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &source_pool_indices); } ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?; + for (target_pool_index, target_guard) in &target_guards { + ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity publication")?; + } outcome.disarm(); Ok(()) } @@ -8536,6 +9397,12 @@ impl ECStore { target_pool_index: usize, expected_data_bytes: usize, ) -> Result<()> { + let target_guard = self + .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index) + .await?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?; + } let mut save_guard = self.pool_meta_save_gate.lock().await; let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed") @@ -8584,6 +9451,13 @@ impl ECStore { .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; info.capacity_reservation = persisted_info.capacity_reservation; info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence( + target_guard, + target_pool_index, + "equivalent target idempotent publication", + )?; + } return Ok(()); } return Err(decommission_capacity_blocked_error( @@ -8624,6 +9498,9 @@ impl ECStore { ) .await?; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity save")?; + } { let persisted_info = outcome .committed @@ -8644,6 +9521,9 @@ impl ECStore { info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; } ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity publication")?; + } outcome.disarm(); Ok(()) } @@ -8659,9 +9539,14 @@ impl ECStore { F: FnOnce() -> Fut, Fut: std::future::Future>, { - self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, true, false, |_| { - operation() - }) + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + expected_data_bytes, + DecommissionCapacityMutationMode::Temporary, + |_| false, + |_| operation(), + ) .await } @@ -8676,21 +9561,64 @@ impl ECStore { F: FnOnce(Option>) -> Fut, Fut: std::future::Future>, { - self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, true, false, operation) - .await + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + expected_data_bytes, + DecommissionCapacityMutationMode::Temporary, + |_| false, + operation, + ) + .await + } + + /// Run an identity-preserving replacement that the caller has already + /// proven cannot grow the target object. A failed or ambiguous write keeps + /// the ordinary temporary-mutation recovery marker, while a successful + /// write resolves the capacity intent without retaining MPU cleanup state. + pub(crate) async fn run_decommission_capacity_non_growing_replacement_with_capacity_lease( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + operation: F, + ) -> Result + where + F: FnOnce(Option>) -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + expected_data_bytes, + DecommissionCapacityMutationMode::NonGrowingReplacement, + |_| false, + operation, + ) + .await } /// Finish the capacity transaction for an identity-preserving temporary /// replacement whose target bytes are already durably present. This is - /// the crash-recovery half of `run_decommission_capacity_temporary_mutation`: - /// it never writes the target again and only resolves a pending intent - /// owned by the exact deterministic mutation id. + /// the crash-recovery half of a non-growing replacement: it never writes + /// the target again and only settles state owned by the exact deterministic + /// mutation id. pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target( &self, owner: DecommissionCapacityOwner, target_pool_index: usize, expected_data_bytes: usize, ) -> Result<()> { + let target_guard = self + .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index) + .await?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence( + target_guard, + target_pool_index, + "equivalent temporary target reconciliation", + )?; + } let mut save_guard = self.pool_meta_save_gate.lock().await; let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed") @@ -8699,7 +9627,7 @@ impl ECStore { let mutation_id = owner .mutation_id .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?; - let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = { + let (target_layout, pending_physical_bytes, pending_mutation_id, has_temporary_mutation) = { let reservation = snapshot .pools .get(source_pool_index) @@ -8722,44 +9650,46 @@ impl ECStore { .any(|mutation| mutation.mutation_id == mutation_id), ) }; - if pending_physical_bytes == 0 { - // Either the successful attempt already saved its progress, or a - // byte-neutral replacement had no inflight delta to record. + if pending_physical_bytes == 0 && !has_temporary_mutation { + // A prior successful attempt already settled both durable halves. ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence( + target_guard, + target_pool_index, + "equivalent temporary target idempotent reconciliation", + )?; + } return Ok(()); } - if pending_mutation_id != Some(mutation_id) { + if pending_physical_bytes > 0 && pending_mutation_id != Some(mutation_id) { return Err(decommission_capacity_blocked_error( "equivalent temporary target pending intent belongs to another mutation", )); } - if already_reconciled { - return Err(decommission_capacity_blocked_error( - "equivalent temporary target has both pending and reconciled state", - )); - } let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?; - if pending_physical_bytes < expected_target_physical_bytes { + if pending_physical_bytes > 0 && pending_physical_bytes != expected_target_physical_bytes { return Err(decommission_capacity_blocked_error( - "equivalent temporary target pending capacity is smaller than the committed checkpoint", + "equivalent temporary target pending capacity does not match the committed checkpoint", )); } - resolve_decommission_target_pending( + if pending_physical_bytes > 0 { + resolve_decommission_target_pending( + &mut snapshot, + source_pool_index, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + )?; + } + // Exact byte equivalence proves that this identity-preserving, + // byte-non-growing replacement committed. Settle both crash windows: + // the pending intent before finalize and the temporary marker written + // when the operation returned an error after committing its target. + settle_decommission_target_non_growing_replacement( &mut snapshot, source_pool_index, target_pool_index, - expected_target_physical_bytes, - mutation_id, - )?; - // The replacement is byte-non-growing and its exact bytes were read - // before this call, so no new physical delta is inferred on replay. - // A prior successful progress save would have taken the idempotent - // pending==0 return above. - record_decommission_target_inflight( - &mut snapshot, - source_pool_index, - target_pool_index, - 0, mutation_id, OffsetDateTime::now_utc(), )?; @@ -8772,6 +9702,13 @@ impl ECStore { ) .await?; ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence( + target_guard, + target_pool_index, + "equivalent temporary target capacity save", + )?; + } let persisted_info = outcome .committed .pools @@ -8792,6 +9729,13 @@ impl ECStore { info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; } ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence( + target_guard, + target_pool_index, + "equivalent temporary target capacity publication", + )?; + } outcome.disarm(); Ok(()) } @@ -8817,13 +9761,20 @@ impl ECStore { .targets .iter() .find(|target| target.pool_index == target_pool_index) + .map(|target| (reservation.model_version, target)) }) - .is_some_and(|target| { - (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id)) - || target - .temporary_mutations - .iter() - .any(|mutation| mutation.mutation_id == mutation_id) + .is_some_and(|(model_version, target)| { + if model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION { + // V1 cannot persist an exact zero-byte staging marker, so + // retain its compatibility scan for admitted cleanup targets. + true + } else { + (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id)) + || target + .temporary_mutations + .iter() + .any(|mutation| mutation.mutation_id == mutation_id) + } }) } @@ -8851,32 +9802,47 @@ impl ECStore { ) -> Result where F: FnOnce(Option>) -> Fut, - Fut: std::future::Future>, + Fut: std::future::Future>, { - self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, None, false, true, operation) - .await + self.run_decommission_capacity_mutation( + target_pool_index, + capacity_owner, + None, + DecommissionCapacityMutationMode::TemporaryRelease, + |result: &(T, bool)| result.1, + operation, + ) + .await + .map(|(result, _)| result) } - async fn run_decommission_capacity_mutation( + async fn run_decommission_capacity_mutation( &self, target_pool_index: usize, capacity_owner: Option, expected_data_bytes: Option, - temporary: bool, - temporary_release: bool, + mode: DecommissionCapacityMutationMode, + clear_pending_on_temporary_release: P, operation: F, ) -> Result where F: FnOnce(Option>) -> Fut, Fut: std::future::Future>, + P: Fn(&T) -> bool, { + let temporary = matches!( + mode, + DecommissionCapacityMutationMode::Temporary | DecommissionCapacityMutationMode::NonGrowingReplacement + ); + let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement); + let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease); let mut operation = Some(operation); let mut save_guard = self.pool_meta_save_gate.lock().await; let (read_guard, snapshot) = self .acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed") .await?; let admission_now = OffsetDateTime::now_utc(); - let owner = capacity_owner.filter(|owner| { + let admitted_owner = capacity_owner.and_then(|owner| { snapshot .pools .get(owner.source_pool_index) @@ -8884,37 +9850,59 @@ impl ECStore { .and_then(|info| info.capacity_reservation.as_ref()) .filter(|reservation| { if temporary_release { - reservation.admits_cleanup_owner(*owner) + reservation.admits_cleanup_owner(owner) } else { - reservation.admits_owner(*owner, admission_now) + reservation.admits_owner(owner, admission_now) } }) - .is_some_and(|reservation| { + .filter(|reservation| { reservation .targets .iter() .any(|target| target.pool_index == target_pool_index) }) + .map(|reservation| (owner, reservation.model_version)) }); - if capacity_owner.is_some() && owner.is_none() { + if capacity_owner.is_some() && admitted_owner.is_none() { return Err(decommission_capacity_blocked_error( "decommission target mutation reservation identity is stale", )); } - if owner.is_none() { + let Some((owner, model_version)) = admitted_owner else { ensure_external_decommission_target_admission(&snapshot, target_pool_index, "mutation")?; drop(save_guard); let capacity_lease = read_guard.lock_lost_signal(); return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await; + }; + + // Per-target reservations always acquire target -> pool metadata. + // Legacy reservations keep the pool metadata write guard through I/O. + drop(read_guard); + drop(save_guard); + if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION { + discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index)); } + let target_guard = match model_version { + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => None, + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => { + Some(match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) { + Some(guard) => guard, + None => self.acquire_decommission_capacity_target_guard(target_pool_index).await?, + }) + } + version => { + return Err(Error::DecommissionCapacity(format!( + "decommission target mutation found unsupported capacity lock model {version}" + ))); + } + }; #[cfg(test)] pause_decommission_capacity_before_owner_write(self.id).await; - drop(read_guard); + let mut save_guard = self.pool_meta_save_gate.lock().await; let (write_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed") .await?; - let owner = owner.expect("capacity owner should remain present"); let mutation_id = owner .mutation_id .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?; @@ -8929,10 +9917,11 @@ impl ECStore { reservation.admits_cleanup_owner(owner) } else { reservation.admits_owner(owner, OffsetDateTime::now_utc()) - }) && reservation - .targets - .iter() - .any(|target| target.pool_index == target_pool_index) + }) && reservation.model_version == model_version + && reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index) }) .is_some(); if !owner_current { @@ -9036,6 +10025,21 @@ impl ECStore { ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?; outcome.disarm(); } + let mut write_guard = Some(write_guard); + let mut save_guard = Some(save_guard); + let capacity_lease = if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity intent prepare")?; + let capacity_lease = target_guard.lock_lost_signal(); + drop(write_guard.take()); + drop(save_guard.take()); + capacity_lease + } else { + write_guard + .as_ref() + .ok_or_else(|| Error::other("legacy decommission capacity fence disappeared before target mutation"))? + .lock_lost_signal() + }; + let capacity_infos = if pending_added > 0 { self.get_decommission_all_pool_capacity_infos().await? } else { @@ -9047,8 +10051,35 @@ impl ECStore { .map(|capacity| capacity.physical_free) .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?; - let capacity_lease = write_guard.lock_lost_signal(); let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await; + let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release); + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?; + let mut finalize_save_guard = self.pool_meta_save_gate.lock().await; + let (finalize_write_guard, finalize_snapshot) = self + .acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed") + .await?; + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity finalize")?; + save_guard = Some(finalize_save_guard); + write_guard = Some(finalize_write_guard); + snapshot = finalize_snapshot; + } + let write_guard = write_guard + .as_ref() + .ok_or_else(|| Error::other("decommission capacity write fence disappeared before finalize"))?; + let save_guard = save_guard + .as_mut() + .ok_or_else(|| Error::other("decommission capacity save fence disappeared before finalize"))?; + ensure_pool_meta_write_fence(write_guard, "decommission target capacity finalize failed")?; + ensure_decommission_capacity_mutation_intent_current( + &snapshot, + owner, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + temporary_release, + model_version, + )?; let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; let after_free = capacity_infos .iter() @@ -9059,15 +10090,19 @@ impl ECStore { let released_physical_bytes = after_free.saturating_sub(before_free); let now = OffsetDateTime::now_utc(); let progress_changed = if temporary_release { - release_decommission_target_inflight( - &mut snapshot, - source_pool_index, - target_pool_index, - released_physical_bytes, - mutation_id, - result.is_ok(), - now, - )? + if result.is_ok() { + release_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + released_physical_bytes, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(clear_pending_on_temporary_release), + now, + )? + } else { + false + } } else if result.is_ok() { resolve_decommission_target_pending( &mut snapshot, @@ -9077,14 +10112,24 @@ impl ECStore { mutation_id, )?; if temporary { - record_decommission_target_inflight( - &mut snapshot, - source_pool_index, - target_pool_index, - observed_physical_bytes, - mutation_id, - now, - )?; + if non_growing_replacement { + settle_decommission_target_non_growing_replacement( + &mut snapshot, + source_pool_index, + target_pool_index, + mutation_id, + now, + )?; + } else { + record_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + observed_physical_bytes, + mutation_id, + now, + )?; + } } else { let consumed_physical_bytes = expected_data_bytes .map(|_| expected_target_physical_bytes) @@ -9105,7 +10150,7 @@ impl ECStore { )?; } true - } else if expected_target_physical_bytes == 0 { + } else if temporary { record_decommission_target_inflight( &mut snapshot, source_pool_index, @@ -9113,8 +10158,7 @@ impl ECStore { observed_physical_bytes, mutation_id, now, - )?; - observed_physical_bytes > 0 + )? } else { record_decommission_target_observation( &mut snapshot, @@ -9134,14 +10178,20 @@ impl ECStore { snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?; } if temporary_release && !progress_changed { - ensure_pool_meta_write_fence(&write_guard, "decommission target capacity cleanup fence failed")?; + ensure_pool_meta_write_fence(write_guard, "decommission target capacity cleanup fence failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity cleanup")?; + } capacity_result?; return result; } let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) + .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) .await?; - ensure_pool_meta_write_fence(&write_guard, "decommission target capacity progress save failed")?; + ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress save")?; + } { let persisted_info = outcome .committed @@ -9161,7 +10211,10 @@ impl ECStore { info.capacity_reservation = persisted_info.capacity_reservation; info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; } - ensure_pool_meta_write_fence(&write_guard, "decommission target capacity progress save failed")?; + ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?; + if let Some(target_guard) = target_guard.as_ref() { + ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress publication")?; + } outcome.disarm(); capacity_result?; result @@ -9423,7 +10476,7 @@ impl ECStore { #[cfg(test)] observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission); let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?; - let activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?; + let mut activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?; let mut rebalance_meta = RebalanceMeta::new(); match rebalance_meta @@ -9464,6 +10517,7 @@ impl ECStore { ensure_decommission_capacity_writer_supported(&latest_pool_meta)?; let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; activation_fence.ensure_held()?; + let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof(); let previous_pool_meta = latest_pool_meta.clone(); let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?; @@ -9483,6 +10537,10 @@ impl ECStore { )?; latest_pool_meta.queue_buckets(idx, decom_buckets.clone()); } + let model_version = select_decommission_capacity_model(&latest_pool_meta, target_fence_proof.is_some())?; + if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION { + activation_fence.set_decommission_target_fence_proof(target_fence_proof); + } reserve_decommission_start_target_capacity( &mut latest_pool_meta, indices, @@ -9490,6 +10548,7 @@ impl ECStore { uuid::Uuid::new_v4(), capacity_generation, now, + model_version, )?; activation_fence.ensure_held()?; @@ -9696,15 +10755,30 @@ impl ECStore { .and_then(|info| info.capacity_reservation.as_ref()) .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc())) .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?; + let candidate = |target: &DecommissionCapacityTarget| { + let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?; + let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies)); + let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies); + (required_peak <= remaining).then_some((target.pool_index, remaining)) + }; + if let Some(permitted_target_pool_index) = decommission_capacity_target_permit_index(self.id, owner) { + if let Some((pool_index, _)) = reservation + .targets + .iter() + .find(|target| target.pool_index == permitted_target_pool_index) + .and_then(&candidate) + { + return Ok(pool_index); + } + // The holder that preceded this waiter may have consumed the + // remaining allocation. Release that guard before selecting a + // different target so one mutation never holds two target gates. + discard_decommission_capacity_target_permit_except(self.id, owner, None); + } reservation .targets .iter() - .filter_map(|target| { - let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?; - let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies)); - let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies); - (required_peak <= remaining).then_some((target.pool_index, remaining)) - }) + .filter_map(candidate) .max_by_key(|(_, remaining)| *remaining) .map(|(pool_index, _)| pool_index) .ok_or_else(|| { @@ -9763,7 +10837,32 @@ impl ECStore { { let owner = owner.as_ref(); ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; + #[cfg(test)] + if acquire_runtime_fence { + pause_decommission_cancel_before_start_gate(self.id).await; + } let _start_guard = self.start_gate.lock().await; + // Read the fence model from durable metadata without retaining the + // global lock, then fence the exact target cohort before taking the + // write lock used to publish the terminal transition. + let terminal_fence_plan = if acquire_runtime_fence { + let mut read_save_guard = self.pool_meta_save_gate.lock().await; + let (read_guard, snapshot) = self + .acquire_pool_meta_read_guard(&mut read_save_guard, "decommission cancel fence planning failed") + .await?; + let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?; + drop(read_guard); + drop(read_save_guard); + plan + } else { + None + }; + let _capacity_target_guards = if acquire_runtime_fence { + self.acquire_decommission_capacity_terminal_guards(terminal_fence_plan.as_ref()) + .await? + } else { + Vec::new() + }; let mut save_guard = self.pool_meta_save_gate.lock().await; let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence { let (guard, pool_meta) = self @@ -9774,18 +10873,34 @@ impl ECStore { save_guard.ensure_write_safe("decommission cancel failed")?; (None, None) }; + if let Some(persisted_pool_meta) = persisted_pool_meta.as_ref() { + let committed_plan = decommission_capacity_terminal_fence_plan(persisted_pool_meta, idx)?; + if committed_plan != terminal_fence_plan { + return Err(decommission_capacity_blocked_error( + "decommission capacity owner or target cohort changed while acquiring terminal fences", + )); + } + } let pool_meta_fence = _pool_meta_guard .as_ref() .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal); - // Lock order: start gate, save gate, distributed pool metadata fence, - // rebalance_meta, decommission_cancelers, then pool_meta. The state - // guards stay held across persistence so the active generation cannot - // change before the cancel is published. + // Lock order: start gate, target gates, save gate, distributed pool + // metadata fence, rebalance_meta, decommission_cancelers, then + // pool_meta. The state guards stay held across persistence so the + // active generation cannot change before the cancel is published. let rebalance_meta = self.rebalance_meta.read().await.clone(); let terminal_at = OffsetDateTime::now_utc(); let mut cancelers = self.decommission_cancelers.write().await; let mut pool_meta = self.pool_meta.write().await; + if acquire_runtime_fence { + let local_plan = decommission_capacity_terminal_fence_plan(&pool_meta, idx)?; + if local_plan != terminal_fence_plan { + return Err(decommission_capacity_blocked_error( + "local decommission capacity owner differs from the durable terminal fence plan", + )); + } + } let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = { let mut already_canceled = false; let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) { @@ -10166,15 +11281,32 @@ impl ECStore { } else { self.get_decommission_all_pool_capacity_infos().await? }; + let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof(); let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } + let capacity_recovery_needed = active_decommission_source_indices(&pool_meta).into_iter().any(|source_idx| { + pool_meta + .pools + .get(source_idx) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .is_none_or(|reservation| !reservation.active()) + }); let capacity_indices = if capacity_infos.is_empty() { Vec::new() } else { - recover_decommission_capacity_reservations(&mut pool_meta, &capacity_infos, OffsetDateTime::now_utc())? + recover_decommission_capacity_reservations( + &mut pool_meta, + &capacity_infos, + OffsetDateTime::now_utc(), + target_fence_proof.is_some(), + )? }; + let target_fence_proof_required = capacity_recovery_needed + && active_decommission_capacity_model(&pool_meta)? == Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION); + ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?; let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref()); let mut changed_indices = capacity_indices; @@ -10188,6 +11320,7 @@ impl ECStore { drop(pool_meta); let (save_outcome, save_error) = if changed { + ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?; match snapshot .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices) .await @@ -10200,6 +11333,7 @@ impl ECStore { }; let generation = self.active_decommission_generation(idx).await?; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?; + ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?; if let Some(outcome) = save_outcome { let mut pool_meta = self.pool_meta.write().await; pool_meta.version = pool_meta.version.max(outcome.committed.version); @@ -10809,6 +11943,63 @@ impl ECStore { Ok(()) } + #[allow(clippy::too_many_arguments)] + async fn wait_decommission_target_gate_retry( + &self, + rx: &CancellationToken, + idx: usize, + generation: OffsetDateTime, + target_pool_index: usize, + capacity_owner: Option, + set: &SetDisks, + entry: &MetaCacheEntry, + bucket: &str, + expected_version: &FileInfo, + target_busy_attempt: usize, + ) -> Result> { + #[cfg(test)] + notify_decommission_target_gate_retry(self.id); + let mut wait_attempt = target_busy_attempt; + let target_guard = loop { + let retry_delay = decommission_retry_backoff_delay( + DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, + wait_attempt.min(DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS), + ); + if wait_decommission_retry_backoff(rx, retry_delay).await { + decommission_cancel_signal_result(rx.is_cancelled())?; + } + if self.decommission_cancel_requested(idx, rx).await { + rx.cancel(); + } + decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; + + match self.acquire_decommission_capacity_target_guard(target_pool_index).await { + Ok(guard) => break guard, + Err(err) if is_decommission_capacity_target_gate_busy(&err) => { + wait_attempt = wait_attempt.saturating_add(1); + } + Err(err) => return Err(err), + } + }; + + #[cfg(test)] + notify_decommission_target_gate_exact_reload(self.id); + let current = load_decommission_entry_exact_versions(set, entry, bucket, "target_gate_retry").await?; + ensure_decommission_capacity_target_fence(&target_guard, target_pool_index, "source identity revalidation")?; + let expected_identity = data_movement::source_cleanup_version_identity(expected_version); + let identity_current = current + .versions + .iter() + .any(|version| data_movement::source_cleanup_version_identity(version) == expected_identity); + if !identity_current { + return Ok(None); + } + let owner = capacity_owner.ok_or_else(|| Error::other("target-gate retry is missing its decommission capacity owner"))?; + let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, bucket, expected_version)); + install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some) + } + #[allow(clippy::too_many_arguments)] #[tracing::instrument(skip( self, @@ -10832,32 +12023,14 @@ impl ECStore { expected_bucket_incarnation_id: Option, source_changed_exhaustions: Arc, ) -> Result<()> { - let uses_capacity_ledger = self - .pool_meta - .read() - .await - .pools - .get(idx) - .and_then(|pool| pool.decommission.as_ref()) - .and_then(|info| info.capacity_reservation.as_ref()) - .is_some_and(DecommissionCapacityReservation::active); let mut counted_versions = HashSet::new(); for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS { let attempt_result = { let mut conflict_attempt = 0; loop { - let result = { - let _capacity_entry_guard = if uses_capacity_ledger { - Some(tokio::select! { - biased; - _ = rx.cancelled() => return decommission_cancel_signal_result(true), - guard = self.decommission_capacity_entry_gate.lock() => guard, - }) - } else { - None - }; - self.decommission_entry_attempt( + let result = self + .decommission_entry_attempt( rx.clone(), idx, generation, @@ -10872,20 +12045,22 @@ impl ECStore { source_changed_exhaustions.as_ref(), &mut counted_versions, ) - .await - }; - if result.as_ref().is_err_and(is_decommission_capacity_intent_conflict) - && conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - { - conflict_attempt += 1; - let retry_delay = - decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt); - if wait_decommission_retry_backoff(&rx, retry_delay).await { - decommission_cancel_signal_result(rx.is_cancelled())?; + .await; + let retry = result + .as_ref() + .err() + .and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt)); + let retry_attempt = match retry { + Some(DecommissionCapacityRetryKind::IntentConflict) => { + conflict_attempt += 1; + conflict_attempt } - continue; + None => break result, + }; + let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, retry_attempt); + if wait_decommission_retry_backoff(&rx, retry_delay).await { + decommission_cancel_signal_result(rx.is_cancelled())?; } - break result; } }; match attempt_result { @@ -11030,9 +12205,12 @@ impl ECStore { let mut migrated = false; let mut consumed = false; let mut capacity_failure = false; - for _ in 0..3 { - match classify_decommission_free_version_attempt( - self.run_guarded_decommission_side_effect(&rx, &operation_gate, || async { + let mut version_attempt = 1; + let mut target_busy_attempt: usize = 0; + let mut target_permit = None; + while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS { + let result = self + .run_guarded_decommission_side_effect(&rx, &operation_gate, || async { self.decommission_tiered_object( bucket.as_str(), &version.name, @@ -11049,8 +12227,33 @@ impl ECStore { ) .await }) - .await, - ) { + .await; + drop(target_permit.take()); + if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index) + { + target_busy_attempt = target_busy_attempt.saturating_add(1); + let Some(permit) = self + .wait_decommission_target_gate_retry( + &rx, + idx, + generation, + target_pool_index, + capacity_owner, + set.as_ref(), + &entry, + &bucket, + version, + target_busy_attempt, + ) + .await? + else { + return Ok(DecommissionEntryAttemptOutcome::SourceChanged); + }; + target_permit = Some(permit); + continue; + } + + match classify_decommission_free_version_attempt(result) { DecommissionFreeVersionAttempt::Migrated => { migrated = true; migration_error = None; @@ -11066,7 +12269,10 @@ impl ECStore { migration_error = Some(err); break; } - DecommissionFreeVersionAttempt::Retry(err) => migration_error = Some(err), + DecommissionFreeVersionAttempt::Retry(err) => { + migration_error = Some(err); + version_attempt += 1; + } } } @@ -11182,7 +12388,10 @@ impl ECStore { let mut failure = false; let mut error = None; if version.deleted { - for version_attempt in 1..=DECOMMISSION_VERSION_COPY_ATTEMPTS { + let mut version_attempt = 1; + let mut target_busy_attempt: usize = 0; + let mut target_permit = None; + while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS { let result = self .run_guarded_decommission_side_effect(&rx, &operation_gate, || async { self.delete_object( @@ -11201,6 +12410,30 @@ impl ECStore { .await }) .await; + drop(target_permit.take()); + if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index) + { + target_busy_attempt = target_busy_attempt.saturating_add(1); + let Some(permit) = self + .wait_decommission_target_gate_retry( + &rx, + idx, + generation, + target_pool_index, + capacity_owner, + set.as_ref(), + &entry, + &bucket, + version, + target_busy_attempt, + ) + .await? + else { + return Ok(DecommissionEntryAttemptOutcome::SourceChanged); + }; + target_permit = Some(permit); + continue; + } #[cfg(test)] let result = decommission_test_wrap_result( "delete_marker_copy", @@ -11281,6 +12514,7 @@ impl ECStore { if wait_decommission_retry_backoff(&rx, retry_delay).await { decommission_cancel_signal_result(rx.is_cancelled())?; } + version_attempt += 1; } } } @@ -11336,7 +12570,10 @@ impl ECStore { continue; } - for version_attempt in 1..=DECOMMISSION_VERSION_COPY_ATTEMPTS { + let mut version_attempt = 1; + let mut target_busy_attempt: usize = 0; + let mut target_permit = None; + while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS { if version.is_remote() { let result = self .run_guarded_decommission_side_effect(&rx, &operation_gate, || async { @@ -11357,6 +12594,30 @@ impl ECStore { .await }) .await; + drop(target_permit.take()); + if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index) + { + target_busy_attempt = target_busy_attempt.saturating_add(1); + let Some(permit) = self + .wait_decommission_target_gate_retry( + &rx, + idx, + generation, + target_pool_index, + capacity_owner, + set.as_ref(), + &entry, + &bucket, + version, + target_busy_attempt, + ) + .await? + else { + return Ok(DecommissionEntryAttemptOutcome::SourceChanged); + }; + target_permit = Some(permit); + continue; + } #[cfg(test)] let result = decommission_test_wrap_result( "decommission_tiered_object", @@ -11422,6 +12683,7 @@ impl ECStore { if wait_decommission_retry_backoff(&rx, retry_delay).await { decommission_cancel_signal_result(rx.is_cancelled())?; } + version_attempt += 1; continue; } } @@ -11474,7 +12736,7 @@ impl ECStore { "Decommission source object read failed" ); error = Some(err); - continue; + break; } let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt); @@ -11497,6 +12759,7 @@ impl ECStore { if wait_decommission_retry_backoff(&rx, retry_delay).await { decommission_cancel_signal_result(rx.is_cancelled())?; } + version_attempt += 1; continue; } }; @@ -11520,6 +12783,33 @@ impl ECStore { .await }) .await; + drop(target_permit.take()); + if let Some(target_pool_index) = migrate_result + .as_ref() + .err() + .and_then(decommission_capacity_target_gate_busy_index) + { + target_busy_attempt = target_busy_attempt.saturating_add(1); + let Some(permit) = self + .wait_decommission_target_gate_retry( + &rx, + idx, + generation, + target_pool_index, + capacity_owner, + set.as_ref(), + &entry, + &bucket_name, + version, + target_busy_attempt, + ) + .await? + else { + return Ok(DecommissionEntryAttemptOutcome::SourceChanged); + }; + target_permit = Some(permit); + continue; + } #[cfg(test)] let migrate_result = decommission_test_wrap_result( DECOMMISSION_STAGE_MIGRATE_OBJECT, @@ -11562,7 +12852,7 @@ impl ECStore { "Decommission object migration failed" ); error = Some(err); - continue; + break; } let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt); @@ -11585,6 +12875,7 @@ impl ECStore { if wait_decommission_retry_backoff(&rx, retry_delay).await { decommission_cancel_signal_result(rx.is_cancelled())?; } + version_attempt += 1; continue; } @@ -11920,7 +13211,12 @@ impl ECStore { let mut pool_meta = self.pool_meta.write().await; let version = pool_meta.version; pool_meta.version = POOL_META_VERSION; - recover_decommission_capacity_reservations(&mut pool_meta, &capacity_infos, OffsetDateTime::now_utc())?; + recover_decommission_capacity_reservations( + &mut pool_meta, + &capacity_infos, + OffsetDateTime::now_utc(), + crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(), + )?; pool_meta.version = version; } let generation = self.active_decommission_generation(idx).await?; @@ -12773,6 +14069,8 @@ impl ECStore { .await?; let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let target_fence_proof_available = + crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(); // Signal cancellation before waiting for the movement writer so active // object operations can observe the signal and release read guards. self.cancel_decommission_routines(&indices).await; @@ -12785,12 +14083,12 @@ impl ECStore { // another start. let mut cancelers = self.decommission_cancelers.write().await; let pool_meta = self.pool_meta.read().await; - ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos)?; + ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?; reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())? } else { let pool_meta = self.pool_meta.read().await; ensure_decommission_start_pool_states(&pool_meta, &indices)?; - ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos)?; + ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?; Vec::new() }; @@ -15012,10 +16310,481 @@ mod tests { .as_ref() .and_then(|info| info.capacity_reservation.as_ref()) .expect("the winning operation must retain its durable reservation"); + assert_eq!( + reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + "an all-v4 proof must persist the per-target lock model through V3 replicas" + ); assert_eq!(reservation.peak_physical_bytes, 60); assert!(persisted.pools[1].decommission.is_none()); } + #[tokio::test] + #[serial_test::serial] + async fn decommission_start_without_dedicated_v4_proof_persists_the_legacy_lock_model() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0), + ]], + ); + let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test(); + + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("a mixed-version fleet should retain the compatible global lock model"); + let reservation = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("proofless start should persist a capacity reservation") + .clone(); + assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION); + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_replans_target_fences_after_a_concurrent_v2_start() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0), + ]], + ); + let barrier = super::DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.pause_cancel_before_start(); + let cancel_store = Arc::clone(&store); + let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + barrier.wait_until_cancel_before_start().await; + + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("the concurrent start should persist a v2 reservation"); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .map(|reservation| { + assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION); + reservation.targets[0].pool_index + }) + .expect("the concurrent start should publish its target cohort"); + let target_lock = store.pools[0] + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{}/{target_pool_index}", super::DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX), + ) + .await + .expect("create the cancel/start target-fence probe"); + let target_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("hold the newly started target fence"); + + barrier.release_cancel_before_start(); + tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel) + .await + .expect_err("cancel must wait for the target fence selected by the concurrent start"); + assert!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .is_some_and(DecommissionCapacityReservation::active), + "cancel must not release the new reservation before fencing its target" + ); + + drop(target_guard); + tokio::time::timeout(std::time::Duration::from_secs(30), cancel) + .await + .expect("cancel should finish after the target fence is released") + .expect("cancel task should not panic") + .expect("cancel should publish the terminal state"); + drop(barrier); + + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("canceled decommission metadata should remain present"); + assert!(info.canceled); + let reservation = info + .capacity_reservation + .as_ref() + .expect("canceled capacity accounting should remain inspectable"); + assert!(!reservation.active()); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + } + + #[tokio::test] + #[serial_test::serial] + async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() { + let (_temp_dirs, first_node, stale_node) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + crate::services::rebalance::promote_test_pool_meta_to_v2(&first_node).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + first_node.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0), + ]], + ); + + first_node + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("start the operation retained by the stale node"); + let stale_operation = first_node.pool_meta.read().await.clone(); + let stale_plan = decommission_capacity_terminal_fence_plan(&stale_operation, 0) + .expect("the stale operation should have a valid terminal fence plan") + .expect("the stale operation should have an active reservation"); + + first_node + .decommission_cancel(0) + .await + .expect("cancel the first durable operation"); + first_node + .clear_decommission(0) + .await + .expect("clear the first terminal operation"); + first_node + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("start the replacement durable operation"); + let replacement_plan = { + let pool_meta = first_node.pool_meta.read().await; + decommission_capacity_terminal_fence_plan(&pool_meta, 0) + .expect("the replacement operation should have a valid terminal fence plan") + .expect("the replacement operation should have an active reservation") + }; + assert_ne!(replacement_plan.operation_id, stale_plan.operation_id); + + *stale_node.pool_meta.write().await = stale_operation; + let err = stale_node + .decommission_cancel(0) + .await + .expect_err("a stale local operation must not be merged over the durable replacement"); + assert!(err.to_string().contains("differs from the durable terminal fence plan")); + + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(first_node.pools.clone()) + .await + .expect("the replacement operation should remain readable from durable replicas"); + assert_eq!( + decommission_capacity_terminal_fence_plan(&durable, 0) + .expect("the durable replacement should retain a valid terminal fence plan") + .expect("the durable replacement reservation should remain active"), + replacement_plan + ); + let replacement = durable.pools[0] + .decommission + .as_ref() + .expect("the durable replacement metadata should remain present"); + assert!(!replacement.canceled); + assert!( + replacement + .capacity_reservation + .as_ref() + .is_some_and(DecommissionCapacityReservation::active) + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn legacy_capacity_model_holds_the_global_pool_meta_fence_through_target_io() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80), + DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0), + ]], + ); + let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test(); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate a legacy reservation spanning two targets"); + let base_owner = { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("legacy capacity reservation should exist"); + assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION); + assert_eq!(reservation.targets.iter().map(|target| target.pool_index).collect::>(), vec![1, 2]); + DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + } + }; + + let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel(); + let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel(); + let first_store = Arc::clone(&store); + let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4()); + let first = tokio::spawn(async move { + first_store + .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async { + first_entered_tx.send(()).expect("first legacy mutation should be observed"); + first_release_rx.await.expect("first legacy mutation should be released"); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx) + .await + .expect("first legacy mutation should enter") + .expect("first legacy mutation should report entry"); + assert!( + store.pool_meta_save_gate.try_lock().is_err(), + "legacy target I/O must retain the local pool metadata save gate" + ); + let pool_meta_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the legacy pool metadata lock probe"); + assert!( + pool_meta_lock + .get_write_lock_quiet(std::time::Duration::from_millis(300)) + .await + .is_err(), + "legacy target I/O must retain the distributed pool metadata write fence" + ); + + let (second_entered_tx, mut second_entered_rx) = tokio::sync::oneshot::channel(); + let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel(); + let second_store = Arc::clone(&store); + let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4()); + let second = tokio::spawn(async move { + second_store + .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async { + second_entered_tx.send(()).expect("second legacy mutation should be observed"); + second_release_rx.await.expect("second legacy mutation should be released"); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_millis(500), &mut second_entered_rx) + .await + .expect_err("a different target must still wait behind the legacy global fence"); + + first_release_tx.send(()).expect("release first legacy mutation"); + first + .await + .expect("first legacy mutation task should not panic") + .expect("first legacy mutation should finalize"); + tokio::time::timeout(std::time::Duration::from_secs(30), &mut second_entered_rx) + .await + .expect("second legacy mutation should enter after the first finalizes") + .expect("second legacy mutation should report entry"); + assert!( + pool_meta_lock + .get_write_lock_quiet(std::time::Duration::from_millis(300)) + .await + .is_err(), + "the second legacy target I/O must also retain the global fence" + ); + second_release_tx.send(()).expect("release second legacy mutation"); + second + .await + .expect("second legacy mutation task should not panic") + .expect("second legacy mutation should finalize"); + + let pool_meta_guard = pool_meta_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("the global pool metadata fence should release after both target tails"); + drop(pool_meta_guard); + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("legacy reservation should remain active"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 2); + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_capacity_mutations_overlap_across_targets_but_serialize_per_target() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80), + DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate a reservation spanning two targets"); + + let base_owner = { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("active capacity reservation should exist"); + assert_eq!( + reservation.targets.iter().map(|target| target.pool_index).collect::>(), + vec![1, 2], + "the fixture must reserve both target pools" + ); + DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + } + }; + + let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel(); + let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel(); + let first_store = Arc::clone(&store); + let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4()); + let first = tokio::spawn(async move { + first_store + .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async { + first_entered_tx.send(()).expect("first target mutation should be observed"); + first_release_rx.await.expect("first target mutation should be released"); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx) + .await + .expect("first target mutation should enter without hanging") + .expect("first target mutation should report entry"); + + let (second_entered_tx, second_entered_rx) = tokio::sync::oneshot::channel(); + let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel(); + let second_store = Arc::clone(&store); + let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4()); + let second = tokio::spawn(async move { + second_store + .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async { + second_entered_tx.send(()).expect("second target mutation should be observed"); + second_release_rx.await.expect("second target mutation should be released"); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(30), second_entered_rx) + .await + .expect("a different target mutation should overlap instead of waiting for the first") + .expect("second target mutation should report entry"); + + let save_guard = store + .pool_meta_save_gate + .try_lock() + .expect("target I/O must not retain the local pool metadata save gate"); + drop(save_guard); + let pool_meta_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create a pool metadata lock probe"); + let pool_meta_guard = pool_meta_lock + .get_write_lock_quiet(std::time::Duration::from_secs(1)) + .await + .expect("target I/O must not retain the distributed pool metadata write lock"); + drop(pool_meta_guard); + + let blocked_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4()); + let blocked = store + .run_decommission_capacity_admitted_mutation(1, Some(blocked_owner), Some(1), || async { Ok(()) }) + .await + .expect_err("a second mutation on the same target must wait behind its target gate"); + assert!(is_decommission_capacity_blocked_error(&blocked)); + assert!( + is_decommission_capacity_target_gate_busy(&blocked), + "same-target serialization must remain distinguishable from durable intent conflicts" + ); + assert!( + decommission_capacity_retry_kind(&blocked, 0).is_none(), + "target contention must be retried inside the current version instead of replaying the entry" + ); + + let cancel_store = Arc::clone(&store); + let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel) + .await + .expect_err("terminal reservation release must wait for prepared target mutations"); + assert!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .is_some_and(DecommissionCapacityReservation::active), + "cancellation must not release the reservation while target mutations are active" + ); + + first_release_tx.send(()).expect("release first target mutation"); + second_release_tx.send(()).expect("release second target mutation"); + first + .await + .expect("first target mutation task should not panic") + .expect("first target mutation should finalize"); + second + .await + .expect("second target mutation task should not panic") + .expect("second target mutation should finalize"); + cancel + .await + .expect("decommission cancellation task should not panic") + .expect("decommission cancellation should finish after target mutations"); + + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("released capacity reservation should remain available for accounting"); + assert!(!reservation.active()); + assert_eq!(reservation.release_reason.as_deref(), Some(DECOMMISSION_CAPACITY_RELEASE_CANCELED)); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 2); + assert_eq!( + reservation + .targets + .iter() + .map(|target| (target.pool_index, target.consumed_physical_bytes)) + .collect::>(), + vec![(1, 1), (2, 1)] + ); + } + #[tokio::test] #[serial_test::serial] async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() { @@ -15344,7 +17113,7 @@ mod tests { let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)]; { let pool_meta = store.pool_meta.read().await; - ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight) + ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight, true) .expect("the pre-lock capacity snapshot should fit exactly"); } @@ -15722,6 +17491,22 @@ mod tests { .expect_err("half-enabled rollout gates must not admit decommission"); assert!(matches!(err, Error::InvalidArgument(..))); assert!(err.to_string().contains("durable unresolved-entry recovery")); + assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_WRITE)); + assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED)); + } + + #[test] + fn decommission_capacity_accepts_v3_only_writer_rollout() { + assert!(decommission_capacity_writer_supported_for( + POOL_META_V1_VERSION, + false, + pool_meta_v3_writer_enabled_for(true, true), + )); + assert!(!decommission_capacity_writer_supported_for( + POOL_META_V1_VERSION, + false, + pool_meta_v3_writer_enabled_for(true, false), + )); } #[test] @@ -16295,6 +18080,11 @@ mod tests { assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull))); assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown))); + let gate_busy = decommission_capacity_blocked_error(format!( + "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}" + )); + assert_eq!(decommission_capacity_target_gate_busy_index(&wrap(gate_busy)), Some(7)); + // Cleanup safety: a not-found surfacing from inside a stage is the same // condition as one surfacing directly, so the source entry stays // eligible for cleanup. @@ -16326,6 +18116,15 @@ mod tests { ); assert!(is_decommission_capacity_intent_conflict(&err)); + assert_eq!( + decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - 1), + Some(DecommissionCapacityRetryKind::IntentConflict) + ); + assert_eq!( + decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS), + None, + "a durable intent conflict must retain its bounded recovery policy" + ); } #[test] @@ -17402,8 +19201,9 @@ mod pools_tests { use super::resolve_decommission_listing_error; use super::resolve_decommission_partial_listing_entry; use super::{ - DECOMMISSION_CAPACITY_RELEASE_CANCELED, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, - DECOMMISSION_CAPACITY_RESERVATION_TTL, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, DECOMMISSION_CAPACITY_MODEL_VERSION, DECOMMISSION_CAPACITY_RELEASE_CANCELED, + DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, DECOMMISSION_CAPACITY_RESERVATION_TTL, + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget, @@ -17413,14 +19213,15 @@ mod pools_tests { POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, - build_decommission_capacity_reservation, cancel_decommission_canceler, clamp_decommission_entry_concurrency, - classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, - decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, - decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, - decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_delay, - decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices, - default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue, - enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available, + build_decommission_capacity_reservation, build_decommission_capacity_reservation_with_model, + cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, + count_decommission_item, decommission_cancel_signal_result, decommission_durable_ilm_receipt_path, + decommission_durable_ilm_receipt_run_prefix, decommission_durable_ilm_receipt_run_token, + decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, + decommission_physical_pool_capacity, decommission_retry_backoff_delay, decommission_start_pool_state, + decommission_unresolved_listing_error, dedup_indices, default_decommission_bucket_concurrency, + default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, + ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available, ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, @@ -17445,22 +19246,23 @@ mod pools_tests { resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices, rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect, - save_pool_meta_object_cas, should_cleanup_decommission_source_entry, should_continue_decommission_queue, - should_count_decommission_version_complete, should_fail_decommission_pool_after_exhausted_source_changed, - should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, - should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, - spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler, - take_decommission_canceler, track_decommission_current_object, track_decommission_current_object_stage, - update_decommission_for_operation, validate_start_decommission_request, wait_decommission_retry_backoff, - wait_decommission_worker_drain, with_decommission_entry_context, + save_pool_meta_object_cas, select_decommission_capacity_model, should_cleanup_decommission_source_entry, + should_continue_decommission_queue, should_count_decommission_version_complete, + should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state, + should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, + should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers, + split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler, + track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation, + validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain, + with_decommission_entry_context, }; use super::{ - DecommissionCapacityOwner, DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, - decommission_capacity_mutation_id, ensure_decommission_target_owner_admission, + DecommissionCapacityOwner, DecommissionCapacityReleaseProof, DecommissionCapacityReservation, + DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id, ensure_decommission_target_owner_admission, ensure_exact_delete_capacity_namespace_fences, ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error, plan_exact_delete_capacity_reconciliations, - record_decommission_target_consumption, reserve_decommission_target_pending, resolve_decommission_target_pending, - set_decommission_capacity_info_overrides_for_test, + record_decommission_target_consumption, release_decommission_target_inflight, reserve_decommission_target_pending, + resolve_decommission_target_pending, set_decommission_capacity_info_overrides_for_test, }; use crate::bucket::lifecycle::{ DurableIlmRecordCheckpoint, @@ -17545,7 +19347,6 @@ mod pools_tests { decommission_cancelers: tokio::sync::RwLock::new(cancelers), start_gate: tokio::sync::Mutex::new(()), pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()), - decommission_capacity_entry_gate: tokio::sync::Mutex::default(), ctx, bucket_fence_registry: Arc::default(), }) @@ -18156,6 +19957,89 @@ mod pools_tests { } } + fn decommission_test_active_model_meta(model_versions: &[u16]) -> PoolMeta { + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + PoolMeta { + version: POOL_META_VERSION, + pools: model_versions + .iter() + .copied() + .enumerate() + .map(|(idx, model_version)| { + let reservation = build_decommission_capacity_reservation_with_model( + DecommissionPoolCapacityInfo::for_test(idx, layout, 0, 10, 10), + layout, + uuid::Uuid::new_v4(), + u64::try_from(idx).unwrap_or_default() + 1, + OffsetDateTime::UNIX_EPOCH, + model_version, + ) + .expect("test capacity model should be supported"); + decommission_test_pool_status( + idx, + Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + capacity_reservation: Some(reservation), + ..Default::default() + }), + ) + }) + .collect(), + ..Default::default() + } + } + + fn decommission_test_cleanup_meta( + model_version: u16, + temporary_mutations: Vec, + inflight_physical_bytes: usize, + pending: Option<(uuid::Uuid, usize)>, + ) -> PoolMeta { + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let mut reservation = build_decommission_capacity_reservation_with_model( + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100), + layout, + uuid::Uuid::new_v4(), + 1, + OffsetDateTime::UNIX_EPOCH, + model_version, + ) + .expect("test cleanup reservation should be valid"); + let (pending_mutation_id, pending_physical_bytes) = pending.unzip(); + let pending_physical_bytes = pending_physical_bytes.unwrap_or_default(); + reservation.observed_target_physical_bytes = inflight_physical_bytes; + reservation.inflight_target_physical_bytes = inflight_physical_bytes; + reservation.pending_target_physical_bytes = pending_physical_bytes; + reservation.targets.push(DecommissionCapacityTarget { + pool_index: 1, + layout, + physical_total_at_reservation: 200, + physical_free_at_reservation: 200, + reserved_physical_bytes: reservation.peak_physical_bytes, + consumed_physical_bytes: 0, + observed_physical_bytes: inflight_physical_bytes, + inflight_physical_bytes, + pending_physical_bytes, + pending_mutation_id, + temporary_mutations, + }); + PoolMeta { + version: POOL_META_VERSION, + pools: vec![ + decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + capacity_reservation: Some(reservation), + ..Default::default() + }), + ), + decommission_test_pool_status(1, None), + ], + ..Default::default() + } + } + #[tokio::test] async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() { let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); @@ -21277,6 +23161,55 @@ mod pools_tests { assert!(err.to_string().contains("at least one active pool must remain")); } + #[test] + fn decommission_capacity_model_selection_uses_v2_only_with_a_live_fleet_proof() { + let meta = PoolMeta::default(); + + assert_eq!( + select_decommission_capacity_model(&meta, false).expect("a proofless empty cohort should remain compatible"), + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION + ); + assert_eq!( + select_decommission_capacity_model(&meta, true).expect("an all-v4 proof should authorize the target fence"), + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ); + } + + #[test] + fn decommission_capacity_model_selection_keeps_an_active_v1_cohort_sticky() { + let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION]); + + assert_eq!( + select_decommission_capacity_model(&meta, true).expect("a fleet upgrade must not change an active v1 lock model"), + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION + ); + } + + #[test] + fn decommission_capacity_model_selection_never_downgrades_an_active_v2_cohort() { + let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION]); + + assert_eq!( + select_decommission_capacity_model(&meta, true).expect("a live proof should admit another v2 reservation"), + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION + ); + let err = select_decommission_capacity_model(&meta, false) + .expect_err("proof loss must block new admission instead of falling back to the global lock model"); + assert!(err.to_string().contains("active per-target capacity cohort")); + } + + #[test] + fn decommission_capacity_model_selection_rejects_a_mixed_active_cohort() { + let meta = decommission_test_active_model_meta(&[ + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + ]); + + let err = select_decommission_capacity_model(&meta, true) + .expect_err("mixed global and per-target lock models cannot be made safe by a fleet proof"); + assert!(err.to_string().contains("mixed lock models")); + } + #[test] fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() { let meta = PoolMeta { @@ -21289,7 +23222,7 @@ mod pools_tests { DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400), ]; - assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos).is_ok()); + assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true).is_ok()); } #[test] @@ -21304,7 +23237,7 @@ mod pools_tests { DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401), ]; - let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos) + let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true) .expect_err("target physical free capacity below the modeled peak should be rejected"); assert!(err.to_string().contains("insufficient reserved physical target capacity")); @@ -21334,7 +23267,7 @@ mod pools_tests { DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401), ]; - let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos) + let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true) .expect_err("completed pools must not contribute target free capacity"); assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available")); @@ -21391,8 +23324,16 @@ mod pools_tests { meta.decommission(0, capacity_infos[0].space).unwrap(); meta.queue_decommission(1, capacity_infos[1].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0, 1], &capacity_infos, operation_id, 17, now) - .expect("the batch reservation should fit exactly"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0, 1], + &capacity_infos, + operation_id, + 17, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the batch reservation should fit exactly"); for idx in [0, 1] { let reservation = meta.pools[idx] .decommission @@ -21435,8 +23376,16 @@ mod pools_tests { ..Default::default() }; meta.decommission(0, initial[0].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) - .expect("the initial target capacity should fit exactly"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &initial, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the initial target capacity should fit exactly"); let mutation_id = uuid::Uuid::from_u128(1); reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1)) @@ -21486,8 +23435,16 @@ mod pools_tests { ..Default::default() }; meta.decommission(0, initial[0].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) - .expect("the initial target capacity should fit exactly"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &initial, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the initial target capacity should fit exactly"); let first_mutation_id = uuid::Uuid::from_u128(1); let second_mutation_id = uuid::Uuid::from_u128(2); @@ -21545,8 +23502,16 @@ mod pools_tests { ..Default::default() }; meta.decommission(0, initial[0].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) - .expect("the initial target capacity should fit exactly"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &initial, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the initial target capacity should fit exactly"); let mutation_id = uuid::Uuid::from_u128(1); assert_eq!( @@ -21608,8 +23573,16 @@ mod pools_tests { ..Default::default() }; meta.decommission(0, capacity_infos[0].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0], &capacity_infos, uuid::Uuid::new_v4(), 1, now) - .expect("the exact-delete test reservation should fit"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &capacity_infos, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the exact-delete test reservation should fit"); let exact = ObjectInfo { bucket: "bucket".to_string(), name: "object".to_string(), @@ -21737,8 +23710,16 @@ mod pools_tests { ..Default::default() }; meta.decommission(0, capacity_infos[0].space).unwrap(); - reserve_decommission_start_target_capacity(&mut meta, &[0], &capacity_infos, uuid::Uuid::new_v4(), 1, now) - .expect("the decommission reservation should fit"); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &capacity_infos, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("the decommission reservation should fit"); assert!( matches!( @@ -21892,6 +23873,360 @@ mod pools_tests { assert_eq!(restored, target); } + #[test] + fn temporary_cleanup_releases_only_its_exact_scoped_mutation_and_is_idempotent() { + let mutation_id = uuid::Uuid::new_v4(); + let foreign_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + vec![ + DecommissionCapacityTemporaryMutation { + mutation_id, + physical_bytes: 10, + }, + DecommissionCapacityTemporaryMutation { + mutation_id: foreign_id, + physical_bytes: 20, + }, + ], + 30, + None, + ); + + assert!( + release_decommission_target_inflight( + &mut meta, + 0, + 1, + 0, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("confirmed absence should release the exact mutation") + ); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(reservation.inflight_target_physical_bytes, 20); + assert_eq!( + reservation.targets[0].temporary_mutations, + vec![DecommissionCapacityTemporaryMutation { + mutation_id: foreign_id, + physical_bytes: 20, + }] + ); + + let before_replay = reservation.clone(); + assert!( + !release_decommission_target_inflight( + &mut meta, + 0, + 1, + 0, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(2), + ) + .expect("repeated confirmed absence should be a metadata no-op") + ); + let after_replay = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(after_replay, &before_replay); + } + + #[test] + fn temporary_cleanup_uses_aggregate_fallback_only_for_legacy_unscoped_state() { + let mutation_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 30, None); + + assert!( + release_decommission_target_inflight( + &mut meta, + 0, + 1, + 12, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("legacy unscoped cleanup should use its observed release delta") + ); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("legacy cleanup reservation should remain present"); + assert_eq!(reservation.inflight_target_physical_bytes, 18); + assert_eq!(reservation.targets[0].inflight_physical_bytes, 18); + } + + #[test] + fn temporary_cleanup_clears_only_a_matching_pending_identity_after_confirmed_absence() { + let mutation_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + Vec::new(), + 0, + Some((mutation_id, 15)), + ); + + assert!( + release_decommission_target_inflight( + &mut meta, + 0, + 1, + 7, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("a confirmed absent upload should clear its matching pending identity") + ); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.targets[0].pending_mutation_id, None); + assert_eq!(reservation.targets[0].pending_physical_bytes, 0); + } + + #[test] + fn temporary_cleanup_preserves_pending_for_an_already_published_target() { + let mutation_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + vec![DecommissionCapacityTemporaryMutation { + mutation_id, + physical_bytes: 5, + }], + 5, + Some((mutation_id, 15)), + ); + + assert!( + release_decommission_target_inflight( + &mut meta, + 0, + 1, + 0, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(false), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("published-target cleanup should release only its temporary staging state") + ); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.pending_target_physical_bytes, 15); + assert_eq!(reservation.targets[0].pending_mutation_id, Some(mutation_id)); + assert_eq!(reservation.targets[0].pending_physical_bytes, 15); + assert!(reservation.targets[0].temporary_mutations.is_empty()); + } + + #[test] + fn published_pending_ignores_a_delayed_release_observation() { + let mutation_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + Vec::new(), + 0, + Some((mutation_id, 15)), + ); + let before = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("published-target reservation should remain present") + .clone(); + + assert!( + !release_decommission_target_inflight( + &mut meta, + 0, + 1, + 7, + mutation_id, + DecommissionCapacityReleaseProof::confirmed_absence(false), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("a delayed statfs release must not block published-target reconciliation") + ); + let after = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("published-target reservation should remain present"); + assert_eq!(after, &before); + } + + #[test] + fn temporary_cleanup_fails_closed_on_a_foreign_scoped_release_delta() { + let foreign_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + vec![DecommissionCapacityTemporaryMutation { + mutation_id: foreign_id, + physical_bytes: 20, + }], + 20, + None, + ); + let before = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present") + .clone(); + + let err = release_decommission_target_inflight( + &mut meta, + 0, + 1, + 5, + uuid::Uuid::new_v4(), + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect_err("an observed release cannot be charged to a foreign scoped mutation"); + assert!(err.to_string().contains("cannot be attributed")); + let after = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(after, &before); + } + + #[test] + fn temporary_cleanup_missing_identity_and_nonzero_delta_is_a_metadata_noop() { + let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None); + let before = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present") + .clone(); + + assert!( + !release_decommission_target_inflight( + &mut meta, + 0, + 1, + 7, + uuid::Uuid::new_v4(), + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect("confirmed absence without tracked state should ignore a delayed capacity observation") + ); + let after = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(after, &before); + } + + #[test] + fn temporary_cleanup_missing_identity_fails_closed_on_foreign_pending_state() { + let foreign_id = uuid::Uuid::new_v4(); + let mut meta = decommission_test_cleanup_meta( + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + Vec::new(), + 0, + Some((foreign_id, 11)), + ); + let before = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present") + .clone(); + + let err = release_decommission_target_inflight( + &mut meta, + 0, + 1, + 7, + uuid::Uuid::new_v4(), + DecommissionCapacityReleaseProof::confirmed_absence(true), + OffsetDateTime::UNIX_EPOCH + Duration::seconds(1), + ) + .expect_err("a delayed release observation must not clear a foreign pending mutation"); + assert!(err.to_string().contains("cannot be attributed")); + let after = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup reservation should remain present"); + assert_eq!(after, &before); + } + + #[test] + fn pool_meta_v2_round_trip_preserves_each_supported_capacity_lock_model() { + for model_version in [ + DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, + DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, + ] { + let meta = decommission_test_cleanup_meta(model_version, Vec::new(), 0, None); + let encoded = meta + .encode_config_data_for_test() + .expect("supported capacity lock model should encode in pool metadata V2"); + let mut restored = PoolMeta::default(); + restored + .load_from_config_data(encoded) + .expect("supported capacity lock model should decode from pool metadata V2"); + assert_eq!( + restored.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("round-tripped reservation should remain present") + .model_version, + model_version + ); + } + } + + #[test] + fn pool_meta_v2_decode_rejects_mixed_active_capacity_lock_models() { + let mut mixed = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 0, None); + let mut target_fence_source = + decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None) + .pools + .remove(0); + target_fence_source.id = 2; + target_fence_source.cmd_line = "pool-2".to_string(); + target_fence_source + .decommission + .as_mut() + .and_then(|info| info.capacity_reservation.as_mut()) + .expect("second active reservation should remain present") + .source_pool_index = 2; + mixed.pools.push(target_fence_source); + + let encoded = mixed + .encode_config_data_for_test() + .expect("the decode test must be able to construct a mixed persisted payload"); + let mut restored = PoolMeta::default(); + let err = restored + .load_from_config_data(encoded) + .expect_err("mixed active lock models must fail closed while loading pool metadata"); + assert!(err.to_string().contains("mixed lock models")); + } + #[test] fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() { let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1); @@ -21984,7 +24319,7 @@ mod pools_tests { DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), ]; - let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now) + let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now, true) .expect("restart recovery should rebuild the missing reservation while capacity still fits"); assert_eq!(recovered_indices, vec![0]); let reservation = meta.pools[0] diff --git a/crates/ecstore/src/core/pools_test.rs b/crates/ecstore/src/core/pools_test.rs index 62fa14252..aa6c5e92a 100644 --- a/crates/ecstore/src/core/pools_test.rs +++ b/crates/ecstore/src/core/pools_test.rs @@ -195,8 +195,9 @@ mod capacity_dedup_tests { mod decommission_lock_order_tests { use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING; use crate::core::pools::{ - DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, DecommissionErasureLayout, DecommissionPoolCapacityInfo, - POOL_META_NAME, decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test, + DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX, DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, + DecommissionErasureLayout, DecommissionPoolCapacityInfo, DecommissionTestFaultGuard, POOL_META_NAME, + decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test, }; use crate::data_movement; use crate::disk::RUSTFS_META_BUCKET; @@ -215,13 +216,72 @@ mod decommission_lock_order_tests { use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations as _}; use http::HeaderMap; + use rustfs_filemeta::MetaCacheEntry; use std::collections::HashMap; + use std::future::Future; use std::sync::{ - Arc, + Arc, Condvar, Mutex as StdMutex, atomic::{AtomicUsize, Ordering}, }; use std::time::Duration; use tokio::io::AsyncReadExt; + use tokio_util::sync::CancellationToken; + + fn run_large_stack_async_test(name: &str, case: C) + where + C: FnOnce() -> F + Send + 'static, + F: Future + 'static, + { + const STACK_SIZE: usize = if cfg!(debug_assertions) { + 8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else if cfg!(target_os = "macos") { + 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else { + rustfs_config::DEFAULT_THREAD_STACK_SIZE + }; + std::thread::Builder::new() + .name(name.to_string()) + .stack_size(STACK_SIZE) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .thread_stack_size(STACK_SIZE) + .build() + .expect("large-stack ecstore test runtime should build"); + runtime.block_on(case()); + }) + .expect("large-stack ecstore test thread should spawn") + .join() + .expect("large-stack ecstore test thread should complete"); + } + + fn run_large_stack_current_thread_async_test(name: &str, case: C) + where + C: FnOnce() -> F + Send + 'static, + F: Future + 'static, + { + const STACK_SIZE: usize = if cfg!(debug_assertions) { + 8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else if cfg!(target_os = "macos") { + 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else { + rustfs_config::DEFAULT_THREAD_STACK_SIZE + }; + std::thread::Builder::new() + .name(name.to_string()) + .stack_size(STACK_SIZE) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("large-stack current-thread ecstore test runtime should build"); + runtime.block_on(case()); + }) + .expect("large-stack current-thread ecstore test should spawn") + .join() + .expect("large-stack current-thread ecstore test should complete"); + } #[derive(Clone, Copy)] enum ExternalObjectMutation { @@ -308,11 +368,20 @@ mod decommission_lock_order_tests { #[derive(Debug)] struct CapacityLeaseLossClient { - target: rustfs_lock::ObjectKey, control: Arc, active: tokio::sync::Mutex>, } + fn is_capacity_lease_resource(resource: &rustfs_lock::ObjectKey) -> bool { + resource.bucket.as_ref() == RUSTFS_META_BUCKET + && (resource.object.as_ref() == POOL_META_NAME + || resource + .object + .as_ref() + .strip_prefix(DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX) + .is_some_and(|suffix| suffix.starts_with('/'))) + } + #[async_trait::async_trait] impl rustfs_lock::LockClient for CapacityLeaseLossClient { async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result { @@ -345,7 +414,7 @@ mod decommission_lock_order_tests { async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { let resource = self.active.lock().await.get(lock_id).cloned(); - if resource.as_ref() == Some(&self.target) { + if resource.as_ref().is_some_and(is_capacity_lease_resource) { self.control.calls.fetch_add(1, Ordering::Release); return Ok(!self.control.fail_refresh.load(Ordering::Acquire)); } @@ -389,7 +458,6 @@ mod decommission_lock_order_tests { set.lockers = (0..set.lockers.len().max(1)) .map(|_| { Arc::new(CapacityLeaseLossClient { - target: rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, POOL_META_NAME), control: Arc::clone(&refresh_calls), active: tokio::sync::Mutex::new(HashMap::new()), }) as Arc @@ -422,7 +490,6 @@ mod decommission_lock_order_tests { pool_meta_save_gate: tokio::sync::Mutex::new( other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(), ), - decommission_capacity_entry_gate: tokio::sync::Mutex::default(), ctx, bucket_fence_registry: Arc::default(), }); @@ -1277,9 +1344,368 @@ mod decommission_lock_order_tests { assert_same_object_copy_fences_capacity_lease_loss(true).await; } + #[test] + #[serial_test::serial] + fn data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker() { + run_large_stack_async_test( + "multipart-zero-delta-restart", + data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker_case, + ); + } + + async fn data_movement_multipart_restart_cleans_exact_zero_delta_upload_with_recovery_marker_case() { + let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-zero-delta-restart"); + let object = "staged-without-statfs-delta.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the zero-delta multipart restart bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the zero-delta multipart bucket incarnation"); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 8, 8), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 16, 16), + DecommissionPoolCapacityInfo::for_test(2, layout, 32, 32, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, (0..8).map(|_| capacity_snapshot()).collect()); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the zero-delta multipart reservation"); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| reservation.targets.first()) + .expect("zero-delta fixture should reserve one target") + .pool_index; + assert_eq!(target_pool_index, 2); + + let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); + let exact_version = uuid::Uuid::new_v4(); + let foreign_version = uuid::Uuid::new_v4(); + let exact_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(11); + let foreign_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(12); + let exact_identity = format!("v1:{exact_version}:{}", exact_mod_time.unix_timestamp_nanos()); + let foreign_identity = format!("v1:{foreign_version}:{}", foreign_mod_time.unix_timestamp_nanos()); + let upload_opts = |identity: &str, version_id: uuid::Uuid, mod_time: time::OffsetDateTime| { + let mut opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + rustfs_utils::http::insert_str( + &mut opts.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + identity.to_string(), + ); + owner.apply_to(&mut opts); + opts + }; + let mut cleanup_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(exact_version.to_string()), + mod_time: Some(exact_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut cleanup_opts); + + store.reset_data_movement_multipart_discovery_count_for_test(); + store + .reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts) + .await + .expect("a fresh mutation without recovery state should take the metadata fast path"); + assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 0); + + let exact_upload_opts = upload_opts(&exact_identity, exact_version, exact_mod_time); + let staging_store = Arc::clone(&store); + let staging_bucket = bucket.clone(); + let exact_err = store + .run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), Some(1), || async move { + new_multipart_upload(&staging_store, target_pool_index, &staging_bucket, object, exact_upload_opts).await?; + Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled) + }) + .await + .expect_err("the fixture should fail after staging the exact zero-delta upload"); + assert!(matches!(exact_err, crate::error::Error::OperationCanceled)); + let foreign_upload = new_multipart_upload( + &store, + target_pool_index, + &bucket, + object, + upload_opts(&foreign_identity, foreign_version, foreign_mod_time), + ) + .await + .expect("stage the foreign zero-delta upload"); + + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("zero-delta reservation should remain active"); + assert_eq!(reservation.pending_target_physical_bytes, 1); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id); + assert_eq!(reservation.targets[0].temporary_mutations.len(), 1); + assert_eq!(reservation.targets[0].temporary_mutations[0].mutation_id, owner.mutation_id.unwrap()); + assert_eq!(reservation.targets[0].temporary_mutations[0].physical_bytes, 0); + } + + store + .reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts) + .await + .expect("restart should remove the exact upload through its zero-byte recovery marker"); + assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 1); + + let set = store.pools[target_pool_index].get_disks_by_key(object); + let exact_remaining = set + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &exact_identity) + .await + .expect("scan for the exact zero-delta upload after reconciliation"); + assert!(exact_remaining.is_empty(), "the exact stale upload must be removed"); + let foreign_remaining = set + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &foreign_identity) + .await + .expect("scan for the foreign upload after reconciliation"); + assert_eq!( + foreign_remaining, + vec![foreign_upload.upload_id], + "identity-scoped cleanup must preserve a foreign upload" + ); + + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("zero-delta reconciliation must preserve the reservation"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert!(reservation.targets[0].temporary_mutations.is_empty()); + + drop(pool_meta); + store + .reconcile_multipart_uploads_for_data_movement(target_pool_index, &bucket, object, &exact_identity, &cleanup_opts) + .await + .expect("repeated cleanup should take the fast path after removing the marker"); + assert_eq!(store.data_movement_multipart_discovery_count_for_test(), 1); + } + #[tokio::test] #[serial_test::serial] - async fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload() { + async fn multipart_cleanup_rediscovers_uploads_and_proves_target_under_capacity_gate() { + let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-cleanup-target-race"); + let object = "published-between-cleanup-proof-and-finalize.bin"; + let body = vec![0x5a; 64 * 1024]; + let version_id = uuid::Uuid::new_v4(); + let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(19); + let upload_identity = format!("v1:{version_id}:{}", mod_time.unix_timestamp_nanos()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the cleanup target-race bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the cleanup target-race bucket incarnation"); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, body.len().saturating_mul(2), body.len().saturating_mul(2)), + DecommissionPoolCapacityInfo::for_test(2, layout, body.len().saturating_mul(2), body.len().saturating_mul(2), 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the cleanup target-race reservation"); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| reservation.targets.first()) + .expect("cleanup target-race fixture should reserve one target") + .pool_index; + assert_eq!(target_pool_index, 2); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); + let mut upload_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + rustfs_utils::http::insert_str( + &mut upload_opts.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + upload_identity.clone(), + ); + owner.apply_to(&mut upload_opts); + let concurrent_upload_opts = upload_opts.clone(); + let mut cleanup_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut cleanup_opts); + + let staging_store = Arc::clone(&store); + let staging_bucket = bucket.clone(); + store + .run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), None, || async move { + new_multipart_upload(&staging_store, target_pool_index, &staging_bucket, object, upload_opts).await?; + Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled) + }) + .await + .expect_err("the fixture should fail after staging its recoverable upload"); + store + .run_decommission_capacity_temporary_mutation(target_pool_index, Some(owner), Some(body.len()), || async { + Err::<(), crate::error::Error>(crate::error::Error::OperationCanceled) + }) + .await + .expect_err("the fixture should retain an exact pending capacity intent"); + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup target-race reservation should remain active"); + assert!(reservation.pending_target_physical_bytes > 0); + assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id); + assert!( + reservation.targets[0] + .temporary_mutations + .iter() + .any(|mutation| mutation.mutation_id == owner.mutation_id.unwrap()) + ); + } + + let target_lock = store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the cleanup target-race gate"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the cleanup target-race gate"); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.disable_owner_pause(); + barrier.pause_target_gate_acquire(target_pool_index); + let mut cleanup = tokio::spawn({ + let cleanup_store = Arc::clone(&store); + let cleanup_bucket = bucket.clone(); + let cleanup_identity = upload_identity.clone(); + async move { + cleanup_store + .reconcile_multipart_uploads_for_data_movement( + target_pool_index, + &cleanup_bucket, + object, + &cleanup_identity, + &cleanup_opts, + ) + .await + } + }); + tokio::select! { + _ = barrier.wait_until_target_gate_acquire_paused() => {} + result = &mut cleanup => panic!("cleanup finished before its target-gate proof point: {result:?}"), + } + + let concurrent_upload = new_multipart_upload(&store, target_pool_index, &bucket, object, concurrent_upload_opts) + .await + .expect("stage a same-identity upload after cleanup reaches its target-gate boundary"); + let mut target_data = PutObjReader::from_vec(body); + let published = store.pools[target_pool_index] + .put_object( + &bucket, + object, + &mut target_data, + &ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("publish the exact owned target while cleanup waits for its capacity gate"); + assert!(crate::data_movement::is_owned_data_movement_target(&published)); + drop(target_guard); + barrier.release_target_gate_acquire(); + tokio::time::timeout(Duration::from_secs(30), cleanup) + .await + .expect("cleanup should finish after acquiring its target gate") + .expect("cleanup task should not panic") + .expect("cleanup should preserve the now-published target intent"); + drop(barrier); + + let remaining_uploads = store.pools[target_pool_index] + .get_disks_by_key(object) + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity) + .await + .expect("check same-identity uploads after gated cleanup"); + assert!( + remaining_uploads.is_empty(), + "gated cleanup must rediscover and remove the concurrent same-identity upload {}", + concurrent_upload.upload_id + ); + + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("cleanup target-race reservation should remain active"); + assert!( + reservation.pending_target_physical_bytes > 0, + "cleanup must not clear capacity for a target published before its gated proof" + ); + assert_eq!(reservation.targets[0].pending_mutation_id, owner.mutation_id); + assert!(reservation.targets[0].temporary_mutations.is_empty()); + } + + #[test] + #[serial_test::serial] + fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload() { + run_large_stack_current_thread_async_test( + "multipart-published-capacity-restart", + data_movement_multipart_restart_reconciles_published_capacity_before_new_upload_case, + ); + } + + async fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload_case() { let (_temp_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; let bucket = test_bucket("multipart-restart"); @@ -1478,7 +1904,12 @@ mod decommission_lock_order_tests { .as_ref() .and_then(|info| info.capacity_reservation.as_ref()) .expect("the failed multipart capacity intent must remain durable"); - assert!(failed_reservation.pending_target_physical_bytes > 0); + assert_eq!( + failed_reservation.pending_target_physical_bytes, + body.len(), + "the published target must retain its durable capacity intent for restart reconciliation" + ); + assert_eq!(failed_reservation.inflight_target_physical_bytes, 0); assert_eq!(failed_reservation.consumed_target_physical_bytes, 0); *other_store.pool_meta.write().await = failed_persisted.clone(); let retry_owner = decommission_capacity_owner(&failed_persisted); @@ -1602,9 +2033,16 @@ mod decommission_lock_order_tests { ); } - #[tokio::test] + #[test] #[serial_test::serial] - async fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry() { + fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry() { + run_large_stack_current_thread_async_test( + "multipart-partial-upload-restart", + data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry_case, + ); + } + + async fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry_case() { let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; let bucket = test_bucket("multipart-part-restart"); let object = "published-part-before-capacity-save.bin"; @@ -1784,7 +2222,7 @@ mod decommission_lock_order_tests { .expect("restart should reread the exact partial multipart source"); let new_upload = NewMultipartUploadCommitObservation::install(&bucket, object); let retry_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::NewUploadBeforeLockLost); - let retry = tokio::spawn({ + let mut retry = tokio::spawn({ let retry_store = Arc::clone(&other_store); let retry_bucket = bucket.clone(); async move { @@ -1800,7 +2238,12 @@ mod decommission_lock_order_tests { .await } }); - retry_barrier.wait_until_paused().await; + tokio::select! { + _ = retry_barrier.wait_until_paused() => {} + result = &mut retry => { + panic!("partial multipart retry completed before reaching the new-upload barrier: {result:?}"); + } + } let uploads_before_retry = other_store.pools[2] .get_disks_by_key(object) .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity) @@ -1895,18 +2338,22 @@ mod decommission_lock_order_tests { .await .expect("activate the abort fence reservation"); let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); + let version_id = uuid::Uuid::new_v4().to_string(); + let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(83); let mut upload_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, versioned: true, - version_id: Some(uuid::Uuid::new_v4().to_string()), + version_id: Some(version_id.clone()), + mod_time: Some(mod_time), expected_bucket_incarnation_id: Some(incarnation), ..Default::default() }; + let upload_identity = data_movement::data_movement_upload_identity_from_options(&upload_opts); rustfs_utils::http::insert_str( &mut upload_opts.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, - "v1:abort-fence:1".to_string(), + upload_identity, ); let upload = new_multipart_upload(&store, 2, &bucket, object, upload_opts) .await @@ -1917,6 +2364,9 @@ mod decommission_lock_order_tests { let mut abort_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, + versioned: true, + version_id: Some(version_id), + mod_time: Some(mod_time), expected_bucket_incarnation_id: Some(incarnation), ..Default::default() }; @@ -1971,9 +2421,16 @@ mod decommission_lock_order_tests { assert!(upload_info.parts.is_empty()); } - #[tokio::test] + #[test] #[serial_test::serial] - async fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() { + fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() { + run_large_stack_current_thread_async_test( + "multipart-abort-restart", + data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss_case, + ); + } + + async fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss_case() { let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; let bucket = test_bucket("multipart-abort-restart"); let object = "deleted-upload-before-release-save.bin"; @@ -2002,15 +2459,18 @@ mod decommission_lock_order_tests { .await .expect("activate the abort restart reservation"); let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); - let upload_identity = format!("v1:{}:{}", uuid::Uuid::new_v4(), 91); + let version_id = uuid::Uuid::new_v4().to_string(); + let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(91); let mut new_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, versioned: true, - version_id: Some(uuid::Uuid::new_v4().to_string()), + version_id: Some(version_id.clone()), + mod_time: Some(mod_time), expected_bucket_incarnation_id: Some(incarnation), ..Default::default() }; + let upload_identity = data_movement::data_movement_upload_identity_from_options(&new_opts); rustfs_utils::http::insert_str( &mut new_opts.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, @@ -2080,6 +2540,9 @@ mod decommission_lock_order_tests { let mut abort_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, + versioned: true, + version_id: Some(version_id), + mod_time: Some(mod_time), expected_bucket_incarnation_id: Some(incarnation), ..Default::default() }; @@ -2624,18 +3087,35 @@ mod decommission_lock_order_tests { ); } - let capacity_lock = other_store - .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reserved_target}"), + ) .await - .expect("create the data-movement tail capacity probe"); - let mut capacity_probe = - tokio::spawn(async move { capacity_lock.get_write_lock(std::time::Duration::from_secs(30)).await }); + .expect("create the data-movement tail target-fence probe"); + let mut target_probe = tokio::spawn(async move { target_lock.get_write_lock(std::time::Duration::from_secs(30)).await }); assert!( - tokio::time::timeout(std::time::Duration::from_millis(100), &mut capacity_probe) + tokio::time::timeout(std::time::Duration::from_millis(100), &mut target_probe) .await .is_err(), - "data-movement completion must retain the capacity write lock while its tail is paused" + "data-movement completion must retain its target fence while the tail is paused" ); + target_probe.abort(); + let _ = target_probe.await; + + let pool_meta_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the data-movement tail pool metadata probe"); + let pool_meta_guard = tokio::time::timeout( + std::time::Duration::from_secs(1), + pool_meta_lock.get_write_lock(std::time::Duration::from_secs(30)), + ) + .await + .expect("per-target data movement must leave pool metadata available during target I/O") + .expect("pool metadata probe should acquire during the target tail"); + drop(pool_meta_guard); tail_barrier.release(); drop(tail_barrier); @@ -2644,12 +3124,21 @@ mod decommission_lock_order_tests { .expect("data-movement CompleteMultipartUpload should finish after its tail resumes") .expect("data-movement CompleteMultipartUpload task should not panic") .expect("data-movement CompleteMultipartUpload should commit after its tail resumes"); - let capacity_guard = tokio::time::timeout(std::time::Duration::from_secs(30), capacity_probe) + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reserved_target}"), + ) .await - .expect("capacity probe should finish after data-movement tail completion") - .expect("capacity probe should not panic") - .expect("capacity probe should acquire after data-movement tail completion"); - drop(capacity_guard); + .expect("create the post-tail target-fence probe"); + let target_guard = tokio::time::timeout( + std::time::Duration::from_secs(30), + target_lock.get_write_lock(std::time::Duration::from_secs(30)), + ) + .await + .expect("target-fence probe should finish after data-movement tail completion") + .expect("target-fence probe should acquire after data-movement tail completion"); + drop(target_guard); let meta = other_store.pool_meta.read().await; let reservation = meta.pools[0] @@ -2976,17 +3465,33 @@ mod decommission_lock_order_tests { tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(), "owner-bearing data-movement PUT must wait for the rename tail instead of early-ACKing" ); - let capacity_lock = other_store - .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reservation_target}"), + ) .await - .expect("create the data-movement PUT tail capacity probe"); - let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await }); + .expect("create the data-movement PUT tail target-fence probe"); + let mut target_probe = tokio::spawn(async move { target_lock.get_write_lock(Duration::from_secs(30)).await }); assert!( - tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe) + tokio::time::timeout(Duration::from_millis(100), &mut target_probe) .await .is_err(), - "data-movement PUT tail must retain its capacity fence" + "data-movement PUT tail must retain its target fence" ); + target_probe.abort(); + let _ = target_probe.await; + + let pool_meta_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the data-movement PUT tail pool metadata probe"); + let pool_meta_guard = + tokio::time::timeout(Duration::from_secs(1), pool_meta_lock.get_write_lock(Duration::from_secs(30))) + .await + .expect("per-target data movement must leave pool metadata available during target I/O") + .expect("pool metadata probe should acquire during the target tail"); + drop(pool_meta_guard); { let meta = other_store.pool_meta.read().await; let reservation = meta.pools[0] @@ -3026,12 +3531,18 @@ mod decommission_lock_order_tests { .expect("successful data-movement PUT should remain readable after its tail"); assert!(!target.delete_marker); - let capacity_guard = tokio::time::timeout(Duration::from_secs(30), capacity_probe) + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{reservation_target}"), + ) .await - .expect("capacity probe should finish after the data-movement PUT tail") - .expect("capacity probe should not panic") - .expect("capacity probe should acquire after the data-movement PUT tail"); - drop(capacity_guard); + .expect("create the post-tail data-movement target-fence probe"); + let target_guard = tokio::time::timeout(Duration::from_secs(30), target_lock.get_write_lock(Duration::from_secs(30))) + .await + .expect("target-fence probe should finish after the data-movement PUT tail") + .expect("target-fence probe should acquire after the data-movement PUT tail"); + drop(target_guard); let meta = other_store.pool_meta.read().await; let reservation = meta.pools[0] @@ -3047,6 +3558,752 @@ mod decommission_lock_order_tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn target_gate_retry_reloads_exact_source_identity_before_copying() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("target-gate-source-change"); + let object = "source-changes-while-target-busy.bin"; + let initial_body = vec![0x41; 64 * 1024]; + let replacement_body = vec![0x42; initial_body.len()]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the target-gate source-change bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the target-gate source-change bucket incarnation"); + let initial_mod_time = time::OffsetDateTime::now_utc(); + let mut initial_data = PutObjReader::from_vec(initial_body); + store.pools[0] + .put_object( + &bucket, + object, + &mut initial_data, + &ObjectOptions { + mod_time: Some(initial_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the initial source identity"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = replacement_body.len().saturating_mul(4); + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, replacement_body.len(), replacement_body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the target-gate source-change reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, replacement_body.len(), replacement_body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]], + ); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| reservation.targets.first()) + .expect("the source-change reservation should allocate a target") + .pool_index; + let in_memory_reservation = other_store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the source-change reservation should be published in memory") + .clone(); + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("the source-change reservation should reload from durable replicas"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the durable source-change reservation should remain present"); + assert_eq!( + ( + persisted_reservation.operation_id, + persisted_reservation.generation, + persisted_reservation.owner_nonce, + persisted_reservation.model_version, + persisted_reservation.targets.clone(), + ), + ( + in_memory_reservation.operation_id, + in_memory_reservation.generation, + in_memory_reservation.owner_nonce, + in_memory_reservation.model_version, + in_memory_reservation.targets.clone(), + ), + "the worker owner identity must match the durable reservation" + ); + assert!( + persisted_reservation.expires_at > time::OffsetDateTime::now_utc(), + "the durable reservation lease must remain active before the worker starts" + ); + + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the target-gate source-change lock"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the target gate before starting migration"); + let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + let source_set = other_store.pools[0].get_disks_by_key(object); + + let cancel_token = CancellationToken::new(); + let canceled_store = Arc::clone(&other_store); + let canceled_bucket = bucket.clone(); + let canceled_set = Arc::clone(&source_set); + let canceled_token = cancel_token.clone(); + let mut canceled_worker = tokio::spawn(async move { + canceled_store + .decommission_entry_with_retry_state_for_test( + canceled_token, + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + canceled_bucket, + canceled_set, + None, + Arc::new(AtomicUsize::new(0)), + ) + .await + }); + tokio::select! { + _ = retry_observer.wait_until_target_gate_retry() => {} + result = &mut canceled_worker => { + panic!("cancellation probe finished before observing target contention: {result:?}"); + } + } + tokio::time::sleep(Duration::from_millis(800)).await; + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 0, + "a long-held target gate must not trigger periodic exact quorum reloads" + ); + cancel_token.cancel(); + let canceled_err = tokio::time::timeout(Duration::from_secs(2), canceled_worker) + .await + .expect("target-gate wait should observe cancellation without waiting for release") + .expect("cancellation probe task should not panic") + .expect_err("the canceled target-gate wait must stop the entry"); + assert!( + crate::error::is_err_operation_canceled(&canceled_err), + "unexpected target-gate cancellation error: {canceled_err:?}" + ); + + let worker_store = Arc::clone(&other_store); + let worker_bucket = bucket.clone(); + let mut worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + tokio::select! { + _ = retry_observer.wait_until_target_gate_retry() => {} + result = &mut worker => { + panic!("decommission entry finished before observing target contention: {result:?}"); + } + } + + let mut replacement_data = PutObjReader::from_vec(replacement_body.clone()); + other_store.pools[0] + .put_object( + &bucket, + object, + &mut replacement_data, + &ObjectOptions { + mod_time: Some(initial_mod_time + time::Duration::seconds(1)), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("replace the source after the target-busy attempt releases its source snapshot"); + retry_observer.release_owner(); + drop(target_guard); + tokio::time::timeout(Duration::from_secs(30), worker) + .await + .expect("source-change retry should finish after releasing the target gate") + .expect("source-change retry task should not panic") + .expect("source-change retry should converge on the replacement identity"); + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 1, + "source identity should be reloaded exactly once after the target gate becomes available" + ); + drop(retry_observer); + + let mut target_reader = other_store.pools[target_pool_index] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("replacement source should be migrated to the reserved target"); + let mut target_body = Vec::new(); + target_reader + .stream + .read_to_end(&mut target_body) + .await + .expect("read the migrated replacement body"); + assert_eq!(target_body, replacement_body, "the stale pre-contention snapshot must never be copied"); + let source_err = other_store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("the converged replacement source should be removed after migration"); + assert!( + matches!(source_err, crate::error::Error::ObjectNotFound(_, _)), + "unexpected source state after replacement migration: {source_err:?}" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn target_gate_retry_pins_or_releases_permits_when_target_rank_changes() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("target-gate-two-waiters"); + let objects = ["first-waiter.bin", "second-waiter.bin"]; + let body = vec![0x54; 64 * 1024]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the two-waiter target-gate bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the two-waiter bucket incarnation"); + for object in objects { + let mut source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_data, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed a same-target waiter source"); + } + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source_bytes = body.len().saturating_mul(4); + let first_target_capacity = body.len().saturating_mul(5); + let second_target_capacity = body.len().saturating_mul(3); + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes), + DecommissionPoolCapacityInfo::for_test(1, layout, first_target_capacity, first_target_capacity, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, second_target_capacity, second_target_capacity, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the two-waiter capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]); + let capacity_owner = { + let pool_meta = other_store.pool_meta.read().await; + decommission_capacity_owner(&pool_meta) + }; + let target_pool_index = other_store + .select_decommission_capacity_target_pool(capacity_owner, body.len()) + .await + .expect("select the initially largest target allocation"); + let target_pool_indices = other_store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the two-waiter reservation should remain active") + .targets + .iter() + .map(|target| target.pool_index) + .collect::>(); + assert_eq!(target_pool_indices.len(), 2, "the rank-change fixture needs two eligible targets"); + let alternate_target_pool_index = target_pool_indices + .iter() + .copied() + .find(|pool_index| *pool_index != target_pool_index) + .expect("the rank-change fixture should have an alternate target"); + + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the two-waiter target gate"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the target gate before starting both waiters"); + let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + + let mut workers = Vec::new(); + for object in objects { + let worker_store = Arc::clone(&other_store); + let worker_bucket = bucket.clone(); + let source_set = other_store.pools[0].get_disks_by_key(object); + workers.push(tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + })); + } + retry_observer.wait_until_target_gate_retries(2).await; + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 0, + "waiters must not poll exact source metadata while the target gate is held" + ); + + let holder_consumed = body.len().saturating_add(1); + { + let mut pool_meta = other_store.pool_meta.write().await; + let source_pool = &mut pool_meta.pools[0]; + let reservation = source_pool + .decommission + .as_mut() + .and_then(|info| info.capacity_reservation.as_mut()) + .expect("the holder commit should update the active reservation"); + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .expect("the holder commit target should remain allocated"); + target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(holder_consumed); + target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(holder_consumed); + reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(holder_consumed); + reservation.consumed_target_physical_bytes = + reservation.consumed_target_physical_bytes.saturating_add(holder_consumed); + reservation.observed_target_physical_bytes = + reservation.observed_target_physical_bytes.saturating_add(holder_consumed); + reservation.prediction_error_bytes = 0; + source_pool.last_update = time::OffsetDateTime::now_utc(); + } + other_store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("persist the holder commit while it owns the original target gate"); + assert_eq!( + other_store + .select_decommission_capacity_target_pool(capacity_owner, body.len()) + .await + .expect("reselect after the holder commit"), + alternate_target_pool_index, + "the holder commit should flip the dynamic target ranking" + ); + + drop(target_guard); + retry_observer.wait_until_owner_paused().await; + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 1, + "the first waiter must retain its original target despite the changed ranking" + ); + let alternate_probe = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{alternate_target_pool_index}"), + ) + .await + .expect("create the alternate target-gate probe") + .get_write_lock_quiet(Duration::from_millis(300)) + .await + .expect("a pinned waiter must not hold the alternate target gate"); + drop(alternate_probe); + retry_observer.release_owner(); + retry_observer.wait_until_owner_paused().await; + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 2, + "the second waiter should reload exactly once before releasing its exhausted original permit" + ); + let original_probe = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the original target-gate probe") + .get_write_lock_quiet(Duration::from_millis(300)) + .await + .expect("a reselected waiter must release its original target gate before taking the alternate"); + drop(original_probe); + retry_observer.release_owner(); + + for worker in workers { + tokio::time::timeout(Duration::from_secs(30), worker) + .await + .expect("same-target waiter should finish after permit transfer") + .expect("same-target waiter task should not panic") + .expect("same-target waiter migration should succeed"); + } + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 2, + "formal mutations must consume their permits without re-entering target-busy recovery" + ); + drop(retry_observer); + + for object in objects { + let mut target_copies = 0; + for pool_index in &target_pool_indices { + if other_store.pools[*pool_index] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_ok() + { + target_copies += 1; + } + } + assert_eq!(target_copies, 1, "each waiter should publish on exactly one reserved target"); + let source_err = other_store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("each migrated waiter source should be removed"); + assert!(matches!(source_err, crate::error::Error::ObjectNotFound(_, _))); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial] + async fn later_version_target_busy_does_not_replay_an_earlier_version() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("target-gate-later-version"); + let object = "two-versions-one-busy.bin"; + let bodies = [vec![0x31; 64 * 1024], vec![0x32; 64 * 1024]]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the later-version target-gate bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the later-version bucket incarnation"); + for body in &bodies { + let mut source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_data, + &ObjectOptions { + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed a source version"); + } + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source_bytes = bodies.iter().map(Vec::len).sum::(); + let target_total = source_bytes.saturating_mul(4); + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the later-version capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| reservation.targets.first()) + .expect("the later-version reservation should allocate a target") + .pool_index; + + let successful_copies = Arc::new(AtomicUsize::new(0)); + let first_copy_ready = Arc::new(tokio::sync::Notify::new()); + let first_copy_release = Arc::new((StdMutex::new(false), Condvar::new())); + let hook_bucket = bucket.clone(); + let hook_object = object.to_string(); + let hook_copies = Arc::clone(&successful_copies); + let hook_ready = Arc::clone(&first_copy_ready); + let hook_release = Arc::clone(&first_copy_release); + let fault_guard = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _attempt, succeeded| { + if succeeded && stage == "migrate_object" && bucket == hook_bucket && object == hook_object { + let copy_number = hook_copies.fetch_add(1, Ordering::AcqRel) + 1; + if copy_number == 1 { + hook_ready.notify_one(); + let (released, wake) = &*hook_release; + let mut released = released.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while !*released { + released = wake.wait(released).unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + } + false + })); + + let worker_store = Arc::clone(&other_store); + let worker_bucket = bucket.clone(); + let source_set = other_store.pools[0].get_disks_by_key(object); + let mut worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), first_copy_ready.notified()) + .await + .expect("the first version should complete its target copy"); + + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the later-version target gate"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the target gate after the first version copy"); + let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + { + let (released, wake) = &*first_copy_release; + *released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; + wake.notify_one(); + } + tokio::select! { + _ = retry_observer.wait_until_target_gate_retry() => {} + result = &mut worker => panic!("later-version worker finished before target contention: {result:?}"), + } + assert_eq!( + successful_copies.load(Ordering::Acquire), + 1, + "the already copied version must not be replayed while the later version waits" + ); + + retry_observer.release_owner(); + drop(target_guard); + tokio::time::timeout(Duration::from_secs(30), worker) + .await + .expect("later-version worker should finish after target release") + .expect("later-version worker task should not panic") + .expect("later-version migration should succeed"); + assert_eq!( + successful_copies.load(Ordering::Acquire), + 2, + "each source version should complete exactly one target copy" + ); + drop(retry_observer); + drop(fault_guard); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial] + async fn target_busy_does_not_reset_the_ordinary_copy_failure_budget() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("target-gate-copy-budget"); + let object = "copy-fails-around-target-busy.bin"; + let body = vec![0x43; 64 * 1024]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the copy-budget target-gate bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the copy-budget bucket incarnation"); + let mut source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_data, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the copy-budget source"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source_bytes = body.len().saturating_mul(3); + let target_total = body.len().saturating_mul(8); + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_bytes, source_bytes), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot.clone()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the copy-budget capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity_snapshot]); + let target_pool_index = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| reservation.targets.first()) + .expect("the copy-budget reservation should allocate a target") + .pool_index; + + let observed_attempts = Arc::new(StdMutex::new(Vec::new())); + let fault_ready = Arc::new(tokio::sync::Notify::new()); + let fault_release = Arc::new((StdMutex::new(0usize), Condvar::new())); + let hook_bucket = bucket.clone(); + let hook_object = object.to_string(); + let hook_attempts = Arc::clone(&observed_attempts); + let hook_ready = Arc::clone(&fault_ready); + let hook_release = Arc::clone(&fault_release); + let fault_guard = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, attempt, succeeded| { + if stage != "migrate_object" || bucket != hook_bucket || object != hook_object { + return false; + } + let copy_number = { + let mut attempts = hook_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + attempts.push(attempt); + attempts.len() + }; + if copy_number <= 2 { + hook_ready.notify_one(); + let (released, wake) = &*hook_release; + let mut released = released.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while *released < copy_number { + released = wake.wait(released).unwrap_or_else(std::sync::PoisonError::into_inner); + } + return succeeded; + } + false + })); + + let worker_store = Arc::clone(&other_store); + let worker_bucket = bucket.clone(); + let source_set = other_store.pools[0].get_disks_by_key(object); + let mut worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), fault_ready.notified()) + .await + .expect("the first successful copy should reach fault injection"); + + let target_lock = other_store + .new_ns_lock( + RUSTFS_META_BUCKET, + &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"), + ) + .await + .expect("create the copy-budget target gate"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the target gate after the first injected copy failure"); + let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + retry_observer.disable_owner_pause(); + { + let (released, wake) = &*fault_release; + *released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = 1; + wake.notify_all(); + } + tokio::select! { + _ = retry_observer.wait_until_target_gate_retry() => {} + result = &mut worker => panic!("copy-budget worker finished before target contention: {result:?}"), + } + drop(target_guard); + tokio::time::timeout(Duration::from_secs(30), fault_ready.notified()) + .await + .expect("the post-contention copy should reach the second injected failure"); + assert_eq!( + *observed_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + vec![1, 2], + "target contention must not reset the attempt consumed before it" + ); + drop(retry_observer); + { + let (released, wake) = &*fault_release; + *released.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = 2; + wake.notify_all(); + } + + tokio::time::timeout(Duration::from_secs(30), worker) + .await + .expect("copy-budget worker should finish on its final attempt") + .expect("copy-budget worker task should not panic") + .expect("copy-budget migration should succeed within its original retry budget"); + assert_eq!( + *observed_attempts.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + vec![1, 2, 3], + "ordinary copy failures before and after contention must consume one shared budget" + ); + drop(fault_guard); + } + #[tokio::test] #[serial_test::serial] async fn exact_delete_reconciles_pending_capacity_before_removing_replicas() { @@ -3239,9 +4496,16 @@ mod decommission_lock_order_tests { } } - #[tokio::test] + #[test] #[serial_test::serial] - async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() { + fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() { + run_large_stack_current_thread_async_test( + "equivalent-target-capacity-restart", + data_movement_equivalent_target_reconciles_published_capacity_after_restart_case, + ); + } + + async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart_case() { let (_temp_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; let bucket = test_bucket("equivalent-target"); @@ -3472,38 +4736,50 @@ mod decommission_lock_order_tests { crate::error::is_err_object_not_found(&interleaved_target_err) || crate::error::is_err_version_not_found(&interleaved_target_err) ); - let retry_reader = other_store.pools[0] - .get_object_reader( - &bucket, - object, - None, - HeaderMap::new(), - &ObjectOptions { - versioned: true, - version_id: Some(source_version), - no_lock: true, - data_movement: true, - raw_data_movement_read: true, - ..Default::default() - }, - ) + let target_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) .await - .expect("restart retry should reread the exact source version"); - tokio::time::timeout( - Duration::from_secs(30), - data_movement::migrate_decommission_object( - Arc::clone(&other_store), - 0, - bucket.clone(), - retry_reader, - Some(incarnation), - "equivalent_target_reconcile_retry", - Some(retry_owner), - ), - ) - .await - .expect("equivalent-target retry should not deadlock") - .expect("equivalent target retry should reconcile the pending capacity intent"); + .expect("create the equivalent-target retry gate"); + let target_guard = target_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("hold the equivalent-target gate before restart retry"); + let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + retry_observer.disable_owner_pause(); + let retry_set = other_store.pools[0].get_disks_by_key(object); + let mut retry = tokio::spawn({ + let retry_store = Arc::clone(&other_store); + let retry_bucket = bucket.clone(); + async move { + retry_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + retry_bucket, + retry_set, + ) + .await + } + }); + tokio::select! { + _ = retry_observer.wait_until_target_gate_retry() => {} + result = &mut retry => panic!("equivalent-target retry finished before target contention: {result:?}"), + } + drop(target_guard); + tokio::time::timeout(Duration::from_secs(30), retry) + .await + .expect("equivalent-target retry should not deadlock after permit transfer") + .expect("equivalent-target retry task should not panic") + .expect("equivalent target retry should reconcile the pending capacity intent"); + assert_eq!( + retry_observer.target_gate_exact_reloads(), + 1, + "equivalent reconciliation must consume its exact target permit without re-entering busy recovery" + ); + drop(retry_observer); let mut reconciled = crate::core::pools::PoolMeta::default(); reconciled @@ -4582,6 +5858,8 @@ mod decommission_lock_order_tests { .is_err(), "early-ACK tail must retain the decommission capacity guard" ); + capacity_probe.abort(); + let _ = capacity_probe.await; tail_barrier.release(); drop(tail_barrier); @@ -4591,11 +5869,17 @@ mod decommission_lock_order_tests { .expect("object guard probe should join") .expect("object guard probe should acquire after the tail"); drop(object_guard); - let capacity_guard = tokio::time::timeout(std::time::Duration::from_secs(30), capacity_probe) + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) .await - .expect("capacity guard probe should finish after the tail") - .expect("capacity guard probe should join") - .expect("capacity guard probe should acquire after the tail"); + .expect("create the post-tail decommission capacity guard probe"); + let capacity_guard = tokio::time::timeout( + std::time::Duration::from_secs(30), + capacity_lock.get_write_lock(std::time::Duration::from_secs(30)), + ) + .await + .expect("capacity guard probe should finish after the tail") + .expect("capacity guard probe should acquire after the tail"); drop(capacity_guard); tokio::time::timeout(std::time::Duration::from_secs(30), &mut ordinary_mutation) .await @@ -4902,12 +6186,14 @@ mod decommission_lock_order_tests { assert_lock_order(ExternalObjectMutation::Delete).await; } - #[tokio::test] + #[test] #[serial_test::serial] - async fn active_decommission_migration_does_not_invert_capacity_and_restore_locks() { - temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { - assert_lock_order_with_tail(ExternalObjectMutation::Restore, true).await - }) - .await; + fn active_decommission_migration_does_not_invert_capacity_and_restore_locks() { + run_large_stack_async_test("decommission-restore-lock-order", || async { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_lock_order_with_tail(ExternalObjectMutation::Restore, true).await + }) + .await; + }); } } diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 87dde08c8..0b517da2e 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -234,6 +234,7 @@ async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) } fn data_movement_abort_opts( + object_info: &ObjectInfo, src_pool_idx: usize, expected_bucket_incarnation_id: Option, lock_lost_signal: Option<&Arc>, @@ -242,6 +243,9 @@ fn data_movement_abort_opts( let mut opts = ObjectOptions { data_movement: true, src_pool_idx, + versioned: object_info.version_id.is_some(), + version_id: object_info.version_id.map(|version_id| version_id.to_string()), + mod_time: object_info.mod_time, expected_bucket_incarnation_id, ..Default::default() }; @@ -265,16 +269,21 @@ fn insert_data_movement_checksum(user_defined: &mut HashMap, obj } } -fn data_movement_upload_identity(object_info: &ObjectInfo) -> String { - let version_id = object_info - .version_id - .map_or_else(|| "none".to_string(), |version_id| version_id.to_string()); - let mod_time = object_info - .mod_time - .map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string()); +fn data_movement_upload_identity_parts(version_id: Option<&str>, mod_time: Option) -> String { + let version_id = version_id.unwrap_or("none"); + let mod_time = mod_time.map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string()); format!("v1:{version_id}:{mod_time}") } +fn data_movement_upload_identity(object_info: &ObjectInfo) -> String { + let version_id = object_info.version_id.map(|version_id| version_id.to_string()); + data_movement_upload_identity_parts(version_id.as_deref(), object_info.mod_time) +} + +pub(crate) fn data_movement_upload_identity_from_options(opts: &ObjectOptions) -> String { + data_movement_upload_identity_parts(opts.version_id.as_deref(), opts.mod_time) +} + fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { let mut user_defined = data_movement_user_defined(object_info); let upload_identity = data_movement_upload_identity(object_info); @@ -486,7 +495,7 @@ pub(crate) fn data_movement_target_precondition() -> HTTPPreconditions { } } -fn is_owned_data_movement_target(target: &ObjectInfo) -> bool { +pub(crate) fn is_owned_data_movement_target(target: &ObjectInfo) -> bool { let rustfs_marker = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED); let minio_marker = format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX); if rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED) != Some("true") @@ -560,9 +569,12 @@ fn resolve_data_movement_abort_result( primary_err: Error, abort_err: Error, ) -> Error { - Error::other(format!( - "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}" - )) + data_movement_context_error( + format!( + "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}" + ), + abort_err, + ) } /// A data-movement stage failure that keeps the error it wrapped. @@ -590,17 +602,23 @@ impl std::error::Error for DataMovementStageError { } } -fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error +pub(crate) fn data_movement_context_error(rendered: String, err: E) -> Error where E: std::error::Error + Send + Sync + 'static, { - let rendered = format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"); Error::other(DataMovementStageError { rendered, source: Box::new(err), }) } +fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error +where + E: std::error::Error + Send + Sync + 'static, +{ + data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err) +} + #[cfg(test)] pub(crate) fn data_movement_stage_error_for_test(op_label: &str, stage: &str, bucket: &str, object: &str, err: Error) -> Error { data_movement_stage_error(op_label, stage, bucket, object, err) @@ -1662,8 +1680,13 @@ async fn migrate_object_inner( ); return Ok(()); } - let mut cleanup_opts = - data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + let mut cleanup_opts = data_movement_abort_opts( + &object_info, + pool_idx, + source_bucket_incarnation_id, + lock_lost_signal.as_ref(), + capacity_owner, + ); if let Some(anchor) = mutation_fence.as_ref() { anchor.guard().add_namespace_lock_fence(&mut cleanup_opts); } @@ -1865,8 +1888,13 @@ async fn migrate_object_inner( .await; if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) { - let mut abort_opts = - data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + let mut abort_opts = data_movement_abort_opts( + &object_info, + pool_idx, + expected_bucket_incarnation_id, + lock_lost_signal.as_ref(), + capacity_owner, + ); if let Some(anchor) = mutation_fence.as_ref() { anchor.guard().add_namespace_lock_fence(&mut abort_opts); } @@ -1943,8 +1971,13 @@ async fn migrate_object_inner( if should_abort_multipart_upload(&abort_multipart_flag) { #[cfg(all(test, feature = "test-util"))] pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await; - let mut abort_opts = - data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + let mut abort_opts = data_movement_abort_opts( + &object_info, + pool_idx, + expected_bucket_incarnation_id, + lock_lost_signal.as_ref(), + capacity_owner, + ); if let Some(anchor) = mutation_fence.as_ref() { anchor.guard().add_namespace_lock_fence(&mut abort_opts); } @@ -2325,6 +2358,7 @@ mod tests { assert!(message.contains("bucket-a/object-a")); assert!(message.contains("upload upload-1")); assert!(message.contains(Error::SlowDown.to_string().as_str())); + assert!(matches!(data_movement_stage_source(&err), Some(Error::OperationCanceled))); } #[test] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index c144435a6..b829208ea 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -2964,7 +2964,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: TokioMutex::new(()), pool_meta_save_gate: TokioMutex::default(), - decommission_capacity_entry_gate: TokioMutex::default(), ctx, bucket_fence_registry: Arc::default(), }) diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index b6eafe086..01d402704 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -57,18 +57,24 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2; const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3; +const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4; type CrossPoolFencePolicyResult = Result>; fn cross_pool_fence_policy_results( peer_epochs: BTreeMap, minimum_version: u32, -) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) { +) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) { let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION { Ok(peer_epochs.clone()) } else { Err(Error::other("tier delete journal v6 policy capability version is unsupported")) }; - (Ok(peer_epochs), journal_result) + let decommission_target_fence_result = if minimum_version >= DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION { + Ok(peer_epochs.clone()) + } else { + Err(Error::other("decommission target fence policy capability version is unsupported")) + }; + (Ok(peer_epochs), journal_result, decommission_target_fence_result) } #[derive(Clone, Debug)] @@ -231,6 +237,9 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken); #[derive(Clone, PartialEq, Eq)] pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct DecommissionTargetFenceFleetProofToken(FleetCapabilityProofToken); + /// A point-in-time proof that every current storage member implements the v6 /// dispatch-manifest policy. It intentionally has no `Clone` implementation: /// one acquisition authorizes one manifest construction attempt. @@ -242,6 +251,7 @@ pub(crate) struct TierDeleteJournalFleetProofToken { static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock> = OnceLock::new(); +static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { @@ -256,6 +266,10 @@ fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock &'static std::sync::RwLock { + DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) +} + fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) { if let Some(proof) = state.proof.take() { proof.generation.revoke(); @@ -368,6 +382,18 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0) } +pub(crate) fn acquire_decommission_target_fence_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let state = decommission_target_fence_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(DecommissionTargetFenceFleetProofToken) +} + +pub(crate) fn decommission_target_fence_fleet_proof_matches(proof: &DecommissionTargetFenceFleetProofToken) -> bool { + fleet_capability_proof_matches(decommission_target_fence_fleet_proof_slot(), &proof.0) +} + pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option { let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; let state = tier_delete_journal_fleet_proof_slot() @@ -468,6 +494,19 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() { journal_state.topology_conflict = false; journal_state.draining_generation = None; journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation); + drop(journal_state); + let mut decommission_state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + debug_assert!( + decommission_state + .proof + .as_ref() + .is_none_or(|current| current.generation.is_drained()) + ); + decommission_state.topology_conflict = false; + decommission_state.draining_generation = None; + decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation); } #[cfg(test)] @@ -476,6 +515,8 @@ pub(crate) struct CrossPoolFenceFleetProofGuard { previous_topology_conflict: bool, previous_journal_proof: Option, previous_journal_topology_conflict: bool, + previous_decommission_proof: Option, + previous_decommission_topology_conflict: bool, } #[cfg(test)] @@ -502,6 +543,17 @@ impl Drop for CrossPoolFenceFleetProofGuard { .map(FleetCapabilityProof::with_fresh_generation); journal_state.draining_generation = None; journal_state.topology_conflict = self.previous_journal_topology_conflict; + drop(journal_state); + let mut decommission_state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + decommission_state.proof = self + .previous_decommission_proof + .take() + .as_ref() + .map(FleetCapabilityProof::with_fresh_generation); + decommission_state.draining_generation = None; + decommission_state.topology_conflict = self.previous_decommission_topology_conflict; } } @@ -515,11 +567,16 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF let mut journal_state = tier_delete_journal_fleet_proof_slot() .write() .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut decommission_state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); let guard = CrossPoolFenceFleetProofGuard { previous_proof: state.proof.clone(), previous_topology_conflict: state.topology_conflict, previous_journal_proof: journal_state.proof.clone(), previous_journal_topology_conflict: journal_state.topology_conflict, + previous_decommission_proof: decommission_state.proof.clone(), + previous_decommission_topology_conflict: decommission_state.topology_conflict, }; if let Some(proof) = state.proof.take() { proof.generation.revoke(); @@ -535,6 +592,49 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF } } journal_state.topology_conflict = true; + if let Some(proof) = decommission_state.proof.take() { + proof.generation.revoke(); + if !proof.generation.is_drained() { + decommission_state.draining_generation = Some(proof.generation); + } + } + decommission_state.topology_conflict = true; + guard +} + +#[cfg(test)] +pub(crate) struct DecommissionTargetFenceFleetProofGuard { + previous_proof: Option, + previous_topology_conflict: bool, +} + +#[cfg(test)] +impl Drop for DecommissionTargetFenceFleetProofGuard { + fn drop(&mut self) { + let mut state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.proof = self + .previous_proof + .take() + .as_ref() + .map(FleetCapabilityProof::with_fresh_generation); + state.draining_generation = None; + state.topology_conflict = self.previous_topology_conflict; + } +} + +#[cfg(test)] +pub(crate) fn without_decommission_target_fence_fleet_proof_for_test() -> DecommissionTargetFenceFleetProofGuard { + let mut state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let guard = DecommissionTargetFenceFleetProofGuard { + previous_proof: state.proof.clone(), + previous_topology_conflict: state.topology_conflict, + }; + revoke_fleet_capability_proof_state(&mut state); + state.topology_conflict = true; guard } @@ -573,6 +673,15 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool { if journal_state.draining_generation.is_none() { journal_state.proof = Some(proof.with_fresh_generation()); } + drop(journal_state); + let mut decommission_state = decommission_target_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + decommission_state.topology_conflict = false; + revoke_fleet_capability_proof_state(&mut decommission_state); + if decommission_state.draining_generation.is_none() { + decommission_state.proof = Some(proof.with_fresh_generation()); + } true } @@ -652,6 +761,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot(), tier_delete_journal_fleet_proof_slot(), + decommission_target_fence_fleet_proof_slot(), ] { mark_fleet_capability_topology_conflict(slot); } @@ -684,11 +794,15 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), }; - let (fence_result, journal_result) = match fence_probe { + let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe { Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version), Err(err) => { let message = err.to_string(); - (Err(Error::other(message.clone())), Err(Error::other(message))) + ( + Err(Error::other(message.clone())), + Err(Error::other(message.clone())), + Err(Error::other(message)), + ) } }; let topology_conflict = remote_version_state_fleet_proof_slot() @@ -699,6 +813,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot()); revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot()); revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot()); + revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot()); } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, @@ -743,6 +858,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { "notification capability probe" ); } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + decommission_target_fence_fleet_proof_slot(), + &topology_fingerprint, + decommission_target_fence_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "decommission_target_fence_v2", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await; } }); @@ -834,7 +967,7 @@ impl NotificationSys { // A single-node deployment has no remote member to lower the local // policy version advertised by this binary. if minimum_version == u32::MAX { - minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION; + minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION; } Ok((peer_epochs, minimum_version)) } @@ -2804,15 +2937,22 @@ mod tests { use super::*; #[test] - fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() { + fn cross_pool_policy_versions_authorize_only_their_supported_protocols() { let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]); - let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2); + let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2); assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing"); assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion"); + assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence"); - let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3); + let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3); assert!(generic_v3.is_ok()); assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion"); + assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence"); + + let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4); + assert!(generic_v4.is_ok()); + assert!(journal_v4.is_ok()); + assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations"); } #[test] diff --git a/crates/ecstore/src/services/rebalance/mod.rs b/crates/ecstore/src/services/rebalance/mod.rs index dab89bac7..352d765e0 100644 --- a/crates/ecstore/src/services/rebalance/mod.rs +++ b/crates/ecstore/src/services/rebalance/mod.rs @@ -83,7 +83,6 @@ pub async fn test_store_with_persisted_rebalance_meta( decommission_cancelers: tokio::sync::RwLock::new(vec![None]), start_gate: tokio::sync::Mutex::new(()), pool_meta_save_gate: tokio::sync::Mutex::default(), - decommission_capacity_entry_gate: tokio::sync::Mutex::default(), ctx, bucket_fence_registry: std::sync::Arc::default(), }); @@ -233,7 +232,6 @@ async fn test_pool_stores_with_contexts( decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]), start_gate: tokio::sync::Mutex::new(()), pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()), - decommission_capacity_entry_gate: tokio::sync::Mutex::default(), ctx: store_ctx, bucket_fence_registry: std::sync::Arc::default(), }) diff --git a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index d280d403a..a544ed733 100644 --- a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -3009,7 +3009,6 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc Result<(Vec>, Vec, usize)> { + ) -> Result<(Vec>, Vec, usize, bool)> { let disks = self.disks.read().await.clone(); if disks.is_empty() { return Err(Error::ErasureReadQuorum); @@ -881,16 +881,24 @@ impl SetDisks { return Err(to_object_err(err.into(), vec![orig_bucket, error_path])); } + let mut has_minority_candidate = false; let mut candidate_paths = candidate_counts .into_iter() - .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path)) + .filter_map(|(path, count)| { + if count >= discovery_quorum { + Some(path) + } else { + has_minority_candidate = true; + None + } + }) .collect::>(); candidate_paths.sort_unstable(); - Ok((disks, candidate_paths, discovery_quorum)) + Ok((disks, candidate_paths, discovery_quorum, has_minority_candidate)) } pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result> { - let (_, paths, _) = self + let (_, paths, _, _) = self .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "") .await?; Ok(paths.into_iter().next()) @@ -904,7 +912,14 @@ impl SetDisks { upload_identity: &str, ) -> Result> { let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object)); - let (_, candidate_paths, _) = self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?; + let (_, candidate_paths, _, has_minority_candidate) = + self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?; + if has_minority_candidate { + return Err(Error::DecommissionCapacityBlocked { + message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks" + .to_string(), + }); + } let mut upload_ids = Vec::new(); for upload_path in candidate_paths { let Some((parent, raw_upload_id)) = upload_path.rsplit_once('/') else { @@ -920,7 +935,11 @@ impl SetDisks { { Ok((file_info, _)) => file_info, Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => { - continue; + return Err(Error::DecommissionCapacityBlocked { + message: format!( + "data movement multipart cleanup found quorum-visible upload path {upload_path} without verifiable metadata: {err}" + ), + }); } Err(err) => return Err(err), }; @@ -1189,7 +1208,7 @@ impl SetDisks { max_uploads: usize, expected_incarnation_id: Option, ) -> Result { - let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix, "").await?; + let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?; let listed_uploads = stream::iter(candidate_paths) .map(|upload_path| { let disks = &disks; @@ -7477,6 +7496,101 @@ mod tests { assert_eq!(bucket_wide.uploads[0].object, "blobs/data/layer.bin"); } + #[tokio::test] + async fn data_movement_cleanup_discovery_rejects_quorum_visible_upload_without_metadata() { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "data-movement-unverifiable-upload"; + let object = "staged/object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let upload_identity = format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()); + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity.clone()); + let opts = ObjectOptions { + data_movement: true, + user_defined: metadata, + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &opts) + .await + .expect("data movement upload should be created"); + let upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload.upload_id, true); + for temp_dir in &temp_dirs { + tokio::fs::remove_file( + temp_dir + .path() + .join(RUSTFS_META_MULTIPART_BUCKET) + .join(&upload_path) + .join("xl.meta"), + ) + .await + .expect("upload metadata should be removable while preserving its quorum-visible directory"); + } + + let err = set_disks + .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity) + .await + .expect_err("cleanup discovery must fail closed on an unverifiable upload path"); + assert!(matches!(err, Error::DecommissionCapacityBlocked { .. })); + } + + #[tokio::test] + async fn data_movement_cleanup_discovery_rejects_minority_upload_until_disks_recover() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "data-movement-minority-upload"; + let object = "staged/object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + { + let mut disks = set_disks.disks.write().await; + disks[3] = None; + } + let upload_identity = format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()); + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity.clone()); + let upload = set_disks + .new_multipart_upload( + bucket, + object, + &ObjectOptions { + data_movement: true, + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("write quorum should create the upload while one disk is offline"); + + { + let mut disks = set_disks.disks.write().await; + disks[1] = None; + disks[2] = None; + disks[3] = Some(disk_stores[3].clone()); + } + let err = set_disks + .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity) + .await + .expect_err("a minority-observed upload must block a destructive absence proof"); + assert!(matches!(err, Error::DecommissionCapacityBlocked { .. })); + + { + let mut disks = set_disks.disks.write().await; + disks[1] = Some(disk_stores[1].clone()); + disks[2] = Some(disk_stores[2].clone()); + } + let recovered = set_disks + .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity) + .await + .expect("recovered quorum should make the staged upload verifiable"); + assert_eq!(recovered.len(), 1); + assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id)); + } + /// Regression (issue #5716): a single upload directory whose `xl.meta` was /// destroyed (crash mid-write, torn disk state) must degrade to that upload /// alone. Failing the whole ListMultipartUploads turns one piece of stale diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index b56dca504..301ba171c 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -824,7 +824,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx: crate::runtime::instance::bootstrap_ctx(), bucket_fence_registry: std::sync::Arc::default(), } @@ -2468,7 +2467,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx: crate::runtime::instance::bootstrap_ctx(), bucket_fence_registry: std::sync::Arc::default(), }; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index d77dd17c4..26c8f0b83 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -580,7 +580,6 @@ impl ECStore { decommission_cancelers, start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::new(pool_meta_write_state), - decommission_capacity_entry_gate: Mutex::default(), // Adopt the caller's context (the process bootstrap one on the // legacy path) so startup writes (erasure type recorded before // this point) and later reads share one cell. @@ -918,6 +917,35 @@ mod tests { use time::OffsetDateTime; use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; + + fn run_large_stack_async_test(name: &str, case: C) + where + C: FnOnce() -> F + Send + 'static, + F: Future + 'static, + { + const STACK_SIZE: usize = if cfg!(debug_assertions) { + 8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else if cfg!(target_os = "macos") { + 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else { + rustfs_config::DEFAULT_THREAD_STACK_SIZE + }; + std::thread::Builder::new() + .name(name.to_string()) + .stack_size(STACK_SIZE) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .thread_stack_size(STACK_SIZE) + .build() + .expect("large-stack store test runtime should build"); + runtime.block_on(case()); + }) + .expect("large-stack store test thread should spawn") + .join() + .expect("large-stack store test thread should complete"); + } use uuid::Uuid; #[test] @@ -2500,9 +2528,17 @@ mod tests { } #[cfg(feature = "test-util")] - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[test] #[serial_test::serial(storage_class_env)] - async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() { + fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() { + run_large_stack_async_test( + "multipart-part-staging-publication-fence", + data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case, + ); + } + + #[cfg(feature = "test-util")] + async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() { let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir"); let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await; @@ -2582,19 +2618,23 @@ mod tests { let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0) .await .with_mutation_id(Uuid::new_v4()); - let mut upload_metadata = source.user_defined.as_ref().clone(); - rustfs_utils::http::insert_str( - &mut upload_metadata, - rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, - "mpu-staging-test".to_string(), - ); + let upload_metadata = source.user_defined.as_ref().clone(); let mut staging_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, + versioned: source.version_id.is_some(), + version_id: source.version_id.map(|version_id| version_id.to_string()), + mod_time: source.mod_time, user_defined: upload_metadata, expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; + let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts); + rustfs_utils::http::insert_str( + &mut staging_opts.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + upload_identity, + ); capacity_owner.apply_to(&mut staging_opts); let (upload, target_pool_idx, staged_incarnation_id) = store .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None) @@ -2664,6 +2704,9 @@ mod tests { let mut abort_opts = ObjectOptions { data_movement: true, src_pool_idx: 0, + versioned: source.version_id.is_some(), + version_id: source.version_id.map(|version_id| version_id.to_string()), + mod_time: source.mod_time, expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; @@ -5166,8 +5209,9 @@ mod tests { let ordinary_faults_for_hook = Arc::clone(&ordinary_faults); let fault_bucket = other_bucket.clone(); let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new( - move |stage, bucket, object, attempt| { - let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT + move |stage, bucket, object, attempt, succeeded| { + let injected = succeeded + && stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT && bucket == fault_bucket.as_str() && object == other_object && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS; @@ -5436,8 +5480,9 @@ mod tests { let fault_calls_for_hook = Arc::clone(&fault_calls); let fault_bucket = bucket.clone(); let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new( - move |stage, called_bucket, called_object, attempt| { - let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER + move |stage, called_bucket, called_object, attempt, succeeded| { + let injected = succeeded + && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER && called_bucket == fault_bucket.as_str() && called_object == object && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS; @@ -5556,8 +5601,9 @@ mod tests { let fault_calls_for_hook = Arc::clone(&fault_calls); let fault_bucket = bucket.clone(); let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new( - move |stage, called_bucket, called_object, attempt| { - let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED + move |stage, called_bucket, called_object, attempt, succeeded| { + let injected = succeeded + && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED && called_bucket == fault_bucket.as_str() && called_object == object && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS; @@ -5662,9 +5708,16 @@ mod tests { shutdown.cancel(); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[test] #[serial_test::serial(storage_class_env)] - async fn decommission_outer_fence_loss_blocks_multipart_commits() { + fn decommission_outer_fence_loss_blocks_multipart_commits() { + run_large_stack_async_test( + "decommission-multipart-outer-fence-loss", + decommission_outer_fence_loss_blocks_multipart_commits_case, + ); + } + + async fn decommission_outer_fence_loss_blocks_multipart_commits_case() { for (object, pause) in [ ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost), ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost), @@ -10213,6 +10266,29 @@ mod tests { .expect("the active decommission should own the checkpoint target"); assert_eq!(targets.len(), 1); let target = targets[0].clone(); + let consumed_before_checkpoint = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("checkpoint shutdown capacity reservation should exist") + .consumed_target_physical_bytes; + let precommit_error = store + .run_decommission_capacity_non_growing_replacement_with_capacity_lease( + target.target_pool_index, + Some(target.capacity_owner), + Some(aborting_data.len()), + |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) }, + ) + .await + .expect_err("the first checkpoint attempt should fail before writing its target") + .to_string(); + assert!(precommit_error.contains("injected checkpoint failure before commit")); + assert!( + store + .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner) + .await, + "a failed checkpoint attempt must retain exact retry state" + ); let barrier = crate::set_disk::PutObjectCommitBarrier::install( RUSTFS_META_BUCKET, &manifest_name, @@ -10263,6 +10339,27 @@ mod tests { .await, "the admitted target PUT must drain its capacity transaction before releasing the recovery fences" ); + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("checkpoint shutdown capacity reservation should remain active"); + let capacity_target = reservation + .targets + .iter() + .find(|candidate| candidate.pool_index == target.target_pool_index) + .expect("checkpoint shutdown capacity target should remain allocated"); + assert_eq!( + reservation.consumed_target_physical_bytes, consumed_before_checkpoint, + "a non-growing checkpoint replacement must not consume durable migration capacity" + ); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(capacity_target.pending_physical_bytes, 0); + assert!(capacity_target.temporary_mutations.is_empty()); + } drop(barrier); let mut reloaded_pool_meta = PoolMeta::default(); reloaded_pool_meta diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 71a636ae7..77aaf98c8 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -468,12 +468,6 @@ pub struct ECStore { /// Lock order: acquire `pool_meta_save_gate`, then the distributed /// `pool.bin` fence, then clone `pool_meta` under a short read lock. pub(crate) pool_meta_save_gate: Mutex, - /// Serializes decommission entries while the durable capacity ledger has - /// one target mutation intent slot. - /// - /// Lock order: acquire this gate before object namespaces or - /// `pool_meta_save_gate`. - pub(crate) decommission_capacity_entry_gate: Mutex<()>, /// Per-instance runtime state (Phase 5, backlog#939). /// /// Carries this instance's identity/runtime out of the process globals so @@ -1728,7 +1722,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx, bucket_fence_registry: Arc::default(), }; @@ -1804,7 +1797,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx, bucket_fence_registry: Arc::default(), }) diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index cbddc8299..908572998 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -17,11 +17,52 @@ use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity use crate::multipart_listing::paginate_multipart_listing; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::multipart::MultipartOperations as _; +use crate::storage_api_contracts::object::ObjectOperations as _; use futures::{StreamExt, stream}; use std::collections::HashSet; const MULTIPART_LIST_SET_CONCURRENCY: usize = 4; +#[cfg(test)] +static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex> { + DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) +} + +fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result { + let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked { + message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(), + })?; + let expected_version_id = opts + .version_id + .as_deref() + .map(Uuid::parse_str) + .transpose() + .map_err(|err| Error::DecommissionCapacityBlocked { + message: format!("multipart cleanup exact target version is invalid: {err}"), + })? + .filter(|version_id| !version_id.is_nil()); + let Some(target) = target else { + return Ok(true); + }; + if target.version_id.filter(|version_id| !version_id.is_nil()) != expected_version_id + || target.mod_time != Some(expected_mod_time) + { + return Err(Error::DecommissionCapacityBlocked { + message: "multipart cleanup found a target but cannot prove the exact staged identity is absent".to_string(), + }); + } + if !crate::data_movement::is_owned_data_movement_target(target) { + return Err(Error::DecommissionCapacityBlocked { + message: "multipart cleanup found an exact target without its ownership proof".to_string(), + }); + } + Ok(false) +} + #[derive(Clone, Debug)] pub(super) struct MultipartUploadListRequest { pub(super) prefix: String, @@ -197,6 +238,24 @@ async fn list_pool_multipart_uploads_for_incarnation( } impl ECStore { + #[cfg(test)] + pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) { + data_movement_multipart_discovery_counts() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(self.id, 0); + } + + #[cfg(test)] + pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize { + data_movement_multipart_discovery_counts() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&self.id) + .copied() + .unwrap_or_default() + } + pub(crate) async fn acquire_decommission_multipart_mutation_fence( &self, owner: DecommissionCapacityOwner, @@ -728,8 +787,16 @@ impl ECStore { upload_id: &str, opts: &ObjectOptions, ) -> Result<()> { - self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[upload_id.to_owned()], None, opts) - .await + let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(opts); + self.abort_multipart_uploads_for_data_movement( + target_pool_idx, + bucket, + object, + &[upload_id.to_owned()], + &upload_identity, + opts, + ) + .await } pub(crate) async fn reconcile_multipart_uploads_for_data_movement( @@ -740,26 +807,37 @@ impl ECStore { upload_identity: &str, opts: &ObjectOptions, ) -> Result<()> { - let pool = self - .pools + self.pools .get(target_pool_idx) .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; - let owner = DecommissionCapacityOwner::from_options(opts); - let has_capacity_state = match owner { - Some(owner) => { - self.has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner) - .await - } - None => false, - }; - if !has_capacity_state { + let owner = DecommissionCapacityOwner::from_options(opts) + .ok_or_else(|| Error::other("data movement multipart cleanup is missing its capacity owner"))?; + if !self + .decommission_capacity_cleanup_target_indices(owner) + .await? + .contains(&target_pool_idx) + { + return Err(Error::DecommissionCapacityBlocked { + message: format!( + "data movement multipart cleanup target pool {target_pool_idx} is outside its capacity reservation" + ), + }); + } + if !self + .has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner) + .await + { return Ok(()); } - let set = pool.get_disks_by_key(object); - let upload_ids = set - .data_movement_multipart_upload_ids(bucket, object, opts.expected_bucket_incarnation_id, upload_identity) - .await?; - self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &upload_ids, Some(upload_identity), opts) + #[cfg(test)] + { + *data_movement_multipart_discovery_counts() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entry(self.id) + .or_default() += 1; + } + self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[], upload_identity, opts) .await } @@ -769,7 +847,7 @@ impl ECStore { bucket: &str, object: &str, upload_ids: &[String], - expected_upload_identity: Option<&str>, + expected_upload_identity: &str, opts: &ObjectOptions, ) -> Result<()> { check_new_multipart_args(bucket, object)?; @@ -781,42 +859,116 @@ impl ECStore { } let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; ensure_decommission_capacity_mutation_id(bucket, object, &mut opts); + let capacity_owner = DecommissionCapacityOwner::from_options(&opts); let pool = self .pools .get(target_pool_idx) .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; - let set = pool.get_disks_by_key(object); - let mut guards = Vec::with_capacity(upload_ids.len()); - for upload_id in upload_ids { - if let Some(guard) = set - .lock_data_movement_multipart_abort(bucket, object, upload_id, expected_upload_identity, &opts) - .await? - { - guard.add_namespace_lock_fence(&mut opts); - guards.push(guard); - } - } opts.no_lock = true; - let capacity_owner = DecommissionCapacityOwner::from_options(&opts); - // Keep every upload namespace guard alive through the final capacity progress save. - let result = self + let set = pool.get_disks_by_key(object); + // Discover and lock uploads only after the target capacity gate is held. + // Return the guards so they remain alive through the final capacity save. + let (cleanup_decision_error, guards) = self .run_decommission_capacity_temporary_release_with_capacity_lease(target_pool_idx, capacity_owner, |capacity_lease| { let mut delete_opts = opts.clone(); - let guards = &guards; let set = &set; + let pool = &pool; async move { - if let Some(capacity_lease) = capacity_lease { - delete_opts.add_namespace_lock_lost_signal(capacity_lease); + if let Some(capacity_lease) = capacity_lease.as_ref() { + delete_opts.add_namespace_lock_lost_signal(Arc::clone(capacity_lease)); } - for guard in guards { - guard.delete(set, bucket, object, &delete_opts).await?; + let mut candidate_upload_ids = upload_ids.to_vec(); + candidate_upload_ids.extend( + set.data_movement_multipart_upload_ids( + bucket, + object, + delete_opts.expected_bucket_incarnation_id, + expected_upload_identity, + ) + .await?, + ); + candidate_upload_ids.sort_unstable(); + candidate_upload_ids.dedup(); + + let mut guards = Vec::with_capacity(candidate_upload_ids.len()); + for upload_id in &candidate_upload_ids { + match set + .lock_data_movement_multipart_abort( + bucket, + object, + upload_id, + Some(expected_upload_identity), + &delete_opts, + ) + .await + { + Ok(Some(guard)) => { + guard.add_namespace_lock_fence(&mut delete_opts); + guards.push(guard); + } + Ok(None) => {} + Err(err) => return Err(err), + } } - Ok(()) + for guard in &guards { + match guard.delete(set, bucket, object, &delete_opts).await { + Ok(()) => {} + Err(err) if is_err_invalid_upload_id(&err) => {} + Err(err) => return Err(err), + } + } + + if !set + .data_movement_multipart_upload_ids( + bucket, + object, + delete_opts.expected_bucket_incarnation_id, + expected_upload_identity, + ) + .await? + .is_empty() + { + return Err(Error::DecommissionCapacityBlocked { + message: "multipart cleanup could not prove the exact staged uploads are absent".to_string(), + }); + } + + // The target capacity gate makes the exact target proof, + // upload absence proof, and pending-ledger decision one + // critical section with cleanup finalize. + let (clear_pending, cleanup_decision_error) = if capacity_owner.is_some() { + let mut lookup_opts = ObjectOptions { + versioned: delete_opts.versioned, + version_suspended: delete_opts.version_suspended, + version_id: delete_opts.version_id.clone(), + metadata_chg: delete_opts.version_id.is_some(), + no_lock: true, + ..Default::default() + }; + if let Some(capacity_lease) = capacity_lease { + lookup_opts.add_namespace_lock_lost_signal(capacity_lease); + } + let target = match pool.get_object_info(bucket, object, &lookup_opts).await { + Ok(target) => Some(target), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None, + Err(err) => return Err(err), + }; + match decommission_multipart_target_clear_pending(&delete_opts, target.as_ref()) { + Ok(clear_pending) => (clear_pending, None), + Err(err) => (false, Some(err)), + } + } else { + (true, None) + }; + Ok(((cleanup_decision_error, guards), clear_pending)) } }) - .await; + .await?; drop(guards); - result + if let Some(err) = cleanup_decision_error { + return Err(err); + } + Ok(()) } #[instrument(skip(self))] @@ -1024,6 +1176,66 @@ mod tests { } } + #[test] + fn decommission_multipart_cleanup_requires_exact_target_evidence() { + let version_id = Uuid::new_v4(); + let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(7); + let opts = ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + ..Default::default() + }; + + assert!( + decommission_multipart_target_clear_pending(&opts, None) + .expect("an exact target miss should authorize pending cleanup") + ); + + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVED, "true".to_string()); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVED_TAGS, "v1:".to_string()); + let owned_target = ObjectInfo { + version_id: Some(version_id), + mod_time: Some(mod_time), + user_defined: Arc::new(metadata), + ..Default::default() + }; + assert!( + !decommission_multipart_target_clear_pending(&opts, Some(&owned_target)) + .expect("an exact owned target should preserve pending capacity") + ); + + let missing_identity = ObjectOptions { + mod_time: None, + ..opts.clone() + }; + assert!(matches!( + decommission_multipart_target_clear_pending(&missing_identity, None), + Err(Error::DecommissionCapacityBlocked { .. }) + )); + + let mismatched_target = ObjectInfo { + version_id: Some(Uuid::new_v4()), + mod_time: Some(mod_time), + ..Default::default() + }; + assert!(matches!( + decommission_multipart_target_clear_pending(&opts, Some(&mismatched_target)), + Err(Error::DecommissionCapacityBlocked { .. }) + )); + + let unowned_target = ObjectInfo { + version_id: Some(version_id), + mod_time: Some(mod_time), + ..Default::default() + }; + assert!(matches!( + decommission_multipart_target_clear_pending(&opts, Some(&unowned_target)), + Err(Error::DecommissionCapacityBlocked { .. }) + )); + } + /// Models a single pool's `list_multipart_uploads`: returns uploads strictly /// after the `(key, upload_id)` marker in `(key, upload_id)` order, capped at /// `max_uploads` (mirroring the per-pool page cap). @@ -1171,7 +1383,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx: crate::runtime::instance::bootstrap_ctx(), bucket_fence_registry: std::sync::Arc::default(), } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 413032852..7383893b8 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -1531,6 +1531,10 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object writer_pool_lookup_opts(opts, no_lock) } +fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool { + opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some()) +} + fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { let mut lookup_opts = version_aware_lookup_opts(opts, no_lock); lookup_opts.skip_decommissioned = true; @@ -3467,7 +3471,12 @@ impl ECStore { } fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> { - result.map_err(|err| Error::other(format!("failed to decommission tiered object for {bucket}/{object}: {err}"))) + result.map_err(|err| { + crate::data_movement::data_movement_context_error( + format!("failed to decommission tiered object for {bucket}/{object}: {err}"), + err, + ) + }) } #[instrument(skip(self, fi, opts))] @@ -3515,7 +3524,7 @@ impl ECStore { ); } - let idx = if opts.data_movement && opts.version_id.is_some() { + let idx = if uses_data_movement_pool_selection(&opts) { Self::resolve_decommission_target_pool_idx_result( self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true) .await, @@ -3713,7 +3722,7 @@ impl ECStore { return Ok(0); } - let idx = if opts.data_movement && opts.version_id.is_some() { + let idx = if uses_data_movement_pool_selection(opts) { self.select_data_movement_pool_idx(bucket, object, size, opts, false).await? } else if opts.no_lock { self.get_pool_idx_no_lock(bucket, object, size).await? @@ -7277,6 +7286,23 @@ mod tests { assert!(rendered.contains("boom"), "{rendered}"); } + #[test] + fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() { + let err = ECStore::resolve_decommission_tiered_object_result( + Err(Error::DecommissionCapacityBlocked { + message: "target gate busy".to_string(), + }), + "bucket", + "object", + ) + .expect_err("expected contextual error"); + + assert!(matches!( + crate::data_movement::data_movement_stage_source(&err), + Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy" + )); + } + #[test] fn version_aware_lookup_opts_enables_version_aware_lookup() { let opts = ObjectOptions { @@ -7502,6 +7528,26 @@ mod tests { assert!(lookup_opts.skip_rebalancing); } + #[test] + fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() { + let mut opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + assert!(!uses_data_movement_pool_selection(&opts)); + + DecommissionCapacityOwner { + source_pool_index: 1, + operation_id: Uuid::new_v4(), + generation: 2, + owner_nonce: Uuid::new_v4(), + mutation_id: None, + } + .apply_to(&mut opts); + + assert!(uses_data_movement_pool_selection(&opts)); + } + #[test] fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() { let lookup_opts = transition_restore_pool_opts(&ObjectOptions { @@ -7576,7 +7622,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx: crate::runtime::instance::bootstrap_ctx(), bucket_fence_registry: std::sync::Arc::default(), } @@ -7669,7 +7714,6 @@ mod tests { decommission_cancelers: RwLock::new(Vec::new()), start_gate: Mutex::new(()), pool_meta_save_gate: Mutex::default(), - decommission_capacity_entry_gate: Mutex::default(), ctx, bucket_fence_registry: std::sync::Arc::default(), } diff --git a/docs/operations/snowball-auto-extract.md b/docs/operations/snowball-auto-extract.md new file mode 100644 index 000000000..7c203106b --- /dev/null +++ b/docs/operations/snowball-auto-extract.md @@ -0,0 +1,36 @@ +# Snowball Auto-Extract Limits + +RustFS accepts MinIO-compatible Snowball auto-extract uploads. Archive +members are streamed into objects while RustFS enforces entry-count, path, +PAX metadata, per-object, cumulative unpacked-size, and decoded-stream +limits. + +## Size limits + +The defaults remain compatible with the existing safety policy: + +| Environment variable | Default | Hard maximum | Meaning | +| --- | ---: | ---: | --- | +| `RUSTFS_SNOWBALL_MAX_ENTRY_BYTES` | 1 GiB | 1 TiB | Maximum unpacked size of one archive member | +| `RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES` | 10 GiB | 10 TiB | Maximum cumulative unpacked object bytes in one request | + +Invalid values use the default. Zero is treated as one byte, values above the +hard maximum are clamped, and the per-entry limit is never allowed to exceed +the cumulative request limit. RustFS derives a separate decoded-stream limit +with bounded room for tar headers and PAX metadata; it cannot be disabled. + +Increasing either limit raises the maximum work performed by one admitted +request. Snowball archive decoder admission remains globally bounded, so a +larger archive cannot create an unbounded number of concurrent decoders. +Restart RustFS after changing these environment variables. + +## Small-member concurrency + +For requests that set Snowball ignore-errors and do not use bucket quota +accounting, RustFS stages members up to 128 KiB and commits at most 16 at a +time. Requests that must stop on the first write error and quota-enabled +requests remain serial so their observable error and accounting behavior does +not change. + +Set `RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT=1` to restore fully serial member +commits. Values are clamped to the range 1 through 16. diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index b773e37a5..43729cbd5 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -373,10 +373,10 @@ const EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES: usize = 8 * 1024; const EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES: usize = 2 * 1024; const EXTRACT_MAX_EFFECTIVE_PAX_FIELDS: usize = 4096; const EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES: u64 = 128 * 1024 * 1024; -const EXTRACT_SMALL_MEMBER_MAX_BYTES: usize = 64 * 1024; -const EXTRACT_DEFAULT_MAX_INFLIGHT: usize = 1; +const EXTRACT_SMALL_MEMBER_MAX_BYTES: usize = 128 * 1024; +const EXTRACT_DEFAULT_MAX_INFLIGHT: usize = 16; const EXTRACT_BATCH_MAX_MEMBERS: usize = 16; -const EXTRACT_BATCH_MAX_STAGING_BYTES: usize = 2 * 1024 * 1024; +const EXTRACT_BATCH_MAX_STAGING_BYTES: usize = 3 * 1024 * 1024; const EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES: usize = 512; const EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES: usize = 64; const ENV_RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT: &str = "RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT"; @@ -1864,7 +1864,34 @@ fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result ArchiveLimits { - ArchiveLimits::default() + static LIMITS: OnceLock = OnceLock::new(); + *LIMITS.get_or_init(|| { + normalize_put_object_extract_limits( + rustfs_utils::get_env_u64( + rustfs_config::ENV_SNOWBALL_MAX_ENTRY_BYTES, + rustfs_config::DEFAULT_SNOWBALL_MAX_ENTRY_BYTES, + ), + rustfs_utils::get_env_u64( + rustfs_config::ENV_SNOWBALL_MAX_UNPACKED_BYTES, + rustfs_config::DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES, + ), + ) + }) +} + +fn normalize_put_object_extract_limits(max_entry_bytes: u64, max_unpacked_bytes: u64) -> ArchiveLimits { + let defaults = ArchiveLimits::default(); + let max_total_unpacked_size = max_unpacked_bytes.clamp(1, rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES); + let max_entry_size = max_entry_bytes + .clamp(1, rustfs_config::MAX_SNOWBALL_ENTRY_BYTES) + .min(max_total_unpacked_size); + + ArchiveLimits { + max_entry_size, + max_total_unpacked_size, + max_decoded_size: max_total_unpacked_size.saturating_add(max_entry_size), + ..defaults + } } fn build_put_object_extract_archive(decoder: R, limits: ArchiveLimits) -> Archive @@ -2175,12 +2202,13 @@ impl DefaultObjectUsecase { .is_some_and(|result| result.uses_durable_reservations); // Without ignore-errors, the legacy contract stops before attempting a // later member after the first storage failure. Parallel commits cannot - // preserve that boundary, so concurrency requires both ignore-errors - // and an explicit max-inflight value above the serial default. Quota - // accounting can fail after storage commit, so quota-enabled imports - // also remain serial. An opted-in micro-batch is always drained; a - // fatal outcome stops later batches but cannot roll back peers that - // already committed in the current batch. + // preserve that boundary, so only ignore-errors requests use the + // configured micro-batch. Quota accounting can fail after storage + // commit, so quota-enabled imports also remain serial. Setting + // RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT=1 restores serial behavior. A + // micro-batch is always drained; a fatal outcome stops later batches + // but cannot roll back peers that already committed in the current + // batch. let max_inflight = select_put_object_extract_max_inflight( put_object_extract_max_inflight(), extract_options.ignore_errors, @@ -2856,7 +2884,7 @@ mod tests { #[test] fn snowball_max_inflight_has_a_serial_compatibility_floor_and_bounded_ceiling() { - assert_eq!(EXTRACT_DEFAULT_MAX_INFLIGHT, 1); + assert_eq!(EXTRACT_DEFAULT_MAX_INFLIGHT, EXTRACT_BATCH_MAX_MEMBERS); assert_eq!(normalize_put_object_extract_max_inflight(0), 1); assert_eq!(normalize_put_object_extract_max_inflight(1), 1); assert_eq!(normalize_put_object_extract_max_inflight(usize::MAX), EXTRACT_BATCH_MAX_MEMBERS); @@ -2877,6 +2905,35 @@ mod tests { ); } + #[test] + fn snowball_archive_limits_preserve_defaults_and_clamp_operator_overrides() { + let defaults = ArchiveLimits::default(); + assert_eq!( + normalize_put_object_extract_limits( + rustfs_config::DEFAULT_SNOWBALL_MAX_ENTRY_BYTES, + rustfs_config::DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES, + ), + defaults + ); + + let minimum = normalize_put_object_extract_limits(0, 0); + assert_eq!(minimum.max_entry_size, 1); + assert_eq!(minimum.max_total_unpacked_size, 1); + assert_eq!(minimum.max_decoded_size, 2); + + let bounded = normalize_put_object_extract_limits(u64::MAX, u64::MAX); + assert_eq!(bounded.max_entry_size, rustfs_config::MAX_SNOWBALL_ENTRY_BYTES); + assert_eq!(bounded.max_total_unpacked_size, rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES); + assert_eq!( + bounded.max_decoded_size, + rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES + rustfs_config::MAX_SNOWBALL_ENTRY_BYTES + ); + + let entry_is_bounded_by_the_request_total = normalize_put_object_extract_limits(1024, 512); + assert_eq!(entry_is_bounded_by_the_request_total.max_entry_size, 512); + assert_eq!(entry_is_bounded_by_the_request_total.max_total_unpacked_size, 512); + } + #[test] fn snowball_batch_state_flushes_on_duplicates_limits_and_serial_barriers() { let mut state = ExtractBatchState::default(); diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 502cec91e..7b5258a98 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -181,10 +181,11 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); -// v3 additionally promises the v6 tier-delete dispatch-manifest policy. The +// v3 additionally promises the v6 tier-delete dispatch-manifest policy; v4 +// promises the sticky per-target decommission capacity fence. The // existing periodic topology probe carries both capabilities so normal object // operations do not add another peer RPC. -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 3; +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 4; fn admit_heal_control_replay( replay_cache: &mut HashMap>, @@ -3770,7 +3771,7 @@ mod tests { } #[tokio::test] - async fn cross_pool_fence_probe_authenticates_supported_v3_state() { + async fn cross_pool_fence_probe_authenticates_supported_v4_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); assert!( diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 4925fe18c..269a7d3df 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -26,7 +26,7 @@ 6|crates/ecstore/src/config/com.rs 14|crates/ecstore/src/config/storageclass.rs 182|crates/ecstore/src/core/pools.rs -8|crates/ecstore/src/data_movement/mod.rs +7|crates/ecstore/src/data_movement/mod.rs 2|crates/ecstore/src/data_usage/local_snapshot.rs 12|crates/ecstore/src/data_usage/mod.rs 5|crates/ecstore/src/disk/local.rs @@ -69,5 +69,5 @@ 12|crates/ecstore/src/store/init.rs 2|crates/ecstore/src/store/init_format.rs 3|crates/ecstore/src/store/multipart.rs -7|crates/ecstore/src/store/object.rs +6|crates/ecstore/src/store/object.rs 5|crates/ecstore/src/store/rebalance/support.rs