From 6c67086d0ba843c6cfdbaff6e2b188804a656a5d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 31 Aug 2026 15:20:09 +0800 Subject: [PATCH] fix(ecstore): reserve decommission capacity safely (#6917) --- .config/nextest.toml | 13 + crates/ecstore/src/bucket/metadata_sys.rs | 10 +- crates/ecstore/src/core/pools.rs | 4190 ++++++++++++++- crates/ecstore/src/core/pools_test.rs | 4528 +++++++++++++++++ crates/ecstore/src/core/sets.rs | 21 +- crates/ecstore/src/data_movement/mod.rs | 329 +- crates/ecstore/src/error/mod.rs | 10 + crates/ecstore/src/object_api/types.rs | 46 + .../ecstore/src/services/rebalance/control.rs | 6 +- crates/ecstore/src/services/rebalance/mod.rs | 74 +- .../src/set_disk/core/io_primitives.rs | 4 +- crates/ecstore/src/set_disk/mod.rs | 2 +- crates/ecstore/src/set_disk/ops/multipart.rs | 306 +- crates/ecstore/src/set_disk/ops/object.rs | 163 +- crates/ecstore/src/set_disk/replication.rs | 52 +- crates/ecstore/src/store/heal.rs | 333 +- crates/ecstore/src/store/init.rs | 12 +- crates/ecstore/src/store/multipart.rs | 278 +- crates/ecstore/src/store/object.rs | 481 +- crates/ecstore/src/store/rebalance.rs | 12 +- .../log-analyzer/src/rules/seed/capacity.rs | 4 +- crates/log-analyzer/src/rules/seed/tests.rs | 4 +- rustfs/src/app/object/copy.rs | 131 +- scripts/error-other-format-baseline.txt | 2 +- 24 files changed, 10523 insertions(+), 488 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 044a3ff47..90e09740a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -46,6 +46,11 @@ e2e-reliability = { max-threads = 1 } e2e-inline-boundaries = { max-threads = 1 } e2e-cluster-nightly = { max-threads = 1 } +# Deep async storage futures are composed into tests across several crates. +# Keep the test stack bounded but above libtest's 2 MiB default. +[scripts.setup.ecstore-base-stack] +command = ['sh', '-c', 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"'] + # These exact regression scenarios build deep async storage futures that exceed # libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes # the same 32 MiB stack already used by the crate's dedicated large-stack tests. @@ -63,6 +68,10 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"'] filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' setup = 'ecstore-large-stack' +[[profile.default.scripts]] +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +setup = 'ecstore-base-stack' + [[profile.default.scripts]] filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))' setup = 'lifecycle-large-stack' @@ -170,6 +179,10 @@ path = "junit.xml" filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' setup = 'ecstore-large-stack' +[[profile.ci.scripts]] +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +setup = 'ecstore-base-stack' + [[profile.ci.scripts]] filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))' setup = 'lifecycle-large-stack' diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 52edc4350..d72ee05fe 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -412,8 +412,14 @@ pub(crate) fn require_bucket_metadata_sys_in( } pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result> { - let sys = bucket_metadata_sys_of(ctx)?; - Ok(sys.read().await.api.clone()) + object_store_if_initialized_in(ctx) + .await + .ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance")) +} + +pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option> { + let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?; + Some(sys.read().await.api.clone()) } pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result> { diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index c5d3bfe4c..7a88dddf1 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -47,7 +47,7 @@ use crate::error::{ is_err_version_not_found, }; use crate::layout::endpoints::EndpointServerPools; -use crate::object_api::{GetObjectReader, ObjectOptions}; +use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions}; use crate::runtime::sources as runtime_sources; use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; @@ -76,6 +76,7 @@ use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; #[cfg(test)] @@ -112,7 +113,18 @@ const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX]; -const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; +const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = 1; +const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1; +const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10); +const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled"; +const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed"; +const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed"; +const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total"; +const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes"; +const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes"; +const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES: &str = "rustfs_decommission_capacity_prediction_error_bytes"; +const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str = + "rustfs_decommission_capacity_prediction_absolute_error_bytes"; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3; @@ -674,6 +686,20 @@ fn spawn_decommission_index_cancelers( async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await } }); if let Err(err) = await_decommission_worker(idx, worker).await { + if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "capacity_blocked", + error = %err, + "Decommission routine paused without a terminal transition" + ); + store.release_decommission_canceler_slot(idx, &canceler).await; + stop_queue = true; + continue; + } if let Err(blocked) = store .ensure_pool_meta_side_effects_safe("decommission paused because pool metadata requires recovery") .await @@ -734,7 +760,7 @@ pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool { } fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool { - info.has_decommission_state() && (!info.queued || !info.unresolved_entries.is_empty()) + info.has_decommission_state() } fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> { @@ -829,38 +855,1073 @@ fn ensure_decommission_start_pool_states(meta: &PoolMeta, indices: &[usize]) -> ensure_decommission_start_keeps_active_pool(meta, indices) } -fn decommission_target_capacity_required(source_used: usize) -> usize { - source_used - .saturating_mul(100 + DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT) - .div_ceil(100) +fn capacity_mul_div_ceil(value: usize, multiplier: usize, divisor: usize) -> usize { + if value == 0 || multiplier == 0 { + return 0; + } + if divisor == 0 { + return usize::MAX; + } + value + .checked_mul(multiplier) + .map(|product| product.div_ceil(divisor)) + .unwrap_or(usize::MAX) +} + +fn capacity_source_data_equivalent(physical_bytes: usize, layout: DecommissionErasureLayout) -> Result { + if !layout.is_valid() { + return Err(Error::DecommissionCapacity(format!( + "failed to model decommission capacity: invalid source layout data={} parity={}", + layout.data, layout.parity + ))); + } + Ok(capacity_mul_div_ceil(physical_bytes, layout.data, layout.width())) +} + +fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasureLayout) -> Result { + if !layout.is_valid() { + return Err(Error::DecommissionCapacity(format!( + "failed to model decommission capacity: invalid target layout data={} parity={}", + layout.data, layout.parity + ))); + } + Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data)) +} + +fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result { + targets + .iter() + .map(|target| target.layout) + .filter(|layout| layout.is_valid()) + .max_by(|left, right| { + ((left.width() as u128) * (right.data as u128)).cmp(&((right.width() as u128) * (left.data as u128))) + }) + .ok_or_else(|| Error::other("failed to start decommission: no valid target erasure layout is available")) +} + +fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool { + matches!(meta.version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || pool_meta_v2_writer_enabled() +} + +fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> { + if decommission_capacity_writer_supported(meta) { + return Ok(()); + } + Err(Error::DecommissionCapacity(format!( + "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable both {} and {} only after every reader and writer supports V2", + rustfs_config::ENV_POOL_META_V2_WRITE, + rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, + ))) +} + +fn decommission_capacity_blocked_error(message: impl Display) -> Error { + Error::DecommissionCapacityBlocked { + message: message.to_string(), + } +} + +fn is_decommission_capacity_blocked_error(err: &Error) -> bool { + if matches!(err, Error::DecommissionCapacityBlocked { .. }) { + return true; + } + data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error) +} + +fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> { + let Some(reservation) = reservation else { + return Ok(()); + }; + if reservation.model_version != DECOMMISSION_CAPACITY_MODEL_VERSION { + return Err(Error::DecommissionCapacity(format!( + "pool metadata load failed: unsupported decommission capacity model version {}", + reservation.model_version + ))); + } + if reservation.operation_id.is_nil() || reservation.generation == 0 || reservation.owner_nonce.is_nil() { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation identity is invalid", + )); + } + if !reservation.source_layout.is_valid() || reservation.targets.iter().any(|target| !target.layout.is_valid()) { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation contains an invalid erasure layout", + )); + } + if reservation.temporary_copies != DECOMMISSION_CAPACITY_TEMPORARY_COPIES { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation temporary-copy model is invalid", + )); + } + if reservation.source_data_equivalent_bytes + != capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)? + { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation source estimate is invalid", + )); + } + if reservation.peak_physical_bytes + != reservation + .predicted_physical_bytes + .saturating_add(reservation.temporary_physical_bytes) + || reservation.temporary_physical_bytes + != reservation + .predicted_physical_bytes + .saturating_mul(reservation.temporary_copies) + { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation peak does not match its physical model", + )); + } + if reservation.committed_data_bytes > reservation.source_data_equivalent_bytes { + return Err(Error::other( + "pool metadata load failed: decommission capacity committed bytes exceed the source estimate", + )); + } + if reservation.consumed_target_physical_bytes > reservation.predicted_physical_bytes { + return Err(Error::other( + "pool metadata load failed: decommission capacity consumed target bytes exceed the prediction", + )); + } + let mut target_indices = HashSet::with_capacity(reservation.targets.len()); + let mut reserved_physical_bytes = 0usize; + let mut consumed_physical_bytes = 0usize; + let mut observed_physical_bytes = 0usize; + let mut inflight_physical_bytes = 0usize; + let mut pending_physical_bytes = 0usize; + for target in &reservation.targets { + let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len()); + let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| { + if mutation.mutation_id.is_nil() + || mutation.physical_bytes == 0 + || !temporary_mutation_ids.insert(mutation.mutation_id) + { + return Err(Error::other( + "pool metadata load failed: decommission capacity temporary mutation is invalid", + )); + } + Ok(total.saturating_add(mutation.physical_bytes)) + })?; + if target.pool_index == reservation.source_pool_index + || !target_indices.insert(target.pool_index) + || target.reserved_physical_bytes == 0 + || target.reserved_physical_bytes > target.physical_free_at_reservation + || target.physical_free_at_reservation > target.physical_total_at_reservation + || target.consumed_physical_bytes > reservation.consumed_target_physical_bytes + || target.observed_physical_bytes > reservation.observed_target_physical_bytes + || target.inflight_physical_bytes > target.observed_physical_bytes + || temporary_mutation_bytes > target.inflight_physical_bytes + { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation target allocation is invalid", + )); + } + if reservation.predicted_physical_bytes + < capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target.layout)? + { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation target estimate is unsafe", + )); + } + reserved_physical_bytes = + reserved_physical_bytes.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies)); + consumed_physical_bytes = consumed_physical_bytes.saturating_add(target.consumed_physical_bytes); + observed_physical_bytes = observed_physical_bytes.saturating_add(target.observed_physical_bytes); + inflight_physical_bytes = inflight_physical_bytes.saturating_add(target.inflight_physical_bytes); + pending_physical_bytes = pending_physical_bytes.saturating_add(target.pending_physical_bytes); + } + if reserved_physical_bytes != reservation.remaining_peak_physical_bytes() { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation target allocation is incomplete", + )); + } + if inflight_physical_bytes != reservation.inflight_target_physical_bytes { + return Err(Error::other( + "pool metadata load failed: decommission capacity inflight target accounting is incomplete", + )); + } + if pending_physical_bytes != reservation.pending_target_physical_bytes { + return Err(Error::other( + "pool metadata load failed: decommission capacity pending target accounting is incomplete", + )); + } + if consumed_physical_bytes != reservation.consumed_target_physical_bytes + || observed_physical_bytes != reservation.observed_target_physical_bytes + { + return Err(Error::other( + "pool metadata load failed: decommission capacity target progress accounting is incomplete", + )); + } + if reservation.released_at.is_some() != reservation.release_reason.is_some() { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation release state is incomplete", + )); + } + Ok(()) +} + +fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool { + let Some(reservation) = info.capacity_reservation.as_mut() else { + return false; + }; + if !reservation.active() { + return false; + } + reservation.released_at = Some(now); + reservation.release_reason = Some(reason.to_string()); + metrics::gauge!( + METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES, + "pool_index" => reservation.source_pool_index.to_string() + ) + .set(0.0); + true +} + +fn renew_decommission_capacity_reservation( + reservation: &mut DecommissionCapacityReservation, + now: OffsetDateTime, + recover_expired: bool, +) -> bool { + if !reservation.active() { + return false; + } + let expired = reservation.expires_at <= now; + if expired && recover_expired { + reservation.owner_nonce = uuid::Uuid::new_v4(); + reservation.recovered_at = Some(now); + } + reservation.renewed_at = now; + reservation.expires_at = now + DECOMMISSION_CAPACITY_RESERVATION_TTL; + true +} + +fn next_decommission_capacity_generation(meta: &PoolMeta) -> Result { + meta.pools + .iter() + .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref()) + .map(|reservation| reservation.generation) + .max() + .unwrap_or_default() + .checked_add(1) + .ok_or_else(|| Error::other("failed to start decommission: capacity reservation generation overflow")) +} + +fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet { + meta.pools + .iter() + .enumerate() + .filter_map(|(idx, pool)| { + pool.decommission.as_ref().and_then(|info| { + (info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)) + .then_some(idx) + }) + }) + .collect() +} + +fn build_decommission_capacity_reservation( + source: DecommissionPoolCapacityInfo, + target_layout: DecommissionErasureLayout, + operation_id: uuid::Uuid, + generation: u64, + now: OffsetDateTime, +) -> Result { + let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?; + let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?; + let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES); + Ok(DecommissionCapacityReservation { + model_version: DECOMMISSION_CAPACITY_MODEL_VERSION, + operation_id, + generation, + owner_nonce: uuid::Uuid::new_v4(), + source_pool_index: source.pool_index, + source_layout: source.layout, + source_physical_total_bytes: source.physical_total, + source_physical_bytes: source.physical_used, + source_data_equivalent_bytes, + predicted_physical_bytes, + temporary_copies: DECOMMISSION_CAPACITY_TEMPORARY_COPIES, + temporary_physical_bytes, + peak_physical_bytes: predicted_physical_bytes.saturating_add(temporary_physical_bytes), + committed_data_bytes: 0, + consumed_target_physical_bytes: 0, + observed_target_physical_bytes: 0, + inflight_target_physical_bytes: 0, + pending_target_physical_bytes: 0, + prediction_error_bytes: 0, + targets: Vec::new(), + created_at: now, + renewed_at: now, + expires_at: now + DECOMMISSION_CAPACITY_RESERVATION_TTL, + recovered_at: None, + released_at: None, + release_reason: None, + }) +} + +fn reserve_decommission_start_target_capacity( + meta: &mut PoolMeta, + requested_indices: &[usize], + capacity_infos: &[DecommissionPoolCapacityInfo], + operation_id: uuid::Uuid, + generation: u64, + now: OffsetDateTime, +) -> Result<()> { + let active_sources = active_decommission_source_indices(meta); + let targets = capacity_infos + .iter() + .copied() + .filter(|capacity| { + !active_sources.contains(&capacity.pool_index) + && meta + .pools + .get(capacity.pool_index) + .is_some_and(is_decommission_start_active_pool) + }) + .collect::>(); + let target_layout = worst_decommission_target_layout(&targets)?; + let requested = requested_indices.iter().copied().collect::>(); + let mut reservations = Vec::with_capacity(active_sources.len()); + + let mut source_indices = active_sources.into_iter().collect::>(); + source_indices.sort_unstable(); + for source_index in source_indices { + let source = capacity_infos + .iter() + .copied() + .find(|capacity| capacity.pool_index == source_index) + .ok_or_else(|| { + Error::DecommissionCapacity(format!( + "failed to start decommission: capacity snapshot is missing source pool {source_index}" + )) + })?; + let pool = meta + .pools + .get(source_index) + .ok_or_else(|| invalid_decommission_pool_index_error(meta.pools.len(), source_index))?; + let info = pool + .decommission + .as_ref() + .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?; + let new_reservation = requested.contains(&source_index); + let mut reservation = if new_reservation { + build_decommission_capacity_reservation(source, target_layout, operation_id, generation, now)? + } else { + info.capacity_reservation.clone().ok_or_else(|| { + Error::DecommissionCapacity(format!( + "failed to start decommission: active source pool {source_index} has no durable capacity reservation" + )) + })? + }; + if !reservation.active() { + return Err(Error::DecommissionCapacity(format!( + "failed to start decommission: active source pool {source_index} has a released capacity reservation" + ))); + } + renew_decommission_capacity_reservation(&mut reservation, now, true); + reservations.push((source_index, reservation, new_reservation)); + } + + let mut remaining_target_capacity = targets + .iter() + .map(|target| (target.pool_index, target.physical_free)) + .collect::>(); + for (_, reservation, new_reservation) in &reservations { + if *new_reservation { + continue; + } + for target in &reservation.targets { + let available = remaining_target_capacity.entry(target.pool_index).or_default(); + *available = available + .saturating_add(target.inflight_physical_bytes) + .saturating_add(target.pending_physical_bytes); + let required = target.remaining_reserved_physical_bytes(reservation.temporary_copies); + if *available < required { + return Err(decommission_capacity_blocked_error(format!( + "failed to start decommission: existing reservation for source pool {} requires {required} bytes on target pool {}, but only {} bytes remain", + reservation.source_pool_index, target.pool_index, *available + ))); + } + *available -= required; + } + } + let required = reservations + .iter() + .filter(|(_, _, new_reservation)| *new_reservation) + .fold(0usize, |total, (_, reservation, _)| { + total.saturating_add(reservation.remaining_peak_physical_bytes()) + }); + let available = remaining_target_capacity + .values() + .copied() + .fold(0usize, usize::saturating_add); + if available < required { + metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => "activation").increment(1); + return Err(decommission_capacity_blocked_error(format!( + "failed to start decommission: insufficient reserved physical target capacity: operation {operation_id} generation {generation} requires {required} bytes, but {available} bytes are available after source/target parity and temporary-copy accounting" + ))); + } + + for (_, reservation, new_reservation) in &mut reservations { + if !*new_reservation { + continue; + } + let mut remaining = reservation.remaining_peak_physical_bytes(); + for target in &targets { + if remaining == 0 { + break; + } + let target_remaining = remaining_target_capacity.entry(target.pool_index).or_default(); + let allocated = remaining.min(*target_remaining); + if allocated == 0 { + continue; + } + *target_remaining -= allocated; + remaining -= allocated; + reservation.targets.push(DecommissionCapacityTarget { + pool_index: target.pool_index, + layout: target.layout, + physical_total_at_reservation: target.physical_total, + physical_free_at_reservation: target.physical_free, + reserved_physical_bytes: allocated, + consumed_physical_bytes: 0, + observed_physical_bytes: 0, + inflight_physical_bytes: 0, + pending_physical_bytes: 0, + pending_mutation_id: None, + temporary_mutations: Vec::new(), + }); + } + debug_assert_eq!(remaining, 0); + } + + for (source_index, reservation, _) in reservations { + metrics::gauge!( + METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES, + "pool_index" => source_index.to_string() + ) + .set(reservation.predicted_physical_bytes as f64); + metrics::gauge!( + METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES, + "pool_index" => source_index.to_string() + ) + .set(reservation.remaining_peak_physical_bytes() as f64); + let pool_count = meta.pools.len(); + let info = meta + .pools + .get_mut(source_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_index))?; + info.capacity_blocked_reason = None; + info.capacity_reservation = Some(reservation); + } + if meta.version != POOL_META_GENERATION_VERSION { + meta.version = POOL_META_VERSION; + } + Ok(()) } fn ensure_decommission_start_target_capacity( meta: &PoolMeta, indices: &[usize], - space_infos: &[(usize, PoolSpaceInfo)], + capacity_infos: &[DecommissionPoolCapacityInfo], ) -> Result<()> { - let mut source_used = 0usize; - let mut target_free = 0usize; - - for (idx, info) in space_infos { - if indices.contains(idx) { - source_used = source_used.saturating_add(info.used); - } else if meta.pools.get(*idx).is_some_and(is_decommission_start_active_pool) { - target_free = target_free.saturating_add(info.free); + let generation = next_decommission_capacity_generation(meta)?; + let mut projected = meta.clone(); + let first_idx = indices.first().copied(); + for idx in indices.iter().copied() { + let capacity = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == idx) + .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?; + if Some(idx) == first_idx { + projected.decommission(idx, capacity.space)?; + } else { + projected.queue_decommission(idx, capacity.space)?; } } + reserve_decommission_start_target_capacity( + &mut projected, + indices, + capacity_infos, + uuid::Uuid::new_v4(), + generation, + OffsetDateTime::now_utc(), + ) +} - let required = decommission_target_capacity_required(source_used); - if target_free < required { - return Err(Error::other(format!( - "failed to start decommission: insufficient target pool capacity: required {required} bytes available {target_free} bytes for {source_used} bytes used in decommission pools with {DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT}% overhead" +fn recover_decommission_capacity_reservations( + meta: &mut PoolMeta, + capacity_infos: &[DecommissionPoolCapacityInfo], + now: OffsetDateTime, +) -> Result> { + ensure_decommission_capacity_writer_supported(meta)?; + let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::>(); + active_indices.sort_unstable(); + let missing_indices = active_indices + .iter() + .copied() + .filter(|idx| { + meta.pools + .get(*idx) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .is_none_or(|reservation| !reservation.active()) + }) + .collect::>(); + let generation = next_decommission_capacity_generation(meta)?; + reserve_decommission_start_target_capacity(meta, &missing_indices, capacity_infos, uuid::Uuid::new_v4(), generation, now)?; + for idx in missing_indices { + if let Some(reservation) = meta + .pools + .get_mut(idx) + .and_then(|pool| pool.decommission.as_mut()) + .and_then(|info| info.capacity_reservation.as_mut()) + { + reservation.recovered_at = Some(now); + } + } + Ok(active_indices) +} + +fn signed_capacity_difference(observed: usize, predicted: usize) -> i64 { + if observed >= predicted { + i64::try_from(observed.saturating_sub(predicted)).unwrap_or(i64::MAX) + } else { + -i64::try_from(predicted.saturating_sub(observed)).unwrap_or(i64::MAX) + } +} + +fn active_decommission_target_reservations(meta: &PoolMeta) -> HashMap { + let mut target_reservations = HashMap::new(); + for reservation in meta + .pools + .iter() + .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref()) + .filter(|reservation| reservation.active()) + { + for target in &reservation.targets { + let reserved = target_reservations.entry(target.pool_index).or_insert(0usize); + *reserved = reserved.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies)); + } + } + target_reservations +} + +fn observe_decommission_capacity_reservation( + info: &mut PoolDecommissionInfo, + _capacity_infos: &[DecommissionPoolCapacityInfo], + _active_target_reservations: &HashMap, +) { + let Some(reservation) = info.capacity_reservation.as_mut() else { + return; + }; + reservation.prediction_error_bytes = + signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes); + metrics::gauge!( + METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES, + "pool_index" => reservation.source_pool_index.to_string() + ) + .set(reservation.prediction_error_bytes as f64); + metrics::histogram!( + METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES, + "pool_index" => reservation.source_pool_index.to_string() + ) + .record(reservation.prediction_error_bytes.unsigned_abs() as f64); +} + +fn ensure_decommission_capacity_reservations_available( + meta: &PoolMeta, + capacity_infos: &[DecommissionPoolCapacityInfo], + phase: &'static str, +) -> Result<()> { + let mut required_by_target = HashMap::::new(); + let mut inflight_by_target = HashMap::::new(); + let mut pending_by_target = HashMap::::new(); + for source_index in active_decommission_source_indices(meta) { + let info = meta.pools[source_index] + .decommission + .as_ref() + .ok_or_else(|| decommission_metadata_not_initialized_error("check decommission capacity reservation"))?; + let reservation = info.capacity_reservation.as_ref().ok_or_else(|| { + decommission_capacity_blocked_error(format!( + "active decommission source pool {source_index} has no durable capacity reservation" + )) + })?; + if !reservation.active() { + return Err(decommission_capacity_blocked_error(format!( + "active decommission source pool {source_index} has a released capacity reservation" + ))); + } + metrics::gauge!( + METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES, + "pool_index" => source_index.to_string() + ) + .set(reservation.remaining_peak_physical_bytes() as f64); + for target in &reservation.targets { + let temporary_budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies) + / 1usize.saturating_add(reservation.temporary_copies); + if target.inflight_physical_bytes > temporary_budget { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_index} target pool {} has {} inflight physical bytes, exceeding its {temporary_budget}-byte temporary-copy reservation during {phase}", + target.pool_index, target.inflight_physical_bytes + ))); + } + if target.pending_physical_bytes > temporary_budget { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_index} target pool {} has {} pending physical bytes, exceeding its {temporary_budget}-byte committed-copy reservation during {phase}", + target.pool_index, target.pending_physical_bytes + ))); + } + let required = required_by_target.entry(target.pool_index).or_default(); + *required = required.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies)); + let inflight = inflight_by_target.entry(target.pool_index).or_default(); + *inflight = inflight.saturating_add(target.inflight_physical_bytes); + let pending = pending_by_target.entry(target.pool_index).or_default(); + *pending = pending.saturating_add(target.pending_physical_bytes); + } + } + for (target_pool_index, required) in required_by_target { + let available = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == target_pool_index) + .map(|capacity| capacity.physical_free) + .ok_or_else(|| { + decommission_capacity_blocked_error(format!( + "capacity snapshot is missing target pool {target_pool_index} during {phase}" + )) + })? + .saturating_add(inflight_by_target.get(&target_pool_index).copied().unwrap_or_default()) + .saturating_add(pending_by_target.get(&target_pool_index).copied().unwrap_or_default()); + if available >= required { + continue; + } + metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1); + return Err(decommission_capacity_blocked_error(format!( + "target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}" ))); } - Ok(()) } +fn ensure_external_decommission_target_admission(meta: &PoolMeta, target_pool_index: usize, phase: &'static str) -> Result<()> { + if active_decommission_source_indices(meta).into_iter().any(|source_pool_index| { + meta.pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .is_none_or(|reservation| !reservation.active()) + }) { + metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1); + return Err(Error::SlowDown); + } + let reserved = active_decommission_target_reservations(meta) + .get(&target_pool_index) + .copied() + .unwrap_or_default(); + if reserved == 0 { + return Ok(()); + } + metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1); + Err(Error::SlowDown) +} + +fn ensure_decommission_target_owner_admission( + meta: &PoolMeta, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + target_physical_bytes: usize, + now: OffsetDateTime, +) -> Result<()> { + let reservation = meta + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_owner(owner, now)) + .ok_or_else(|| Error::SlowDown)?; + let target = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| Error::SlowDown)?; + let required_peak = target_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies)); + let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies); + if required_peak <= remaining { + return Ok(()); + } + Err(decommission_capacity_blocked_error(format!( + "source pool {} target pool {target_pool_index} operation requires {required_peak} physical bytes, but only {remaining} reserved bytes remain", + owner.source_pool_index + ))) +} + +fn reserve_decommission_target_pending( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + requested_physical_bytes: usize, + mutation_id: uuid::Uuid, + now: OffsetDateTime, +) -> Result { + if requested_physical_bytes == 0 { + return Ok(0); + } + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("reserve target capacity mutation"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared before target write"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared before target write"))?; + if target.pending_physical_bytes == 0 { + target.pending_mutation_id = None; + } + if target.pending_physical_bytes > 0 { + match target.pending_mutation_id { + Some(pending_mutation_id) if pending_mutation_id == mutation_id => { + if requested_physical_bytes <= target.pending_physical_bytes { + return Ok(0); + } + let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies) + / 1usize.saturating_add(reservation.temporary_copies); + if requested_physical_bytes > budget { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} mutation intent size changed during retry" + ))); + } + let additional = requested_physical_bytes.saturating_sub(target.pending_physical_bytes); + target.pending_physical_bytes = requested_physical_bytes; + reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional); + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + return Ok(additional); + } + _ => { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent" + ))); + } + } + } + let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies) + / 1usize.saturating_add(reservation.temporary_copies); + if requested_physical_bytes > budget { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} mutation requires {requested_physical_bytes} physical bytes, but only {budget} committed-copy bytes remain" + ))); + } + let additional = requested_physical_bytes; + target.pending_mutation_id = Some(mutation_id); + target.pending_physical_bytes = target.pending_physical_bytes.saturating_add(additional); + reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional); + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + Ok(additional) +} + +fn resolve_decommission_target_pending( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + resolved_physical_bytes: usize, + mutation_id: uuid::Uuid, +) -> Result<()> { + if resolved_physical_bytes == 0 { + return Ok(()); + } + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let reservation = pool + .decommission + .as_mut() + .and_then(|info| info.capacity_reservation.as_mut()) + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared after target write"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared after target write"))?; + if target.pending_physical_bytes > 0 && target.pending_mutation_id != Some(mutation_id) { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} pending capacity intent belongs to another mutation" + ))); + } + let resolved = target.pending_physical_bytes.min(resolved_physical_bytes); + target.pending_physical_bytes = target.pending_physical_bytes.saturating_sub(resolved); + reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(resolved); + if target.pending_physical_bytes == 0 { + target.pending_mutation_id = None; + } + Ok(()) +} + +fn release_decommission_target_temporary_mutation( + target: &mut DecommissionCapacityTarget, + mutation_id: uuid::Uuid, + maximum_physical_bytes: usize, +) -> usize { + let Some(index) = target + .temporary_mutations + .iter() + .position(|mutation| mutation.mutation_id == mutation_id) + else { + return 0; + }; + let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes); + target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released); + if target.temporary_mutations[index].physical_bytes == 0 { + target.temporary_mutations.remove(index); + } + target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); + released +} + +struct DecommissionTargetConsumption { + committed_data_bytes: usize, + target_physical_bytes: usize, + observed_physical_bytes: usize, +} + +fn record_decommission_target_consumption( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + consumption: DecommissionTargetConsumption, + mutation_id: uuid::Uuid, + now: OffsetDateTime, +) -> Result<()> { + let DecommissionTargetConsumption { + committed_data_bytes, + target_physical_bytes, + observed_physical_bytes, + } = consumption; + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity consumption"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?; + let remaining_target_bytes = target.remaining_reserved_physical_bytes(reservation.temporary_copies) + / 1usize.saturating_add(reservation.temporary_copies); + let remaining_total_bytes = reservation + .predicted_physical_bytes + .saturating_sub(reservation.consumed_target_physical_bytes); + let consumed = target_physical_bytes.min(remaining_target_bytes).min(remaining_total_bytes); + target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed); + target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes); + let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty(); + let released_inflight = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX); + let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations { + let released = target.inflight_physical_bytes.min(consumed); + target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); + released + } else { + released_inflight + }; + reservation.consumed_target_physical_bytes = reservation.consumed_target_physical_bytes.saturating_add(consumed); + let committed = committed_data_bytes.min( + reservation + .source_data_equivalent_bytes + .saturating_sub(reservation.committed_data_bytes), + ); + reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(committed); + reservation.observed_target_physical_bytes = reservation + .observed_target_physical_bytes + .saturating_add(observed_physical_bytes); + reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released_inflight); + reservation.prediction_error_bytes = + signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes); + renew_decommission_capacity_reservation(reservation, now, true); + info.capacity_blocked_reason = None; + pool.last_update = now; + Ok(()) +} + +fn record_decommission_target_inflight( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + observed_physical_bytes: usize, + mutation_id: uuid::Uuid, + now: OffsetDateTime, +) -> Result<()> { + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity inflight bytes"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?; + if observed_physical_bytes > 0 { + if let Some(mutation) = target + .temporary_mutations + .iter_mut() + .find(|mutation| mutation.mutation_id == mutation_id) + { + mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes); + } else { + target.temporary_mutations.push(DecommissionCapacityTemporaryMutation { + mutation_id, + physical_bytes: observed_physical_bytes, + }); + } + } + target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes); + target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes); + reservation.observed_target_physical_bytes = reservation + .observed_target_physical_bytes + .saturating_add(observed_physical_bytes); + reservation.inflight_target_physical_bytes = reservation + .inflight_target_physical_bytes + .saturating_add(observed_physical_bytes); + reservation.prediction_error_bytes = + signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes); + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + Ok(()) +} + +fn record_decommission_target_observation( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + observed_physical_bytes: usize, + now: OffsetDateTime, +) -> Result<()> { + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity observation"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?; + target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes); + reservation.observed_target_physical_bytes = reservation + .observed_target_physical_bytes + .saturating_add(observed_physical_bytes); + reservation.prediction_error_bytes = + signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes); + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + Ok(()) +} + +fn release_decommission_target_inflight( + meta: &mut PoolMeta, + source_pool_index: usize, + target_pool_index: usize, + released_physical_bytes: usize, + mutation_id: uuid::Uuid, + confirmed_absent: bool, + now: OffsetDateTime, +) -> Result { + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("release target capacity inflight bytes"))?; + let reservation = info + .capacity_reservation + .as_mut() + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?; + let target = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?; + let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty(); + let released = release_decommission_target_temporary_mutation( + target, + mutation_id, + if confirmed_absent { + usize::MAX + } else { + released_physical_bytes + }, + ); + let released = if released == 0 && !has_scoped_temporary_mutations { + let released = target.inflight_physical_bytes.min(released_physical_bytes); + target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released); + released + } else { + released + }; + reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released); + let cleared_pending = if confirmed_absent && target.pending_mutation_id == Some(mutation_id) { + let cleared = target.pending_physical_bytes; + target.pending_physical_bytes = 0; + target.pending_mutation_id = None; + reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(cleared); + cleared + } else { + 0 + }; + let changed = released > 0 || cleared_pending > 0; + if changed { + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = now; + } + Ok(changed) +} + fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> { if idx >= pool_count { return Err(invalid_decommission_pool_index_error(pool_count, idx)); @@ -2999,6 +4060,13 @@ impl PoolMetaWriteState { Arc::clone(&self.aborted_transaction) } + #[cfg(test)] + pub(crate) fn independent_clone_for_test(&self) -> Self { + let mut cloned = self.clone(); + cloned.aborted_transaction = Arc::new(AtomicBool::new(self.aborted_transaction.load(Ordering::Acquire))); + cloned + } + #[cfg(any(test, feature = "test-util"))] fn for_test_bootstrap() -> Self { Self { @@ -4202,6 +5270,10 @@ struct PersistedPoolDecommissionInfo { pub terminal_reload_failures: Vec, #[serde(rename = "unresolvedEntries", default)] pub unresolved_entries: Vec, + #[serde(rename = "capacityReservation", default)] + pub capacity_reservation: Option, + #[serde(rename = "capacityBlockedReason", default)] + pub capacity_blocked_reason: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -4372,11 +5444,21 @@ impl TryFrom for PoolStatus { type Error = Error; fn try_from(value: PersistedPoolStatus) -> Result { + let decommission = value.decommission.map(TryInto::try_into).transpose()?; + if decommission + .as_ref() + .and_then(|info: &PoolDecommissionInfo| info.capacity_reservation.as_ref()) + .is_some_and(|reservation| reservation.source_pool_index != value.id) + { + return Err(Error::other( + "pool metadata load failed: decommission capacity reservation source pool does not match its owner", + )); + } Ok(Self { id: value.id, cmd_line: value.cmd_line, last_update: value.last_update, - decommission: value.decommission.map(TryInto::try_into).transpose()?, + decommission, }) } } @@ -4412,6 +5494,7 @@ impl TryFrom for PoolDecommissionInfo { fn try_from(value: PersistedPoolDecommissionInfo) -> Result { validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?; + validate_decommission_capacity_reservation(value.capacity_reservation.as_ref())?; Ok(Self { start_time: value.start_time, start_size: value.start_size, @@ -4433,6 +5516,8 @@ impl TryFrom for PoolDecommissionInfo { bytes_failed: value.bytes_failed, terminal_reload_attempt_at: value.terminal_reload_attempt_at, terminal_reload_failures: value.terminal_reload_failures, + capacity_reservation: value.capacity_reservation, + capacity_blocked_reason: value.capacity_blocked_reason, unresolved_entries: value.unresolved_entries, progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed), progress_save_retry_after: None, @@ -4466,6 +5551,8 @@ impl TryFrom for PoolDecommissionInfo { bytes_failed: value.bytes_failed, terminal_reload_attempt_at: value.terminal_reload_attempt_at, terminal_reload_failures: value.terminal_reload_failures, + capacity_reservation: None, + capacity_blocked_reason: None, unresolved_entries: Vec::new(), progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed), progress_save_retry_after: None, @@ -4499,6 +5586,8 @@ impl TryFrom for PoolDecommissionInfo { bytes_failed: value.bytes_failed, terminal_reload_attempt_at: None, terminal_reload_failures: Vec::new(), + capacity_reservation: None, + capacity_blocked_reason: None, unresolved_entries: Vec::new(), progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed), progress_save_retry_after: None, @@ -4568,6 +5657,8 @@ impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo { bytes_failed: value.bytes_failed, terminal_reload_attempt_at: value.terminal_reload_attempt_at, terminal_reload_failures: value.terminal_reload_failures.clone(), + capacity_reservation: value.capacity_reservation.clone(), + capacity_blocked_reason: value.capacity_blocked_reason.clone(), unresolved_entries: value.unresolved_entries.clone(), } } @@ -4762,6 +5853,21 @@ impl PoolMeta { queued: info.queued, counted_items: info.counted_items(), checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta), + capacity_operation_id: info + .capacity_reservation + .as_ref() + .filter(|reservation| reservation.lease_active_at(now)) + .map(|reservation| reservation.operation_id), + capacity_owner_nonce: info + .capacity_reservation + .as_ref() + .filter(|reservation| reservation.lease_active_at(now)) + .map(|reservation| reservation.owner_nonce), + capacity_lease_expires_at: info + .capacity_reservation + .as_ref() + .filter(|reservation| reservation.lease_active_at(now)) + .map(|_| now + DECOMMISSION_CAPACITY_RESERVATION_TTL), })) } @@ -4782,6 +5888,18 @@ impl PoolMeta { info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items); info.progress_save_retry_after = None; + if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = ( + checkpoint.capacity_operation_id, + checkpoint.capacity_owner_nonce, + checkpoint.capacity_lease_expires_at, + info.capacity_reservation.as_mut(), + ) && reservation.operation_id == operation_id + && reservation.owner_nonce == owner_nonce + && reservation.active() + { + reservation.renewed_at = checkpoint.checkpoint_at; + reservation.expires_at = expires_at; + } pool.last_update = pool.last_update.max(checkpoint.checkpoint_at); true } @@ -4904,18 +6022,22 @@ impl PoolMeta { } else { POOL_META_V1_VERSION }; - if version == POOL_META_V1_VERSION - && self - .pools - .iter() - .filter_map(|pool| pool.decommission.as_ref()) - .any(|info| !info.unresolved_entries.is_empty()) - { - return Err(Error::other(format!( - "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2", - rustfs_config::ENV_POOL_META_V2_WRITE, - rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, - ))); + if version == POOL_META_V1_VERSION { + let mut decommission_infos = self.pools.iter().filter_map(|pool| pool.decommission.as_ref()); + if decommission_infos.clone().any(|info| !info.unresolved_entries.is_empty()) { + return Err(Error::other(format!( + "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2", + rustfs_config::ENV_POOL_META_V2_WRITE, + rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, + ))); + } + if decommission_infos.any(|info| info.capacity_reservation.is_some()) { + return Err(Error::DecommissionCapacity(format!( + "pool metadata V2 is required to persist decommission capacity reservations; enable both {} and {} only after every reader and writer supports V2", + rustfs_config::ENV_POOL_META_V2_WRITE, + rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, + ))); + } } let mut data = Vec::new(); data.write_u16::(POOL_META_FORMAT)?; @@ -5337,6 +6459,8 @@ impl PoolMeta { pd.start_time = None; pd.terminal_reload_attempt_at = None; pd.terminal_reload_failures.clear(); + pd.capacity_blocked_reason = None; + release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_CANCELED, last_update); let Some(stats) = self.pools.get_mut(idx) else { return false; @@ -5375,6 +6499,8 @@ impl PoolMeta { pd.start_time = None; pd.terminal_reload_attempt_at = None; pd.terminal_reload_failures.clear(); + pd.capacity_blocked_reason = None; + release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_FAILED, last_update); let Some(stats) = self.pools.get_mut(idx) else { return false; @@ -5462,6 +6588,8 @@ impl PoolMeta { pd.complete = true; pd.terminal_reload_attempt_at = None; pd.terminal_reload_failures.clear(); + pd.capacity_blocked_reason = None; + release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, last_update); let Some(stats) = self.pools.get_mut(idx) else { return false; @@ -5573,6 +6701,28 @@ impl PoolMeta { Ok(true) } + fn mark_decommission_capacity_blocked(&mut self, idx: usize, reason: String, now: OffsetDateTime) -> Result { + let pool_count = self.pools.len(); + let pool = self + .pools + .get_mut(idx) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, idx))?; + let info = pool + .decommission + .as_mut() + .ok_or_else(|| decommission_metadata_not_initialized_error("pause decommission for target capacity"))?; + if !is_decommission_active(info.complete, info.failed, info.canceled) { + return Ok(false); + } + if let Some(reservation) = info.capacity_reservation.as_mut().filter(|reservation| reservation.active()) { + renew_decommission_capacity_reservation(reservation, now, true); + } + let changed = info.capacity_blocked_reason.as_deref() != Some(reason.as_str()); + info.capacity_blocked_reason = Some(reason); + pool.last_update = now; + Ok(changed) + } + pub fn promote_queued_decommission(&mut self, idx: usize) -> bool { self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None) } @@ -5794,6 +6944,227 @@ pub struct DecommissionUnresolvedEntry { pub reason: String, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DecommissionErasureLayout { + pub data: usize, + pub parity: usize, +} + +impl DecommissionErasureLayout { + fn width(self) -> usize { + self.data.saturating_add(self.parity) + } + + fn is_valid(self) -> bool { + self.data > 0 && self.data.checked_add(self.parity).is_some() + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DecommissionCapacityTarget { + pub pool_index: usize, + pub layout: DecommissionErasureLayout, + pub physical_total_at_reservation: usize, + pub physical_free_at_reservation: usize, + pub reserved_physical_bytes: usize, + #[serde(default)] + pub consumed_physical_bytes: usize, + #[serde(default)] + pub observed_physical_bytes: usize, + #[serde(default)] + pub inflight_physical_bytes: usize, + #[serde(default)] + pub pending_physical_bytes: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_mutation_id: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub temporary_mutations: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DecommissionCapacityTemporaryMutation { + pub mutation_id: uuid::Uuid, + pub physical_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DecommissionCapacityReservation { + pub model_version: u16, + pub operation_id: uuid::Uuid, + pub generation: u64, + pub owner_nonce: uuid::Uuid, + pub source_pool_index: usize, + pub source_layout: DecommissionErasureLayout, + pub source_physical_total_bytes: usize, + pub source_physical_bytes: usize, + pub source_data_equivalent_bytes: usize, + pub predicted_physical_bytes: usize, + pub temporary_copies: usize, + pub temporary_physical_bytes: usize, + pub peak_physical_bytes: usize, + pub committed_data_bytes: usize, + #[serde(default)] + pub consumed_target_physical_bytes: usize, + #[serde(default)] + pub observed_target_physical_bytes: usize, + #[serde(default)] + pub inflight_target_physical_bytes: usize, + #[serde(default)] + pub pending_target_physical_bytes: usize, + pub prediction_error_bytes: i64, + pub targets: Vec, + #[serde(with = "time::serde::rfc3339")] + pub created_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub renewed_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub expires_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339::option", default)] + pub recovered_at: Option, + #[serde(with = "time::serde::rfc3339::option", default)] + pub released_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub release_reason: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DecommissionCapacityOwner { + pub(crate) source_pool_index: usize, + pub(crate) operation_id: uuid::Uuid, + pub(crate) generation: u64, + pub(crate) owner_nonce: uuid::Uuid, + pub(crate) mutation_id: Option, +} + +impl DecommissionCapacityOwner { + pub(crate) fn apply_to(self, opts: &mut ObjectOptions) { + opts.src_pool_idx = self.source_pool_index; + let admission = opts + .decommission_capacity + .get_or_insert_with(|| Box::new(DecommissionCapacityOptions::default())); + admission.operation_id = Some(self.operation_id); + admission.generation = Some(self.generation); + admission.owner_nonce = Some(self.owner_nonce); + admission.mutation_id = self.mutation_id; + } + + pub(crate) fn from_options(opts: &ObjectOptions) -> Option { + let capacity = opts.decommission_capacity.as_deref()?; + Some(Self { + source_pool_index: opts.src_pool_idx, + operation_id: capacity.operation_id?, + generation: capacity.generation?, + owner_nonce: capacity.owner_nonce?, + mutation_id: capacity.mutation_id, + }) + } + + pub(crate) fn with_mutation_id(self, mutation_id: uuid::Uuid) -> Self { + Self { + mutation_id: Some(mutation_id), + ..self + } + } +} + +pub(crate) fn decommission_capacity_mutation_id( + owner: DecommissionCapacityOwner, + bucket: &str, + object: &str, + version_id: Option<&str>, + delete_marker: bool, + mod_time: Option, +) -> uuid::Uuid { + let mut hasher = Sha256::new(); + hasher.update(owner.operation_id.as_bytes()); + hasher.update(owner.generation.to_le_bytes()); + hasher.update(owner.source_pool_index.to_le_bytes()); + for value in [bucket, object] { + hasher.update((value.len() as u64).to_le_bytes()); + hasher.update(value.as_bytes()); + } + hasher.update([u8::from(delete_marker)]); + if let Some(version_id) = version_id { + hasher.update([1]); + hasher.update((version_id.len() as u64).to_le_bytes()); + hasher.update(version_id.as_bytes()); + } else { + hasher.update([0]); + } + if let Some(mod_time) = mod_time { + hasher.update([1]); + hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes()); + } else { + hasher.update([0]); + } + let digest = hasher.finalize(); + let mut bytes = [0; 16]; + bytes.copy_from_slice(&digest[..16]); + uuid::Uuid::from_bytes(bytes) +} + +pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) { + if opts + .decommission_capacity + .as_deref() + .is_none_or(|capacity| capacity.mutation_id.is_some()) + { + return; + } + let Some(owner) = DecommissionCapacityOwner::from_options(opts) else { + return; + }; + let mutation_id = + decommission_capacity_mutation_id(owner, bucket, object, opts.version_id.as_deref(), opts.delete_marker, opts.mod_time); + if let Some(capacity) = opts.decommission_capacity.as_mut() { + capacity.mutation_id = Some(mutation_id); + } +} + +impl DecommissionCapacityReservation { + fn active(&self) -> bool { + self.released_at.is_none() + } + + fn lease_active_at(&self, now: OffsetDateTime) -> bool { + self.active() && self.expires_at > now + } + + fn admits_owner(&self, owner: DecommissionCapacityOwner, now: OffsetDateTime) -> bool { + self.lease_active_at(now) + && self.source_pool_index == owner.source_pool_index + && self.operation_id == owner.operation_id + && self.generation == owner.generation + && self.owner_nonce == owner.owner_nonce + } + + fn admits_cleanup_owner(&self, owner: DecommissionCapacityOwner) -> bool { + self.active() + && self.source_pool_index == owner.source_pool_index + && self.operation_id == owner.operation_id + && self.generation == owner.generation + } + + fn remaining_peak_physical_bytes(&self) -> usize { + self.predicted_physical_bytes + .saturating_sub(self.consumed_target_physical_bytes) + .saturating_mul(1usize.saturating_add(self.temporary_copies)) + } +} + +impl DecommissionCapacityTarget { + fn remaining_reserved_physical_bytes(&self, temporary_copies: usize) -> usize { + self.reserved_physical_bytes.saturating_sub( + self.consumed_physical_bytes + .saturating_mul(1usize.saturating_add(temporary_copies)), + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct PoolDecommissionInfo { #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] @@ -5838,6 +7209,10 @@ pub struct PoolDecommissionInfo { pub terminal_reload_attempt_at: Option, #[serde(rename = "terminalReloadFailures", default)] pub terminal_reload_failures: Vec, + #[serde(rename = "capacityReservation", default, skip_serializing_if = "Option::is_none")] + pub capacity_reservation: Option, + #[serde(rename = "capacityBlockedReason", default, skip_serializing_if = "Option::is_none")] + pub capacity_blocked_reason: Option, #[serde(skip)] pub unresolved_entries: Vec, #[serde(skip)] @@ -5852,6 +7227,9 @@ struct DecommissionProgressCheckpoint { queued: bool, counted_items: usize, checkpoint_at: OffsetDateTime, + capacity_operation_id: Option, + capacity_owner_nonce: Option, + capacity_lease_expires_at: Option, } impl PoolDecommissionInfo { @@ -5874,6 +7252,8 @@ impl PoolDecommissionInfo { || self.bytes_failed > 0 || self.terminal_reload_attempt_at.is_some() || !self.terminal_reload_failures.is_empty() + || self.capacity_reservation.is_some() + || self.capacity_blocked_reason.is_some() || !self.unresolved_entries.is_empty() } @@ -5940,19 +7320,72 @@ impl PoolDecommissionInfo { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PoolSpaceInfo { pub free: usize, pub total: usize, pub used: usize, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DecommissionPoolCapacityInfo { + pool_index: usize, + space: PoolSpaceInfo, + layout: DecommissionErasureLayout, + physical_free: usize, + physical_total: usize, + physical_used: usize, +} + +impl DecommissionPoolCapacityInfo { + #[cfg(test)] + pub(crate) fn for_test( + pool_index: usize, + layout: DecommissionErasureLayout, + physical_free: usize, + physical_total: usize, + physical_used: usize, + ) -> Self { + Self { + pool_index, + space: PoolSpaceInfo { + free: physical_free, + total: physical_total, + used: physical_used, + }, + layout, + physical_free, + physical_total, + physical_used, + } + } + + #[cfg(test)] + fn from_logical(pool_index: usize, space: PoolSpaceInfo) -> Self { + Self { + pool_index, + space, + layout: DecommissionErasureLayout { data: 1, parity: 0 }, + physical_free: space.free, + physical_total: space.total, + physical_used: space.used, + } + } +} + #[cfg(test)] type DecommissionSpaceInfoOverrides = std::sync::Mutex>>; +#[cfg(test)] +type DecommissionCapacityInfoOverrides = + std::sync::Mutex>>>; + #[cfg(test)] static DECOMMISSION_SPACE_INFO_OVERRIDES: std::sync::OnceLock = std::sync::OnceLock::new(); +#[cfg(test)] +static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock = std::sync::OnceLock::new(); + #[cfg(test)] pub(crate) fn set_decommission_space_info_override_for_test(store_id: uuid::Uuid, space_infos: Vec<(usize, PoolSpaceInfo)>) { DECOMMISSION_SPACE_INFO_OVERRIDES @@ -5971,6 +7404,285 @@ fn take_decommission_space_info_override_for_test(store_id: uuid::Uuid) -> Optio .remove(&store_id) } +#[cfg(test)] +pub(crate) fn set_decommission_capacity_info_overrides_for_test( + store_id: uuid::Uuid, + snapshots: Vec>, +) { + DECOMMISSION_CAPACITY_INFO_OVERRIDES + .get_or_init(|| std::sync::Mutex::new(HashMap::new())) + .lock() + .expect("decommission capacity info override should not be poisoned") + .insert(store_id, snapshots.into()); +} + +#[cfg(test)] +fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option> { + let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES + .get_or_init(|| std::sync::Mutex::new(HashMap::new())) + .lock() + .expect("decommission capacity info override should not be poisoned"); + let snapshot = overrides.get_mut(&store_id)?.pop_front(); + if overrides.get(&store_id).is_some_and(std::collections::VecDeque::is_empty) { + overrides.remove(&store_id); + } + snapshot +} + +#[cfg(test)] +struct DecommissionCapacityLockOrderBarrierState { + owner_store_id: uuid::Uuid, + external_store_id: uuid::Uuid, + owner_arrived: tokio::sync::Notify, + owner_release: tokio::sync::Notify, + external_capacity_released: tokio::sync::Notify, + external_object_capacity_probe_acquired: tokio::sync::Notify, + external_object_capacity_probe_release: tokio::sync::Notify, + external_object_capacity_probe_paused: AtomicBool, + external_object_commit_phase_started: tokio::sync::Notify, + external_object_commit_phase_release: tokio::sync::Notify, + external_object_commit_phase_paused: AtomicBool, + external_heal_operation_started: tokio::sync::Notify, + external_heal_target_lock_attempted: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct DecommissionCapacityLockOrderBarrier { + state: Arc, +} + +#[cfg(test)] +static DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DecommissionCapacityLockOrderBarrier { + pub(crate) fn install(owner_store_id: uuid::Uuid, external_store_id: uuid::Uuid) -> Self { + let state = Arc::new(DecommissionCapacityLockOrderBarrierState { + owner_store_id, + external_store_id, + owner_arrived: tokio::sync::Notify::new(), + owner_release: tokio::sync::Notify::new(), + external_capacity_released: tokio::sync::Notify::new(), + external_object_capacity_probe_acquired: tokio::sync::Notify::new(), + external_object_capacity_probe_release: tokio::sync::Notify::new(), + external_object_capacity_probe_paused: AtomicBool::new(false), + external_object_commit_phase_started: tokio::sync::Notify::new(), + external_object_commit_phase_release: tokio::sync::Notify::new(), + external_object_commit_phase_paused: AtomicBool::new(false), + external_heal_operation_started: tokio::sync::Notify::new(), + external_heal_target_lock_attempted: tokio::sync::Notify::new(), + }); + let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned"); + assert!(slot.is_none(), "decommission capacity lock-order barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_owner_paused(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified()) + .await + .expect("owned capacity mutation should reach admission before its metadata write"); + } + + pub(crate) async fn wait_until_external_capacity_released(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_capacity_released.notified()) + .await + .expect("external mutation should release capacity before waiting for the object namespace"); + } + + pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) { + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.state.external_object_capacity_probe_acquired.notified(), + ) + .await + .expect("external object mutation should acquire its no-active capacity probe"); + } + + pub(crate) async fn wait_until_external_object_commit_phase_started(&self) { + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.state.external_object_commit_phase_started.notified(), + ) + .await + .expect("external object mutation should reach its staged commit phase before waiting for the namespace"); + } + + pub(crate) async fn wait_until_external_heal_operation_started(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_heal_operation_started.notified()) + .await + .expect("external heal should reach the target operation after capacity admission"); + } + + pub(crate) async fn wait_until_external_heal_target_lock_attempted(&self) { + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.state.external_heal_target_lock_attempted.notified(), + ) + .await + .expect("external heal should attempt the target namespace lock after capacity admission"); + } + + pub(crate) fn release_owner(&self) { + self.state.owner_release.notify_one(); + } + + pub(crate) fn pause_external_object_commit_phase(&self) { + self.state.external_object_commit_phase_paused.store(true, Ordering::Release); + } + + pub(crate) fn release_external_object_commit_phase(&self) { + self.state.external_object_commit_phase_release.notify_one(); + } + + pub(crate) fn pause_external_object_capacity_probe(&self) { + self.state + .external_object_capacity_probe_paused + .store(true, Ordering::Release); + } + + pub(crate) fn release_external_object_capacity_probe(&self) { + self.state.external_object_capacity_probe_release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for DecommissionCapacityLockOrderBarrier { + fn drop(&mut self) { + self.state.owner_release.notify_one(); + self.state.external_object_capacity_probe_release.notify_one(); + self.state.external_object_commit_phase_release.notify_one(); + let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.owner_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.owner_arrived.notify_one(); + barrier.owner_release.notified().await; + } +} + +#[cfg(test)] +pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.external_capacity_released.notify_one(); + } +} + +#[cfg(test)] +pub(crate) fn notify_decommission_external_object_capacity_probe_acquired(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.external_object_capacity_probe_acquired.notify_one(); + } +} + +#[cfg(test)] +pub(crate) async fn wait_for_decommission_external_object_capacity_probe_release(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier + && barrier.external_object_capacity_probe_paused.load(Ordering::Acquire) + { + barrier.external_object_capacity_probe_release.notified().await; + } +} + +#[cfg(test)] +pub(crate) fn notify_decommission_external_object_commit_phase_started(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.external_object_commit_phase_started.notify_one(); + } +} + +#[cfg(test)] +pub(crate) async fn wait_for_decommission_external_object_commit_phase_release(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier + && barrier.external_object_commit_phase_paused.load(Ordering::Acquire) + { + barrier.external_object_commit_phase_release.notified().await; + } +} + +#[cfg(test)] +pub(crate) fn notify_decommission_external_heal_operation_started(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.external_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.external_heal_operation_started.notify_one(); + } +} + +#[cfg(test)] +pub(crate) fn notify_decommission_external_heal_target_lock_attempted() { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .cloned(); + if let Some(barrier) = barrier { + barrier.external_heal_target_lock_attempted.notify_one(); + } +} + #[derive(Debug, Default, Clone)] pub struct DecomBucketInfo { pub name: String, @@ -6077,6 +7789,13 @@ fn decommission_remote_tiered_opts( } } +fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option) -> ObjectOptions { + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut opts); + } + opts +} + fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool { matches!( action, @@ -6314,11 +8033,13 @@ impl ECStore { ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> { write_state.ensure_write_safe(operation)?; load_pool_meta_identity_observing(self.pools.clone(), write_state).await?; - let pool = self - .pools - .first() - .cloned() - .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?; + let pool = self.pools.first().cloned().ok_or_else(|| { + Error::InvalidArgument( + operation.to_string(), + "storage-pools".to_string(), + "no storage pools available".to_string(), + ) + })?; let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; let pool_meta_guard = pool_meta_lock .get_write_lock(get_lock_acquire_timeout()) @@ -6330,6 +8051,608 @@ impl ECStore { Ok((pool_meta_guard, selection.meta)) } + async fn acquire_pool_meta_read_guard( + &self, + write_state: &mut PoolMetaWriteState, + operation: &str, + ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> { + write_state.ensure_write_safe(operation)?; + let pool = self.pools.first().cloned().ok_or_else(|| { + Error::InvalidArgument( + operation.to_string(), + "storage-pools".to_string(), + "no storage pools available".to_string(), + ) + })?; + let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; + let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?; + let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?; + write_state.observe_replicas(selection.replica_state); + write_state.ensure_write_safe(operation)?; + Ok((pool_meta_guard, selection.meta)) + } + + pub(crate) async fn acquire_external_decommission_capacity_fence( + &self, + target_pool_indices: &[usize], + phase: &'static str, + ) -> Result { + Ok(self + .acquire_external_decommission_capacity_fence_with_active_source(target_pool_indices, phase) + .await? + .0) + } + + pub(crate) async fn acquire_external_decommission_capacity_fence_with_active_source( + &self, + target_pool_indices: &[usize], + phase: &'static str, + ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (pool_meta_guard, snapshot) = self + .acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed") + .await?; + for target_pool_index in target_pool_indices.iter().copied() { + ensure_external_decommission_target_admission(&snapshot, target_pool_index, phase)?; + } + let has_active_source = pool_meta_has_active_decommission(&snapshot); + drop(save_guard); + Ok((pool_meta_guard, has_active_source)) + } + + pub(crate) async fn run_decommission_capacity_admitted_mutation( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + operation: F, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, false, false, |_| { + operation() + }) + .await + } + + pub(crate) async fn run_decommission_capacity_admitted_mutation_with_capacity_lease( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + operation: F, + ) -> Result + where + F: FnOnce(Option>) -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, false, false, operation) + .await + } + + pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target( + &self, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + expected_data_bytes: usize, + ) -> Result<()> { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (pool_meta_guard, mut snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed") + .await?; + let source_pool_index = owner.source_pool_index; + let (target_layout, target_pending_physical_bytes, target_consumed_physical_bytes) = { + let reservation = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| { + reservation.active() + && reservation.source_pool_index == owner.source_pool_index + && reservation.operation_id == owner.operation_id + && reservation.generation == owner.generation + && reservation.owner_nonce == owner.owner_nonce + }) + .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation owner is stale"))?; + let target = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation allocation is missing"))?; + (target.layout, target.pending_physical_bytes, target.consumed_physical_bytes) + }; + let mutation_id = owner.mutation_id.ok_or_else(|| { + decommission_capacity_blocked_error("equivalent target reconciliation mutation identity is missing") + })?; + let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?; + if target_pending_physical_bytes == 0 { + if target_consumed_physical_bytes >= expected_target_physical_bytes { + let persisted_info = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .cloned() + .ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent target reconciliation"))?; + let mut pool_meta = self.pool_meta.write().await; + let pool_count = pool_meta.pools.len(); + pool_meta.version = pool_meta.version.max(snapshot.version); + let info = pool_meta + .pools + .get_mut(source_pool_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + info.capacity_reservation = persisted_info.capacity_reservation; + info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + return Ok(()); + } + return Err(decommission_capacity_blocked_error( + "equivalent target has no pending capacity intent to reconcile", + )); + } + if target_pending_physical_bytes < expected_target_physical_bytes { + return Err(decommission_capacity_blocked_error( + "equivalent target pending capacity is smaller than the committed object", + )); + } + + resolve_decommission_target_pending( + &mut snapshot, + source_pool_index, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + )?; + record_decommission_target_consumption( + &mut snapshot, + source_pool_index, + target_pool_index, + DecommissionTargetConsumption { + committed_data_bytes: expected_data_bytes, + target_physical_bytes: expected_target_physical_bytes, + observed_physical_bytes: 0, + }, + mutation_id, + OffsetDateTime::now_utc(), + )?; + let outcome = snapshot + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + pool_meta_guard.lock_lost_signal(), + &[source_pool_index], + ) + .await?; + ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?; + { + let persisted_info = outcome + .committed + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .cloned() + .ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent target reconciliation"))?; + let mut pool_meta = self.pool_meta.write().await; + let pool_count = pool_meta.pools.len(); + pool_meta.version = pool_meta.version.max(outcome.committed.version); + let info = pool_meta + .pools + .get_mut(source_pool_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + info.capacity_reservation = persisted_info.capacity_reservation; + info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + } + ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?; + outcome.disarm(); + Ok(()) + } + + pub(crate) async fn run_decommission_capacity_temporary_mutation( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + operation: F, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, true, false, |_| { + operation() + }) + .await + } + + pub(crate) async fn run_decommission_capacity_temporary_mutation_with_capacity_lease( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + operation: F, + ) -> Result + where + F: FnOnce(Option>) -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, expected_data_bytes, true, false, operation) + .await + } + + pub(crate) async fn has_decommission_capacity_temporary_mutation_state( + &self, + target_pool_index: usize, + owner: DecommissionCapacityOwner, + ) -> bool { + let Some(mutation_id) = owner.mutation_id else { + return false; + }; + self.pool_meta + .read() + .await + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_cleanup_owner(owner)) + .and_then(|reservation| { + reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + }) + .is_some_and(|target| { + (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id)) + || target + .temporary_mutations + .iter() + .any(|mutation| mutation.mutation_id == mutation_id) + }) + } + + pub(crate) async fn decommission_capacity_cleanup_target_indices( + &self, + owner: DecommissionCapacityOwner, + ) -> Result> { + self.pool_meta + .read() + .await + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_cleanup_owner(owner)) + .map(|reservation| reservation.targets.iter().map(|target| target.pool_index).collect()) + .ok_or_else(|| decommission_capacity_blocked_error("decommission multipart cleanup reservation is stale")) + } + + pub(crate) async fn run_decommission_capacity_temporary_release_with_capacity_lease( + &self, + target_pool_index: usize, + capacity_owner: Option, + operation: F, + ) -> Result + where + F: FnOnce(Option>) -> Fut, + Fut: std::future::Future>, + { + self.run_decommission_capacity_mutation(target_pool_index, capacity_owner, None, false, true, operation) + .await + } + + async fn run_decommission_capacity_mutation( + &self, + target_pool_index: usize, + capacity_owner: Option, + expected_data_bytes: Option, + temporary: bool, + temporary_release: bool, + operation: F, + ) -> Result + where + F: FnOnce(Option>) -> Fut, + Fut: std::future::Future>, + { + let mut operation = Some(operation); + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (read_guard, snapshot) = self + .acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed") + .await?; + let admission_now = OffsetDateTime::now_utc(); + let owner = capacity_owner.filter(|owner| { + snapshot + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| { + if temporary_release { + reservation.admits_cleanup_owner(*owner) + } else { + reservation.admits_owner(*owner, admission_now) + } + }) + .is_some_and(|reservation| { + reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index) + }) + }); + if capacity_owner.is_some() && owner.is_none() { + return Err(decommission_capacity_blocked_error( + "decommission target mutation reservation identity is stale", + )); + } + if owner.is_none() { + ensure_external_decommission_target_admission(&snapshot, target_pool_index, "mutation")?; + drop(save_guard); + let capacity_lease = read_guard.lock_lost_signal(); + return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await; + } + + #[cfg(test)] + pause_decommission_capacity_before_owner_write(self.id).await; + drop(read_guard); + let (write_guard, mut snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed") + .await?; + let owner = owner.expect("capacity owner should remain present"); + let mutation_id = owner + .mutation_id + .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?; + let source_pool_index = owner.source_pool_index; + let owner_current = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| { + (if temporary_release { + reservation.admits_cleanup_owner(owner) + } else { + reservation.admits_owner(owner, OffsetDateTime::now_utc()) + }) && reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index) + }) + .is_some(); + if !owner_current { + return Err(decommission_capacity_blocked_error( + "decommission target mutation reservation identity changed before commit", + )); + } + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + if !temporary_release { + ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?; + } + let target_layout = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| { + reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + }) + .map(|target| target.layout) + .ok_or_else(|| Error::SlowDown)?; + let expected_target_physical_bytes = if temporary_release { + 0 + } else { + capacity_target_physical_bytes(expected_data_bytes.unwrap_or(1).max(1), target_layout)? + }; + if temporary { + let (remaining, inflight) = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .and_then(|reservation| { + reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .map(|target| { + ( + target.remaining_reserved_physical_bytes(reservation.temporary_copies) + / 1usize.saturating_add(reservation.temporary_copies), + target.inflight_physical_bytes, + ) + }) + }) + .unwrap_or_default(); + let available = remaining.saturating_sub(inflight); + if expected_target_physical_bytes > available { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} temporary operation requires {expected_target_physical_bytes} physical bytes, but only {available} temporary-copy bytes remain" + ))); + } + } else { + ensure_decommission_target_owner_admission( + &snapshot, + owner, + target_pool_index, + expected_target_physical_bytes, + OffsetDateTime::now_utc(), + )?; + } + let pending_added = if temporary_release { + 0 + } else { + reserve_decommission_target_pending( + &mut snapshot, + source_pool_index, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + OffsetDateTime::now_utc(), + )? + }; + if pending_added > 0 { + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) + .await?; + ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?; + snapshot = outcome.committed.clone(); + { + let persisted_info = snapshot + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .cloned() + .ok_or_else(|| decommission_metadata_not_initialized_error("publish target capacity intent"))?; + let mut pool_meta = self.pool_meta.write().await; + let pool_count = pool_meta.pools.len(); + pool_meta.version = pool_meta.version.max(snapshot.version); + let info = pool_meta + .pools + .get_mut(source_pool_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + info.capacity_reservation = persisted_info.capacity_reservation; + info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + } + ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?; + outcome.disarm(); + } + let capacity_infos = if pending_added > 0 { + self.get_decommission_all_pool_capacity_infos().await? + } else { + capacity_infos + }; + let before_free = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == target_pool_index) + .map(|capacity| capacity.physical_free) + .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?; + + let capacity_lease = write_guard.lock_lost_signal(); + let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await; + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let after_free = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == target_pool_index) + .map(|capacity| capacity.physical_free) + .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing after mutation"))?; + let observed_physical_bytes = before_free.saturating_sub(after_free); + let released_physical_bytes = after_free.saturating_sub(before_free); + let now = OffsetDateTime::now_utc(); + let progress_changed = if temporary_release { + release_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + released_physical_bytes, + mutation_id, + result.is_ok(), + now, + )? + } else if result.is_ok() { + resolve_decommission_target_pending( + &mut snapshot, + source_pool_index, + target_pool_index, + expected_target_physical_bytes, + mutation_id, + )?; + if temporary { + record_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + observed_physical_bytes, + mutation_id, + now, + )?; + } else { + let consumed_physical_bytes = expected_data_bytes + .map(|_| expected_target_physical_bytes) + .unwrap_or(observed_physical_bytes.max(expected_target_physical_bytes)); + let committed_data_bytes = + expected_data_bytes.unwrap_or(capacity_source_data_equivalent(observed_physical_bytes, target_layout)?); + record_decommission_target_consumption( + &mut snapshot, + source_pool_index, + target_pool_index, + DecommissionTargetConsumption { + committed_data_bytes, + target_physical_bytes: consumed_physical_bytes, + observed_physical_bytes, + }, + mutation_id, + now, + )?; + } + true + } else if expected_target_physical_bytes == 0 { + record_decommission_target_inflight( + &mut snapshot, + source_pool_index, + target_pool_index, + observed_physical_bytes, + mutation_id, + now, + )?; + observed_physical_bytes > 0 + } else { + record_decommission_target_observation( + &mut snapshot, + source_pool_index, + target_pool_index, + observed_physical_bytes, + now, + )?; + observed_physical_bytes > 0 + }; + let capacity_result = if temporary_release { + Ok(()) + } else { + ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation") + }; + if let Err(err) = &capacity_result { + snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?; + } + if temporary_release && !progress_changed { + ensure_pool_meta_write_fence(&write_guard, "decommission target capacity cleanup fence failed")?; + capacity_result?; + return result; + } + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) + .await?; + ensure_pool_meta_write_fence(&write_guard, "decommission target capacity progress save failed")?; + { + let persisted_info = outcome + .committed + .pools + .get(source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .cloned() + .ok_or_else(|| decommission_metadata_not_initialized_error("publish target capacity progress"))?; + let mut pool_meta = self.pool_meta.write().await; + let pool_count = pool_meta.pools.len(); + pool_meta.version = pool_meta.version.max(outcome.committed.version); + let info = pool_meta + .pools + .get_mut(source_pool_index) + .and_then(|pool| pool.decommission.as_mut()) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + info.capacity_reservation = persisted_info.capacity_reservation; + info.capacity_blocked_reason = persisted_info.capacity_blocked_reason; + } + ensure_pool_meta_write_fence(&write_guard, "decommission target capacity progress save failed")?; + outcome.disarm(); + capacity_result?; + result + } + pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> { self.pool_meta_save_gate.lock().await.ensure_write_safe(operation) } @@ -6494,6 +8817,18 @@ impl ECStore { return Err(invalid_decommission_pool_index_error(current_count, idx)); }; pool.last_update = checkpoint.checkpoint_at; + if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = ( + checkpoint.capacity_operation_id, + checkpoint.capacity_owner_nonce, + checkpoint.capacity_lease_expires_at, + pool.decommission.as_mut().and_then(|info| info.capacity_reservation.as_mut()), + ) && reservation.operation_id == operation_id + && reservation.owner_nonce == owner_nonce + && reservation.active() + { + reservation.renewed_at = checkpoint.checkpoint_at; + reservation.expires_at = expires_at; + } merge_pool_meta_updates_for_save(&mut snapshot, ¤t, &[idx], "decommission progress save failed")?; (snapshot, checkpoint) }; @@ -6559,10 +8894,9 @@ impl ECStore { Ok(true) } - async fn save_current_pool_meta_for_decommission_start( + pub(crate) async fn save_current_pool_meta_for_decommission_start( &self, indices: &[usize], - space_infos: Vec<(usize, PoolSpaceInfo)>, decom_buckets: Vec, ) -> Result { let mut save_guard = self.pool_meta_save_gate.lock().await; @@ -6613,19 +8947,36 @@ impl ECStore { ensure_decommission_start_pool_states(&latest_pool_meta, indices)?; ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?; + ensure_decommission_capacity_writer_supported(&latest_pool_meta)?; + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + activation_fence.ensure_held()?; let previous_pool_meta = latest_pool_meta.clone(); + let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?; let first_idx = indices.first().copied(); - for (idx, pi) in space_infos { + let now = OffsetDateTime::now_utc(); + for idx in indices.iter().copied() { + let capacity = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == idx) + .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?; latest_pool_meta.set_decommission_state_at( idx, - pi, + capacity.space, Some(idx) != first_idx, - OffsetDateTime::now_utc(), + now, Some(&rebalance_meta), )?; latest_pool_meta.queue_buckets(idx, decom_buckets.clone()); } + reserve_decommission_start_target_capacity( + &mut latest_pool_meta, + indices, + &capacity_infos, + uuid::Uuid::new_v4(), + capacity_generation, + now, + )?; activation_fence.ensure_held()?; let outcome = latest_pool_meta @@ -6678,11 +9029,20 @@ impl ECStore { } pub async fn status(&self, idx: usize) -> Result { - let space_info = self.get_decommission_pool_space_info(idx).await?; + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let space_info = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == idx) + .map(|capacity| capacity.space) + .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), idx))?; let pool_meta = self.pool_meta.read().await; + let active_target_reservations = active_decommission_target_reservations(&pool_meta); - let pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone(); + let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone(); + if let Some(info) = pool_info.decommission.as_mut() { + observe_decommission_capacity_reservation(info, &capacity_infos, &active_target_reservations); + } Ok(apply_decommission_status_space_info(pool_info, space_info)) } @@ -6707,35 +9067,146 @@ impl ECStore { Ok(()) } - async fn get_decommission_pool_space_info(&self, idx: usize) -> Result { + async fn get_decommission_pool_capacity_info(&self, idx: usize) -> Result { if let Some(sets) = self.pools.get(idx) { let mut info = sets.storage_info_snapshot().await; info.backend = StorageAdminApi::backend_info(self).await; let total = get_total_usable_capacity(&info.disks, &info); let free = get_total_usable_capacity_free(&info.disks, &info); - - Ok(PoolSpaceInfo { + let space = PoolSpaceInfo { free, total, - used: total - free, + used: total.saturating_sub(free), + }; + let layout = DecommissionErasureLayout { + data: info + .backend + .standard_sc_data + .get(idx) + .copied() + .unwrap_or_else(|| sets.set_drive_count.saturating_sub(sets.parity_count)), + parity: info + .backend + .standard_sc_parities + .get(idx) + .copied() + .unwrap_or(sets.parity_count), + }; + if !layout.is_valid() { + return Err(Error::DecommissionCapacity(format!( + "failed to read decommission capacity for pool {idx}: invalid erasure layout data={} parity={}", + layout.data, layout.parity + ))); + } + let (physical_total, physical_free, physical_used) = + decommission_physical_pool_capacity(&info.disks, idx, layout, space); + + Ok(DecommissionPoolCapacityInfo { + pool_index: idx, + space, + layout, + physical_free, + physical_total, + physical_used, }) } else { Err(invalid_decommission_pool_index_error(self.pools.len(), idx)) } } - async fn get_decommission_all_pool_space_infos(&self) -> Result> { + async fn get_decommission_all_pool_capacity_infos(&self) -> Result> { #[cfg(test)] - if let Some(space_infos) = take_decommission_space_info_override_for_test(self.id) { - return Ok(space_infos); + if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) { + return Ok(capacity_infos); } - let mut space_infos = Vec::with_capacity(self.pools.len()); - for idx in 0..self.pools.len() { - space_infos.push((idx, self.get_decommission_pool_space_info(idx).await?)); + #[cfg(test)] + if let Some(space_infos) = take_decommission_space_info_override_for_test(self.id) { + return Ok(space_infos + .into_iter() + .map(|(pool_index, space)| DecommissionPoolCapacityInfo::from_logical(pool_index, space)) + .collect()); } - Ok(space_infos) + + let mut capacity_infos = Vec::with_capacity(self.pools.len()); + for idx in 0..self.pools.len() { + capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?); + } + Ok(capacity_infos) + } + + async fn ensure_decommission_runtime_capacity_available(&self, idx: usize, generation: OffsetDateTime) -> Result<()> { + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; + ensure_decommission_capacity_reservations_available(&pool_meta, &capacity_infos, "migration") + } + + async fn decommission_capacity_owner_for_worker( + &self, + idx: usize, + generation: OffsetDateTime, + ) -> Result> { + let active_worker = self + .decommission_cancelers + .read() + .await + .get(idx) + .and_then(Option::as_ref) + .is_some_and(DecommissionCanceler::is_active); + if !active_worker { + return Ok(None); + } + + let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; + let reservation = pool_meta + .pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.lease_active_at(OffsetDateTime::now_utc())) + .ok_or_else(|| decommission_capacity_blocked_error(format!("source pool {idx} has no active reservation")))?; + Ok(Some(DecommissionCapacityOwner { + source_pool_index: idx, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + })) + } + + pub(crate) async fn select_decommission_capacity_target_pool( + &self, + owner: DecommissionCapacityOwner, + expected_data_bytes: usize, + ) -> Result { + let pool_meta = self.pool_meta.read().await; + let reservation = pool_meta + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc())) + .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?; + reservation + .targets + .iter() + .filter_map(|target| { + let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?; + let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies)); + let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies); + (required_peak <= remaining).then_some((target.pool_index, remaining)) + }) + .max_by_key(|(_, remaining)| *remaining) + .map(|(pool_index, _)| pool_index) + .ok_or_else(|| { + decommission_capacity_blocked_error(format!( + "source pool {} has no target allocation for {expected_data_bytes} data bytes", + owner.source_pool_index + )) + }) } pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime { @@ -7184,21 +9655,35 @@ impl ECStore { .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed") .await?; let rebalance_meta = self.rebalance_meta.read().await.clone(); + let capacity_infos = if self.pools.is_empty() { + Vec::new() + } else { + self.get_decommission_all_pool_capacity_infos().await? + }; let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } + let capacity_indices = if capacity_infos.is_empty() { + Vec::new() + } else { + recover_decommission_capacity_reservations(&mut pool_meta, &capacity_infos, OffsetDateTime::now_utc())? + }; let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref()); - let changed = reconciled || promoted; + let mut changed_indices = capacity_indices; + let changed = !changed_indices.is_empty() || reconciled || promoted; if changed { - merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &[idx], "decommission promotion failed")?; + if !changed_indices.contains(&idx) { + changed_indices.push(idx); + } + merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &changed_indices, "decommission promotion failed")?; } drop(pool_meta); let (save_outcome, save_error) = if changed { match snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices) .await { Ok(outcome) => (Some(outcome), None), @@ -7275,6 +9760,25 @@ impl ECStore { Ok(()) } + async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> { + let mut save_guard = self.pool_meta_save_gate.lock().await; + let (pool_meta_guard, mut snapshot) = self + .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed") + .await?; + snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?; + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) + .await?; + ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?; + { + let mut pool_meta = self.pool_meta.write().await; + publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]); + } + ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?; + outcome.disarm(); + Ok(()) + } + pub async fn is_decommission_running(&self) -> bool { { let cancelers = self.decommission_cancelers.read().await; @@ -7918,6 +10422,8 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; self.ensure_decommission_generation_current(idx, generation).await?; + self.ensure_decommission_runtime_capacity_available(idx, generation).await?; + let capacity_owner = self.decommission_capacity_owner_for_worker(idx, generation).await?; let operation_gate = self.ctx.data_movement_operation_gate(); let bucket_incarnation_fence = match expected_bucket_incarnation_id { @@ -7955,11 +10461,14 @@ impl ECStore { bucket.as_str(), &version.name, version, - &decommission_remote_tiered_opts( - version, - version_id.clone(), - idx, - expected_bucket_incarnation_id, + &decommission_capacity_owned_opts( + decommission_remote_tiered_opts( + version, + version_id.clone(), + idx, + expected_bucket_incarnation_id, + ), + capacity_owner, ), ) .await @@ -8103,7 +10612,15 @@ impl ECStore { self.delete_object( bucket.as_str(), &version.name, - decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), + decommission_capacity_owned_opts( + decommission_delete_marker_opts( + version, + version_id.clone(), + idx, + expected_bucket_incarnation_id, + ), + capacity_owner, + ), ) .await }) @@ -8251,11 +10768,14 @@ impl ECStore { bucket.as_str(), &version.name, version, - &decommission_remote_tiered_opts( - version, - version_id.clone(), - idx, - expected_bucket_incarnation_id, + &decommission_capacity_owned_opts( + decommission_remote_tiered_opts( + version, + version_id.clone(), + idx, + expected_bucket_incarnation_id, + ), + capacity_owner, ), ) .await @@ -8420,7 +10940,7 @@ impl ECStore { let migrate_result = self .run_guarded_decommission_side_effect(&rx, &operation_gate, || async { self.clone() - .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) + .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id, capacity_owner) .await }) .await; @@ -8802,6 +11322,25 @@ impl ECStore { expected_bucket_incarnation_id: Option, source_changed_exhaustions: Arc, ) -> Result<()> { + let needs_capacity_reservation = { + let pool_meta = self.pool_meta.read().await; + pool_meta + .pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .is_none_or(|reservation| !reservation.active()) + }; + if needs_capacity_reservation { + let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + { + let mut pool_meta = self.pool_meta.write().await; + let version = pool_meta.version; + pool_meta.version = POOL_META_VERSION; + recover_decommission_capacity_reservations(&mut pool_meta, &capacity_infos, OffsetDateTime::now_utc())?; + pool_meta.version = version; + } + } let generation = self.active_decommission_generation(idx).await?; self.decommission_entry( rx, @@ -9029,6 +11568,14 @@ impl ECStore { let generation = match self.promote_queued_decommission(idx, canceler).await { Ok(generation) => generation, Err(Error::OperationCanceled) => return Ok(()), + Err(err) if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) => { + if let Err(pause_err) = self.pause_decommission_for_capacity(idx, &err).await { + return Err(decommission_capacity_blocked_error(format!( + "failed to persist paused state for pool {idx}: {pause_err}" + ))); + } + return Ok(()); + } Err(err) => { resolve_decommission_terminal_mark_after_error_result( self.decommission_failed_for_operation(idx, canceler).await, @@ -9066,6 +11613,17 @@ impl ECStore { } let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await; + if let Err(err) = &result + && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err)) + { + if let Err(pause_err) = self.pause_decommission_for_capacity(idx, err).await { + return Err(decommission_capacity_blocked_error(format!( + "failed to persist paused state for pool {idx}: {pause_err}" + ))); + } + return Ok(()); + } + let (final_state, canceled, cmd_line) = { let pool_meta = self.pool_meta.read().await; let Some(pool) = pool_meta.pools.get(idx) else { @@ -9601,6 +12159,7 @@ impl ECStore { let pool_meta = self.pool_meta.read().await; ensure_decommission_start_pool_states(&pool_meta, &indices)?; ensure_decommission_ledger_persistence_supported(&pool_meta)?; + ensure_decommission_capacity_writer_supported(&pool_meta)?; } #[cfg(test)] @@ -9631,7 +12190,7 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate() .await?; - let all_space_infos = self.get_decommission_all_pool_space_infos().await?; + let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; // Signal cancellation before waiting for the movement writer so active // object operations can observe the signal and release read guards. self.cancel_decommission_routines(&indices).await; @@ -9644,24 +12203,17 @@ impl ECStore { // another start. let mut cancelers = self.decommission_cancelers.write().await; let pool_meta = self.pool_meta.read().await; - ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; + ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos)?; reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())? } else { let pool_meta = self.pool_meta.read().await; ensure_decommission_start_pool_states(&pool_meta, &indices)?; - ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; + ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos)?; Vec::new() }; - let mut space_infos = Vec::with_capacity(indices.len()); - for (idx, pi) in all_space_infos.iter().copied() { - if indices.contains(&idx) { - space_infos.push((idx, pi)); - } - } - let previous_pool_meta = self - .save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets) + .save_current_pool_meta_for_decommission_start(&indices, decom_buckets) .await?; self.ctx.advance_data_movement_operation_epoch(); // The local durable transition is now fenced. Release the writer @@ -11120,6 +13672,7 @@ impl ECStore { bucket: String, rd: GetObjectReader, expected_bucket_incarnation_id: Option, + capacity_owner: Option, ) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_name = rd.object_info.name.clone(); @@ -11131,6 +13684,7 @@ impl ECStore { rd, expected_bucket_incarnation_id, "decommission_object", + capacity_owner, )); let result = migration .join_next() @@ -11144,11 +13698,43 @@ impl ECStore { } } +#[cfg(test)] +async fn persist_v3_pool_meta_for_test(store: &Arc) { + let mut meta = PoolMeta::default(); + meta.load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the baseline pool metadata should be readable before V3 migration"); + meta.version = POOL_META_GENERATION_VERSION; + let mut write_state = store.pool_meta_save_gate.lock().await.clone(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")), + (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("true")), + ], + async { + meta.save_no_lock_observing(store.pools.clone(), &mut write_state) + .await + .expect("the baseline metadata should be upgraded to a durable V3 replica"); + }, + ) + .await; + + let mut loaded = PoolMeta::default(); + loaded + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the upgraded V3 pool metadata should be readable"); + assert_eq!(loaded.version, POOL_META_GENERATION_VERSION); + store.pool_meta.write().await.version = POOL_META_GENERATION_VERSION; +} + #[cfg(test)] #[allow(clippy::items_after_test_module)] mod tests { use super::*; use crate::bucket::replication::{ReplicationState, ReplicationStatusType}; + use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + use crate::storage_api_contracts::multipart::MultipartOperations as _; use serde::Serialize; #[test] @@ -11164,6 +13750,22 @@ mod tests { #[serial_test::serial] async fn decommission_v1_start_preflights_reject_before_metadata_writes() { let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + let mut v1_meta = store.pool_meta.read().await.clone(); + v1_meta.version = POOL_META_V1_VERSION; + let v1_data = pool_meta_v1_replica_test_data(&v1_meta); + for pool in &store.pools { + save_config_with_opts( + pool.clone(), + POOL_META_NAME, + v1_data.clone(), + &ObjectOptions { + max_parity: true, + ..Default::default() + }, + ) + .await + .expect("the V1 baseline should be persisted to every pool"); + } let baseline = load_pool_meta_replicas(store.pools.clone(), true) .await .expect("baseline pool metadata should be readable"); @@ -11204,18 +13806,7 @@ mod tests { drop(start_probe); let err = store - .save_current_pool_meta_for_decommission_start( - &[0], - vec![( - 0, - PoolSpaceInfo { - free: 50, - total: 100, - used: 50, - }, - )], - Vec::new(), - ) + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) .await .expect_err("the authoritative V1 start preflight must reject before saving"); assert!(matches!(err, Error::InvalidArgument(..))); @@ -11252,26 +13843,26 @@ mod tests { async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() { let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100), + ]], + ); let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]); let start_store = Arc::clone(&store); - let start_task = tokio::spawn(async move { + let mut start_task = tokio::spawn(async move { start_store - .save_current_pool_meta_for_decommission_start( - &[0], - vec![( - 0, - PoolSpaceInfo { - free: 50, - total: 100, - used: 50, - }, - )], - Vec::new(), - ) + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) .await }); - barrier.wait_until_paused().await; + tokio::select! { + result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"), + () = barrier.wait_until_paused() => {} + } barrier.release_after_fence_loss(); let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task) .await @@ -11307,26 +13898,26 @@ mod tests { async fn decommission_activation_adopts_canonical_commit_after_replica_failure() { let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100), + ]], + ); let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]); let start_store = Arc::clone(&store); - let start_task = tokio::spawn(async move { + let mut start_task = tokio::spawn(async move { start_store - .save_current_pool_meta_for_decommission_start( - &[0], - vec![( - 0, - PoolSpaceInfo { - free: 50, - total: 100, - used: 50, - }, - )], - Vec::new(), - ) + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) .await }); - barrier.wait_until_paused().await; + tokio::select! { + result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"), + () = barrier.wait_until_paused() => {} + } let mut replica_disks = Vec::new(); for set in &store.pools[1].disk_set { let mut disks = set.disks.write().await; @@ -11392,6 +13983,521 @@ mod tests { assert!(admitted_cancel.is_cancelled()); } + #[tokio::test] + #[serial_test::serial(pool_meta_version_env)] + async fn decommission_capacity_reservation_serializes_two_nodes_without_overselling() { + let (_temp_dirs, first_node, second_node) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + persist_v3_pool_meta_for_test(&first_node).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0), + ]; + set_decommission_capacity_info_overrides_for_test(first_node.id, vec![capacity_snapshot.clone()]); + set_decommission_capacity_info_overrides_for_test(second_node.id, vec![capacity_snapshot]); + + let barrier = + PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, POOL_META_NAME, PutObjectCommitPause::BeforeQuotaRename); + let first_store = Arc::clone(&first_node); + let first = tokio::spawn(async move { + first_store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + }); + barrier.wait_until_paused().await; + + let second_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission); + let second_store = Arc::clone(&second_node); + let second = tokio::spawn(async move { + second_store + .save_current_pool_meta_for_decommission_start(&[1], Vec::new()) + .await + }); + second_probe.wait_until_attempted().await; + assert!( + !second.is_finished(), + "the competing node must wait behind the distributed activation fence" + ); + + drop(barrier); + first + .await + .expect("first-node activation should not panic") + .expect("first-node reservation should fit"); + let mut started_v3 = PoolMeta::default(); + started_v3 + .load_no_lock_from_replicas(first_node.pools.clone()) + .await + .expect("the started capacity reservation should reload from V3 replicas"); + assert_eq!(started_v3.version, POOL_META_GENERATION_VERSION); + let err = second + .await + .expect("second-node activation should not panic") + .expect_err("the second reservation must observe and reject the committed first reservation"); + assert!(err.to_string().contains("requires 60 bytes, but 40 bytes are available")); + + let mut persisted = PoolMeta::default(); + persisted + .load_no_lock_from_replicas(first_node.pools.clone()) + .await + .expect("the winning capacity reservation should remain readable"); + let reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the winning operation must retain its durable reservation"); + assert_eq!(reservation.peak_physical_bytes, 60); + assert!(persisted.pools[1].decommission.is_none()); + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let write_store = Arc::clone(&store); + let ordinary_write = tokio::spawn(async move { + write_store + .run_decommission_capacity_admitted_mutation(1, None, None, || async move { + entered_tx.send(()).expect("ordinary write admission should be observed"); + release_rx.await.expect("ordinary write should be released"); + Ok(()) + }) + .await + }); + entered_rx + .await + .expect("ordinary write should hold the shared capacity fence"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1), + ]], + ); + let activation_store = Arc::clone(&store); + let activation = tokio::spawn(async move { + activation_store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + }); + tokio::task::yield_now().await; + assert!( + !activation.is_finished(), + "activation must wait until the already-admitted ordinary write leaves the distributed capacity boundary" + ); + + release_tx.send(()).expect("ordinary write should be released"); + ordinary_write + .await + .expect("ordinary write task should join") + .expect("the pre-activation ordinary write should complete"); + let err = activation + .await + .expect("activation task should join") + .expect_err("activation must recheck and reject capacity consumed by the ordinary write"); + assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available")); + } + + #[tokio::test] + #[serial_test::serial] + async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = format!("multipart-capacity-routing-{}", uuid::Uuid::new_v4()); + let object = "later-pool.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create multipart routing bucket"); + let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation"); + let bucket_guard = store + .acquire_bucket_lifecycle_read_lock(&bucket) + .await + .expect("acquire multipart routing bucket lifecycle guard"); + let mut upload_opts = ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard); + let upload = store.pools[1] + .new_multipart_upload(&bucket, object, &upload_opts) + .await + .expect("seed an upload in the later pool"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 20, 20, 0), + DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, 10, 10), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[2], Vec::new()) + .await + .expect("reserve the first target pool"); + { + let pool_meta = store.pool_meta.read().await; + let targets = &pool_meta.pools[2] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("source reservation should exist") + .targets; + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].pool_index, 0, "the first probed pool must be reserved"); + } + + let mut data = crate::object_api::PutObjReader::from_vec(b"multipart body".to_vec()); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default()) + .await + .expect("put-part must locate the later upload before capacity admission"); + store + .clone() + .complete_multipart_upload( + &bucket, + object, + &upload.upload_id, + vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("complete must locate the later upload before capacity admission"); + + store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("the later-pool multipart upload should commit in place"); + let first_pool_result = store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await; + assert!( + first_pool_result + .as_ref() + .err() + .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)), + "the reserved first pool must not receive the multipart mutation" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + let bucket = format!("stale-capacity-owner-{}", uuid::Uuid::new_v4()); + let multipart_object = "stale-multipart.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create stale owner bucket"); + let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation"); + let bucket_guard = store + .acquire_bucket_lifecycle_read_lock(&bucket) + .await + .expect("acquire stale owner bucket lifecycle guard"); + let mut upload_opts = ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard); + let upload = store.pools[1] + .new_multipart_upload(&bucket, multipart_object, &upload_opts) + .await + .expect("seed multipart upload before reserving the target"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate target reservation"); + + let stale_owner = { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("active reservation should exist"); + DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + } + }; + { + let mut pool_meta = store.pool_meta.write().await; + let reservation = pool_meta.pools[0] + .decommission + .as_mut() + .and_then(|info| info.capacity_reservation.as_mut()) + .expect("active reservation should remain mutable"); + reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1); + } + store + .save_current_pool_meta(&[0]) + .await + .expect("persist the expired lease before recovery"); + store + .pause_decommission_for_capacity(0, &decommission_capacity_blocked_error("test lease recovery")) + .await + .expect("pause recovery should renew the lease with a new owner nonce"); + + let (current_owner, before_progress) = { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("renewed reservation should exist"); + ( + DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + }, + ( + reservation.consumed_target_physical_bytes, + reservation.inflight_target_physical_bytes, + reservation.pending_target_physical_bytes, + reservation.observed_target_physical_bytes, + ), + ) + }; + assert_ne!(stale_owner.owner_nonce, current_owner.owner_nonce); + { + let pool_meta = store.pool_meta.read().await; + ensure_decommission_target_owner_admission(&pool_meta, current_owner, 1, 1, OffsetDateTime::now_utc()) + .expect("the renewed owner should retain its reservation"); + assert!( + ensure_decommission_target_owner_admission(&pool_meta, stale_owner, 1, 1, OffsetDateTime::now_utc(),).is_err(), + "the expired owner token must be rejected after nonce rotation" + ); + } + + let stale_put_object = "stale-put.bin"; + let mut stale_put_opts = ObjectOptions { + data_movement: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..ObjectOptions::with_capacity_expected_data_bytes(Some(9)) + }; + stale_owner.apply_to(&mut stale_put_opts); + let mut stale_put_data = crate::object_api::PutObjReader::from_vec(b"stale put".to_vec()); + let put_err = store + .put_object_for_data_movement(&bucket, stale_put_object, &mut stale_put_data, &stale_put_opts, None) + .await + .expect_err("stale owner PUT must fail before selecting a target"); + assert!(is_decommission_capacity_blocked_error(&put_err)); + + let mut stale_part_opts = ObjectOptions { + data_movement: true, + part_number: Some(1), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + stale_owner.apply_to(&mut stale_part_opts); + let mut stale_part_data = crate::object_api::PutObjReader::from_vec(b"stale part".to_vec()); + let part_err = store + .put_object_part_for_data_movement( + 1, + &bucket, + multipart_object, + &upload.upload_id, + &mut stale_part_data, + &stale_part_opts, + ) + .await + .expect_err("stale owner multipart PUT must fail before mutation"); + assert!(is_decommission_capacity_blocked_error(&part_err)); + + let after_progress = { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("renewed reservation should remain active"); + ( + reservation.consumed_target_physical_bytes, + reservation.inflight_target_physical_bytes, + reservation.pending_target_physical_bytes, + reservation.observed_target_physical_bytes, + ) + }; + assert_eq!(after_progress, before_progress, "stale mutations must not consume the capacity ledger"); + let parts = store.pools[1] + .list_object_parts( + &bucket, + multipart_object, + &upload.upload_id, + None, + 1_000, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("the seeded upload should remain readable"); + assert!(parts.parts.is_empty(), "the stale multipart write must not stage a part"); + assert!( + store.pools[1] + .get_object_info(&bucket, stale_put_object, &ObjectOptions::default()) + .await + .is_err(), + "the stale PUT must not create a target object" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30); + let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)]; + { + let pool_meta = store.pool_meta.read().await; + ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight) + .expect("the pre-lock capacity snapshot should fit exactly"); + } + + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]], + ); + let err = store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect_err("the capacity snapshot inside the activation lock must be authoritative"); + assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available")); + + let local = store.pool_meta.read().await; + assert!(local.pools[0].decommission.is_none()); + drop(local); + let mut persisted = PoolMeta::default(); + persisted + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the rejected activation must leave baseline metadata readable"); + assert!(persisted.pools[0].decommission.is_none()); + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + let bucket = format!("capacity-blocked-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("the production-path bucket should be created"); + let mut source = crate::object_api::PutObjReader::from_vec(b"source remains authoritative".to_vec()); + store.pools[0] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("the source object should be written before decommission starts"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let enough = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]); + store + .save_current_pool_meta_for_decommission_start( + &[0], + vec![DecomBucketInfo { + name: bucket.clone(), + prefix: String::new(), + }], + ) + .await + .expect("the initial reservation should be activated"); + + let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]); + let canceler = DecommissionCanceler::new(CancellationToken::new()); + store.decommission_cancelers.write().await[0] = Some(canceler.clone()); + store + .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1))) + .await + .expect("runtime capacity shortage should pause the worker, not fail it"); + + store.pools[0] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("capacity pause must not delete the source object"); + let target_result = store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await; + assert!( + target_result + .as_ref() + .err() + .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)), + "capacity pause must happen before a target mutation" + ); + + let local = store.pool_meta.read().await; + let info = local.pools[0] + .decommission + .as_ref() + .expect("the blocked decommission state should remain present"); + assert!(!info.complete && !info.failed && !info.canceled); + assert_eq!(info.items_decommission_failed, 0); + assert_eq!(info.bytes_failed, 0); + assert!(info.capacity_blocked_reason.is_some()); + assert!( + info.capacity_reservation + .as_ref() + .is_some_and(DecommissionCapacityReservation::active), + "blocked work must retain and renew its reservation" + ); + drop(local); + + let mut persisted = PoolMeta::default(); + persisted + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the blocked state should be durable"); + let persisted_info = persisted.pools[0] + .decommission + .as_ref() + .expect("the durable blocked state should remain nonterminal"); + assert!(!persisted_info.complete && !persisted_info.failed && !persisted_info.canceled); + assert!(persisted_info.capacity_blocked_reason.is_some()); + assert!( + persisted_info + .capacity_reservation + .as_ref() + .is_some_and(DecommissionCapacityReservation::active) + ); + } + fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta { PoolMeta { version: POOL_META_VERSION, @@ -11914,6 +15020,33 @@ mod tests { assert!(err.to_string().contains("pool metadata V2 is required")); } + #[test] + fn pool_meta_v1_writer_rejects_capacity_reservations() { + let mut meta = pool_meta_replica_test_meta("pool-0"); + meta.version = POOL_META_V1_VERSION; + let reservation = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 10, 10), + DecommissionErasureLayout { data: 1, parity: 0 }, + uuid::Uuid::new_v4(), + 1, + OffsetDateTime::UNIX_EPOCH, + ) + .expect("test reservation should be valid"); + meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + capacity_reservation: Some(reservation), + ..Default::default() + }); + + let err = meta + .encode_config_data_for_v2_gate(false) + .expect_err("v1 must not drop a distributed capacity reservation"); + assert!( + err.to_string() + .contains("pool metadata V2 is required to persist decommission capacity reservations") + ); + } + #[test] fn pool_meta_replica_selection_preserves_observed_v2_floor() { let selection = select_pool_meta_replica(vec![ @@ -13034,6 +16167,72 @@ fn is_disk_online_state(state: &str) -> bool { true } +fn decommission_physical_pool_capacity( + disks: &[rustfs_madmin::Disk], + pool_index: usize, + layout: DecommissionErasureLayout, + logical: PoolSpaceInfo, +) -> (usize, usize, usize) { + let width = layout.width(); + let mut sets: HashMap> = HashMap::new(); + let mut seen = HashSet::new(); + for disk in disks { + let state = disk.state.trim().to_ascii_lowercase(); + if disk.pool_index != pool_index as i32 + || disk.set_index < 0 + || disk.disk_index < 0 + || disk.disk_index as usize >= width + || !matches!(state.as_str(), "ok" | "online") + { + continue; + } + let identity = (disk.set_index, disk.disk_index); + if seen.insert(identity) { + sets.entry(disk.set_index).or_default().push(disk); + } + } + + let mut physical_total = 0usize; + let mut physical_free = 0usize; + let mut physical_used = 0usize; + let mut observed_set = false; + for set_disks in sets.values() { + if set_disks.is_empty() { + continue; + } + observed_set = true; + let min_total = set_disks + .iter() + .map(|disk| disk.total_space as usize) + .min() + .unwrap_or_default(); + let min_free = set_disks + .iter() + .map(|disk| disk.available_space as usize) + .min() + .unwrap_or_default(); + let max_used = set_disks + .iter() + .map(|disk| (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize))) + .max() + .unwrap_or_default(); + physical_total = physical_total.saturating_add(min_total.saturating_mul(width)); + physical_used = physical_used.saturating_add(max_used.saturating_mul(width)); + if set_disks.len() >= width { + physical_free = physical_free.saturating_add(min_free.saturating_mul(width)); + } + } + + if observed_set { + return (physical_total, physical_free, physical_used); + } + + let fallback_total = capacity_mul_div_ceil(logical.total, width, layout.data); + let fallback_used = capacity_mul_div_ceil(logical.used, width, layout.data) + .max(fallback_total.saturating_sub(capacity_mul_div_ceil(logical.free, width, layout.data))); + (fallback_total, 0, fallback_used) +} + #[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")] #[allow( dead_code, @@ -13202,39 +16401,45 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi #[cfg(test)] mod pools_tests { use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; + use super::persist_v3_pool_meta_for_test; use super::record_decommission_entry_error; use super::resolve_decommission_listing_error; use super::resolve_decommission_partial_listing_entry; use super::{ - DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, - DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES, - DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, - DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionDurableIlmReceipt, - DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, DecommissionUnresolvedEntry, - ListCallback, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION, POOL_META_VERSION, PoolDecommissionInfo, - PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, - REBAL_META_NAME, acquire_pool_rebalance_activation_locks, apply_decommission_status_space_info, + DECOMMISSION_CAPACITY_RELEASE_CANCELED, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, + DECOMMISSION_CAPACITY_RESERVATION_TTL, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, + DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, + DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget, + DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionErasureLayout, DecommissionPoolCapacityInfo, + DecommissionStartPoolState, DecommissionTargetConsumption, DecommissionTerminalState, DecommissionUnresolvedEntry, + ListCallback, POOL_META_GENERATION_VERSION, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION, + POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus, + QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, - cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_durable_ilm_receipt_path, - decommission_durable_ilm_receipt_run_prefix, decommission_durable_ilm_receipt_run_token, - decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, - decommission_retry_backoff_delay, decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices, + build_decommission_capacity_reservation, cancel_decommission_canceler, clamp_decommission_entry_concurrency, + classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, + decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, + decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, + decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_delay, + decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices, default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue, - enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, - ensure_decommission_generation, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, - ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, - ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed, - ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, - ensure_decommission_unresolved_verification_disk_count, ensure_local_decommission_pool_leaders, - ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index, guard_decommission_cancelers, - has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, + enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available, + ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available, + ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, + ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, + ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, + ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count, + ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index, + guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh, - missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, - publish_pool_meta_updates, read_pool_meta_replica, reconcile_decommission_meta_buckets, - reconcile_decommission_unresolved_entries_for_completion, record_decommission_unresolved_entry, - require_decommission_store, reserve_decommission_start_cancelers, resolve_decommission_bucket_state, + missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result, + pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica, + reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion, + record_decommission_unresolved_entry, recover_decommission_capacity_reservations, + renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers, + reserve_decommission_start_target_capacity, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, @@ -13253,6 +16458,12 @@ mod pools_tests { update_decommission_for_operation, validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain, with_decommission_entry_context, }; + use super::{ + DecommissionCapacityOwner, DecommissionCapacityReservation, decommission_capacity_mutation_id, + ensure_decommission_target_owner_admission, ensure_external_decommission_target_admission, + is_decommission_capacity_blocked_error, record_decommission_target_consumption, reserve_decommission_target_pending, + resolve_decommission_target_pending, set_decommission_capacity_info_overrides_for_test, + }; use crate::bucket::lifecycle::{ DurableIlmRecordCheckpoint, bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions}, @@ -13264,6 +16475,7 @@ mod pools_tests { use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint}; use crate::error::{Error, StorageError}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::object_api::ObjectOptions; use crate::runtime::instance::InstanceContext; use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; @@ -17045,66 +20257,41 @@ mod pools_tests { #[test] fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() { let meta = PoolMeta { + version: POOL_META_VERSION, pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], ..Default::default() }; - let space_infos = vec![ - ( - 0, - PoolSpaceInfo { - free: 100, - total: 1_000, - used: 900, - }, - ), - ( - 1, - PoolSpaceInfo { - free: 1_170, - total: 2_000, - used: 830, - }, - ), + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600), + DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400), ]; - assert!(ensure_decommission_start_target_capacity(&meta, &[0], &space_infos).is_ok()); + assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos).is_ok()); } #[test] fn test_ensure_decommission_start_target_capacity_rejects_insufficient_free_space() { let meta = PoolMeta { + version: POOL_META_VERSION, pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], ..Default::default() }; - let space_infos = vec![ - ( - 0, - PoolSpaceInfo { - free: 100, - total: 1_000, - used: 900, - }, - ), - ( - 1, - PoolSpaceInfo { - free: 1_169, - total: 2_000, - used: 831, - }, - ), + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600), + DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401), ]; - let err = ensure_decommission_start_target_capacity(&meta, &[0], &space_infos) - .expect_err("target free capacity below 130% of source used should be rejected"); + let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos) + .expect_err("target physical free capacity below the modeled peak should be rejected"); - assert!(err.to_string().contains("insufficient target pool capacity")); - assert!(err.to_string().contains("required 1170 bytes available 1169 bytes")); + assert!(err.to_string().contains("insufficient reserved physical target capacity")); + assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available")); } #[test] fn test_ensure_decommission_start_target_capacity_ignores_non_active_target_pool() { let meta = PoolMeta { + version: POOL_META_VERSION, pools: vec![ decommission_test_pool_status(0, None), decommission_test_pool_status( @@ -17118,37 +20305,514 @@ mod pools_tests { ], ..Default::default() }; - let space_infos = vec![ - ( - 0, - PoolSpaceInfo { - free: 100, - total: 1_000, - used: 900, - }, - ), - ( - 1, - PoolSpaceInfo { - free: 10_000, - total: 10_000, - used: 0, - }, - ), - ( - 2, - PoolSpaceInfo { - free: 1_169, - total: 2_000, - used: 831, - }, - ), + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600), + DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 10_000, 10_000, 0), + DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401), ]; - let err = ensure_decommission_start_target_capacity(&meta, &[0], &space_infos) + let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos) .expect_err("completed pools must not contribute target free capacity"); - assert!(err.to_string().contains("required 1170 bytes available 1169 bytes")); + assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available")); + } + + #[test] + fn decommission_capacity_model_converts_different_source_and_target_parity() { + let now = OffsetDateTime::UNIX_EPOCH; + let high_target_parity = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 0, 600, 600), + DecommissionErasureLayout { data: 4, parity: 4 }, + uuid::Uuid::new_v4(), + 1, + now, + ) + .expect("the source and target layouts should be valid"); + assert_eq!(high_target_parity.source_data_equivalent_bytes, 400); + assert_eq!(high_target_parity.predicted_physical_bytes, 800); + assert_eq!(high_target_parity.temporary_physical_bytes, 800); + assert_eq!(high_target_parity.peak_physical_bytes, 1_600); + + let low_target_parity = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 4 }, 0, 800, 800), + DecommissionErasureLayout { data: 4, parity: 2 }, + uuid::Uuid::new_v4(), + 2, + now, + ) + .expect("the inverse source and target layouts should be valid"); + assert_eq!(low_target_parity.source_data_equivalent_bytes, 400); + assert_eq!(low_target_parity.predicted_physical_bytes, 600); + assert_eq!(low_target_parity.peak_physical_bytes, 1_200); + } + + #[test] + fn decommission_batch_persists_one_replayable_operation_identity_and_generation() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1); + let operation_id = uuid::Uuid::new_v4(); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 20, 20), + DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![ + decommission_test_pool_status(0, None), + decommission_test_pool_status(1, None), + decommission_test_pool_status(2, None), + ], + ..Default::default() + }; + meta.decommission(0, capacity_infos[0].space).unwrap(); + meta.queue_decommission(1, capacity_infos[1].space).unwrap(); + + reserve_decommission_start_target_capacity(&mut meta, &[0, 1], &capacity_infos, operation_id, 17, now) + .expect("the batch reservation should fit exactly"); + for idx in [0, 1] { + let reservation = meta.pools[idx] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("each source pool should carry the batch reservation identity"); + assert_eq!(reservation.operation_id, operation_id); + assert_eq!(reservation.generation, 17); + } + } + + #[test] + fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() { + let reservation = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 1_024, 1_024), + DecommissionErasureLayout { data: 2, parity: 2 }, + uuid::Uuid::new_v4(), + 1, + OffsetDateTime::UNIX_EPOCH, + ) + .expect("physical source usage should produce a reservation"); + + assert_eq!(reservation.source_physical_bytes, 1_024); + assert_eq!(reservation.predicted_physical_bytes, 1_024); + assert_eq!(reservation.temporary_copies, 1); + assert_eq!(reservation.peak_physical_bytes, 2_048); + } + + #[test] + fn decommission_runtime_capacity_tracks_own_target_consumption_but_rejects_external_drop() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let initial = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + meta.decommission(0, initial[0].space).unwrap(); + reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) + .expect("the initial target capacity should fit exactly"); + + let mutation_id = uuid::Uuid::from_u128(1); + reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1)) + .expect("the target intent should be durable before its write"); + let own_consumption = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 50, 60, 10)]; + ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "restart") + .expect("a persisted target intent must recognize its own write after restart"); + resolve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id) + .expect("the persisted target intent should resolve at commit"); + record_decommission_target_consumption( + &mut meta, + 0, + 1, + DecommissionTargetConsumption { + committed_data_bytes: 10, + target_physical_bytes: 10, + observed_physical_bytes: 10, + }, + mutation_id, + now + Duration::seconds(1), + ) + .expect("the operation's own target consumption should be persisted"); + ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "migration") + .expect("the operation's own committed target bytes must not look like external capacity loss"); + + let dropped = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 39, 60, 21)]; + let err = ensure_decommission_capacity_reservations_available(&meta, &dropped, "migration") + .expect_err("runtime migration must stop after a sudden competing write consumes reserved capacity"); + assert!( + err.to_string() + .contains("requires 40 physical bytes, but only 39 bytes remain") + ); + assert!(is_decommission_capacity_blocked_error(&err)); + } + + #[test] + fn decommission_target_capacity_reuses_only_matching_pending_intent() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let initial = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + meta.decommission(0, initial[0].space).unwrap(); + reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) + .expect("the initial target capacity should fit exactly"); + + let first_mutation_id = uuid::Uuid::from_u128(1); + let second_mutation_id = uuid::Uuid::from_u128(2); + reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(1)) + .expect("the first mutation should persist its target intent"); + assert_eq!( + reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(2)) + .expect("the same mutation should reuse its persisted target intent"), + 0 + ); + let err = reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(2)) + .expect_err("a second mutation must not reuse the first mutation's pending intent"); + assert!(is_decommission_capacity_blocked_error(&err)); + assert!(err.to_string().contains("unresolved target capacity intent")); + + resolve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id) + .expect("the first mutation should finalize its intent"); + record_decommission_target_consumption( + &mut meta, + 0, + 1, + DecommissionTargetConsumption { + committed_data_bytes: 10, + target_physical_bytes: 10, + observed_physical_bytes: 10, + }, + first_mutation_id, + now + Duration::seconds(2), + ) + .expect("the first mutation's consumption should be durable"); + reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(3)) + .expect("the second mutation should retry only after the first intent is finalized"); + assert_eq!( + meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the reservation should remain present") + .pending_target_physical_bytes, + 10 + ); + } + + #[test] + fn decommission_target_capacity_allows_same_mutation_to_grow_its_pending_stage() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let initial = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + meta.decommission(0, initial[0].space).unwrap(); + reserve_decommission_start_target_capacity(&mut meta, &[0], &initial, uuid::Uuid::new_v4(), 1, now) + .expect("the initial target capacity should fit exactly"); + + let mutation_id = uuid::Uuid::from_u128(1); + assert_eq!( + reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(1)) + .expect("the initial multipart stage should persist its one-byte intent"), + 1 + ); + assert_eq!( + reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(2)) + .expect("the same mutation should grow its intent for the commit stage"), + 9 + ); + assert_eq!( + meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the reservation should remain present") + .targets[0] + .pending_physical_bytes, + 10 + ); + assert_eq!( + reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(3)) + .expect("a retry of an earlier stage must reuse the larger same-mutation intent"), + 0 + ); + } + + #[test] + fn decommission_capacity_mutation_identity_survives_lease_nonce_rotation() { + let owner = DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: uuid::Uuid::from_u128(1), + generation: 2, + owner_nonce: uuid::Uuid::from_u128(3), + mutation_id: None, + }; + let recovered_owner = DecommissionCapacityOwner { + owner_nonce: uuid::Uuid::from_u128(4), + ..owner + }; + let first = decommission_capacity_mutation_id(owner, "bucket", "object", Some("version"), false, None); + let recovered = decommission_capacity_mutation_id(recovered_owner, "bucket", "object", Some("version"), false, None); + assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity"); + } + + #[test] + fn ordinary_write_admission_cannot_race_into_a_reserved_target() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + meta.decommission(0, capacity_infos[0].space).unwrap(); + reserve_decommission_start_target_capacity(&mut meta, &[0], &capacity_infos, uuid::Uuid::new_v4(), 1, now) + .expect("the decommission reservation should fit"); + + assert!( + matches!( + ensure_external_decommission_target_admission(&meta, 1, "ordinary_put"), + Err(Error::SlowDown) + ), + "an ordinary write must not consume a target reservation" + ); + let rebalance_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }; + assert!( + DecommissionCapacityOwner::from_options(&rebalance_opts).is_none(), + "rebalance data movement must remain an external capacity consumer" + ); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the active reservation should remain available"); + let expected_owner = DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + }; + ensure_decommission_target_owner_admission(&meta, expected_owner, 1, 10, now) + .expect("the reservation owner should consume its own allocation under the shared boundary"); + let mut decommission_opts = rebalance_opts; + expected_owner.apply_to(&mut decommission_opts); + assert_eq!(DecommissionCapacityOwner::from_options(&decommission_opts), Some(expected_owner)); + } + + #[test] + fn decommission_physical_capacity_uses_per_set_bottleneck_and_widest_usage() { + let disks = [10_u64, 20, 30, 40] + .into_iter() + .enumerate() + .map(|(disk_index, available_space)| rustfs_madmin::Disk { + endpoint: format!("http://node-{disk_index}"), + drive_path: format!("/disk-{disk_index}"), + state: "ok".to_string(), + total_space: 100, + used_space: 100 - available_space, + available_space, + pool_index: 0, + set_index: 0, + disk_index: disk_index as i32, + ..Default::default() + }) + .collect::>(); + let capacity = decommission_physical_pool_capacity( + &disks, + 0, + DecommissionErasureLayout { data: 2, parity: 2 }, + PoolSpaceInfo { + free: 0, + total: 0, + used: 0, + }, + ); + + assert_eq!(capacity, (400, 40, 360)); + } + + #[test] + fn decommission_terminal_transitions_release_capacity_reservations() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1); + let reservation = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 50, 50), + DecommissionErasureLayout { data: 1, parity: 0 }, + uuid::Uuid::new_v4(), + 7, + now, + ) + .expect("test reservation should be valid"); + let active = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(now), + capacity_reservation: Some(reservation), + ..Default::default() + }), + )], + ..Default::default() + }; + + for (reason, transition) in [ + ( + DECOMMISSION_CAPACITY_RELEASE_CANCELED, + PoolMeta::decommission_cancel as fn(&mut PoolMeta, usize) -> bool, + ), + (DECOMMISSION_CAPACITY_RELEASE_FAILED, PoolMeta::decommission_failed), + (DECOMMISSION_CAPACITY_RELEASE_COMPLETED, PoolMeta::decommission_complete), + ] { + let mut meta = active.clone(); + assert!(transition(&mut meta, 0)); + let released = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("terminal metadata should retain reservation observability"); + assert!(released.released_at.is_some()); + assert_eq!(released.release_reason.as_deref(), Some(reason)); + assert_eq!( + released.operation_id, + active.pools[0] + .decommission + .as_ref() + .unwrap() + .capacity_reservation + .as_ref() + .unwrap() + .operation_id + ); + } + } + + #[test] + fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() { + let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1); + let recovered_at = created_at + DECOMMISSION_CAPACITY_RESERVATION_TTL + Duration::seconds(1); + let mut reservation = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 100, 100), + DecommissionErasureLayout { data: 2, parity: 2 }, + uuid::Uuid::new_v4(), + 41, + created_at, + ) + .expect("test reservation should be valid"); + reservation.targets.push(DecommissionCapacityTarget { + pool_index: 1, + layout: DecommissionErasureLayout { data: 2, parity: 2 }, + physical_total_at_reservation: 200, + physical_free_at_reservation: 200, + reserved_physical_bytes: 200, + consumed_physical_bytes: 0, + observed_physical_bytes: 0, + inflight_physical_bytes: 0, + pending_physical_bytes: 0, + pending_mutation_id: None, + temporary_mutations: Vec::new(), + }); + let operation_id = reservation.operation_id; + let owner_nonce = reservation.owner_nonce; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(created_at), + capacity_reservation: Some(reservation), + ..Default::default() + }), + )], + ..Default::default() + }; + + let encoded = meta + .encode_config_data_for_test() + .expect("capacity reservation should persist in V2"); + let mut restored = PoolMeta::default(); + restored + .load_from_config_data(encoded) + .expect("capacity reservation should survive a restart round trip"); + let info = restored.pools[0] + .decommission + .as_mut() + .expect("active decommission should restore"); + let recovered = info.capacity_reservation.as_mut().expect("reservation should remain present"); + assert!(renew_decommission_capacity_reservation(recovered, recovered_at, true)); + assert_eq!(recovered.operation_id, operation_id); + assert_eq!(recovered.generation, 41); + assert_ne!(recovered.owner_nonce, owner_nonce); + assert_eq!(recovered.recovered_at, Some(recovered_at)); + assert_eq!(recovered.expires_at, recovered_at + DECOMMISSION_CAPACITY_RESERVATION_TTL); + assert_eq!(next_decommission_capacity_generation(&restored).unwrap(), 42); + + meta.decommission_failed(0); + let terminal = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("failed metadata should retain the released reservation"); + assert!(!terminal.active()); + } + + #[test] + fn decommission_restart_reconstructs_a_missing_capacity_reservation_fail_closed() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(2); + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![ + decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + start_time: Some(now - Duration::minutes(1)), + ..Default::default() + }), + ), + decommission_test_pool_status(1, None), + ], + ..Default::default() + }; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]; + + let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now) + .expect("restart recovery should rebuild the missing reservation while capacity still fits"); + assert_eq!(recovered_indices, vec![0]); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("restart recovery should persist a replacement reservation"); + assert_eq!(reservation.peak_physical_bytes, 60); + assert_eq!(reservation.recovered_at, Some(now)); + assert_eq!(reservation.generation, 1); } #[test] @@ -17756,7 +21420,7 @@ mod pools_tests { } #[test] - fn test_pool_meta_queued_decommission_is_not_suspended_until_promoted() { + fn test_pool_meta_queued_decommission_is_suspended_to_preserve_reserved_capacity() { let mut meta = PoolMeta { pools: vec![PoolStatus { id: 0, @@ -17777,7 +21441,7 @@ mod pools_tests { ) .expect("queued decommission should be stored"); - assert!(!meta.is_suspended(0)); + assert!(meta.is_suspended(0)); assert!(meta.promote_queued_decommission(0)); assert!(meta.is_suspended(0)); } @@ -18603,6 +22267,98 @@ mod pools_tests { assert!(info.start_time.is_none()); } + #[tokio::test] + #[serial_test::serial(pool_meta_version_env)] + async fn test_decommission_promotion_persists_all_recovered_capacity_sources() { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + persist_v3_pool_meta_for_test(&store).await; + let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(3); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let mut queued_reservation = build_decommission_capacity_reservation( + DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10), + layout, + uuid::Uuid::new_v4(), + 7, + now, + ) + .expect("queued source reservation should be valid"); + queued_reservation.targets.push(DecommissionCapacityTarget { + pool_index: 2, + layout, + physical_total_at_reservation: 200, + physical_free_at_reservation: 100, + reserved_physical_bytes: queued_reservation.peak_physical_bytes, + consumed_physical_bytes: 0, + observed_physical_bytes: 0, + inflight_physical_bytes: 0, + pending_physical_bytes: 0, + pending_mutation_id: None, + temporary_mutations: Vec::new(), + }); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(now), + ..Default::default() + }); + pool_meta.pools[1].decommission = Some(PoolDecommissionInfo { + queued: true, + start_time: Some(now), + capacity_reservation: Some(queued_reservation), + ..Default::default() + }); + } + let mut loaded_v3 = PoolMeta::default(); + loaded_v3 + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the startup fixture must provide a durable V3 pool metadata replica"); + assert_eq!(loaded_v3.version, POOL_META_GENERATION_VERSION); + store + .save_current_pool_meta_for_test(&[0, 1, 2]) + .await + .expect("active and queued source metadata should be persisted before promotion"); + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 90, 100, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10), + DecommissionPoolCapacityInfo::for_test(2, layout, 100, 200, 100), + ]], + ); + + store + .promote_queued_decommission_for_test(1) + .await + .expect("queued promotion should recover all active source reservations"); + + let mut persisted = PoolMeta::default(); + persisted + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("promoted active and recovered source metadata should reload"); + assert_eq!(persisted.version, POOL_META_GENERATION_VERSION); + assert!( + persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .is_some_and(DecommissionCapacityReservation::active) + ); + let promoted = persisted.pools[1] + .decommission + .as_ref() + .expect("queued source should remain after promotion"); + assert!(!promoted.queued, "queued source should be promoted durably"); + assert!( + promoted + .capacity_reservation + .as_ref() + .is_some_and(DecommissionCapacityReservation::active) + ); + } + #[tokio::test] async fn test_decommission_bucket_done_rechecks_sticky_gate_before_mutation() { let bucket = DecomBucketInfo { diff --git a/crates/ecstore/src/core/pools_test.rs b/crates/ecstore/src/core/pools_test.rs index dd7fa752d..162077c00 100644 --- a/crates/ecstore/src/core/pools_test.rs +++ b/crates/ecstore/src/core/pools_test.rs @@ -190,3 +190,4531 @@ mod capacity_dedup_tests { assert_eq!(free, 500_000_000_000); } } + +#[cfg(all(test, feature = "test-util"))] +mod decommission_lock_order_tests { + use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING; + use crate::core::pools::{ + DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, DecommissionErasureLayout, DecommissionPoolCapacityInfo, + POOL_META_NAME, set_decommission_capacity_info_overrides_for_test, + }; + use crate::data_movement; + use crate::disk::RUSTFS_META_BUCKET; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::services::rebalance::{ + test_three_pool_stores_with_isolated_node_contexts, + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts, + }; + use crate::services::tier::test_util::register_mock_tier; + use crate::set_disk::{ + DeleteObjectCommitBarrier, MultipartCommitBarrier, MultipartCommitPause, NewMultipartUploadCommitObservation, + PutObjectCommitBarrier, PutObjectCommitPause, + }; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::multipart::MultipartOperations as _; + use crate::storage_api_contracts::namespace::NamespaceLocking as _; + use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations as _}; + use http::HeaderMap; + use std::collections::HashMap; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + #[derive(Clone, Copy)] + enum ExternalObjectMutation { + Put, + CompleteMultipart, + Copy, + PutTags, + DeleteTags, + PutMetadata, + Delete, + Restore, + } + + impl ExternalObjectMutation { + fn label(self) -> &'static str { + match self { + Self::Put => "put", + Self::CompleteMultipart => "complete-multipart", + Self::Copy => "copy", + Self::PutTags => "put-tags", + Self::DeleteTags => "delete-tags", + Self::PutMetadata => "put-metadata", + Self::Delete => "delete", + Self::Restore => "restore", + } + } + } + + fn decommission_capacity_owner(meta: &crate::core::pools::PoolMeta) -> DecommissionCapacityOwner { + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("test decommission capacity reservation should exist"); + DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: reservation.operation_id, + generation: reservation.generation, + owner_nonce: reservation.owner_nonce, + mutation_id: None, + } + } + + fn test_bucket(label: &str) -> String { + format!("dc-{label}-{}", &uuid::Uuid::new_v4().simple().to_string()[..12]) + } + + async fn new_multipart_upload( + store: &Arc, + pool_index: usize, + bucket: &str, + object: &str, + mut opts: ObjectOptions, + ) -> crate::error::Result { + let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(bucket).await?; + opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard); + store.pools[pool_index].new_multipart_upload(bucket, object, &opts).await + } + + async fn multipart_options(store: &Arc, bucket: &str, mut opts: ObjectOptions) -> ObjectOptions { + let lifecycle_guard = store + .acquire_bucket_lifecycle_read_lock(bucket) + .await + .expect("multipart test should acquire the bucket lifecycle fence"); + opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard); + opts + } + + #[derive(Debug)] + struct CapacityLeaseLossControl { + calls: AtomicUsize, + fail_refresh: std::sync::atomic::AtomicBool, + } + + impl CapacityLeaseLossControl { + fn arm(&self) { + self.fail_refresh.store(true, Ordering::Release); + } + + fn load(&self, ordering: Ordering) -> usize { + self.calls.load(ordering) + } + } + + #[derive(Debug)] + struct CapacityLeaseLossClient { + target: rustfs_lock::ObjectKey, + control: Arc, + active: tokio::sync::Mutex>, + } + + #[async_trait::async_trait] + impl rustfs_lock::LockClient for CapacityLeaseLossClient { + async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result { + self.active + .lock() + .await + .insert(request.lock_id.clone(), request.resource.clone()); + let now = std::time::SystemTime::now(); + Ok(rustfs_lock::LockResponse::success( + rustfs_lock::LockInfo { + id: request.lock_id.clone(), + resource: request.resource.clone(), + lock_type: request.lock_type, + status: rustfs_lock::LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: now, + expires_at: now + request.ttl, + last_refreshed: now, + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }, + Duration::ZERO, + )) + } + + async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + Ok(self.active.lock().await.remove(lock_id).is_some()) + } + + async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + let resource = self.active.lock().await.get(lock_id).cloned(); + if resource.as_ref() == Some(&self.target) { + self.control.calls.fetch_add(1, Ordering::Release); + return Ok(!self.control.fail_refresh.load(Ordering::Acquire)); + } + Ok(resource.is_some()) + } + + async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + self.release(lock_id).await + } + + async fn check_status(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result> { + Ok(None) + } + + async fn get_stats(&self) -> rustfs_lock::Result { + Ok(rustfs_lock::LockStats::default()) + } + + async fn close(&self) -> rustfs_lock::Result<()> { + Ok(()) + } + + async fn is_online(&self) -> bool { + true + } + + async fn is_local(&self) -> bool { + false + } + } + + async fn store_with_capacity_lease_loss( + other_store: &Arc, + ) -> (Arc, Arc) { + let mut pool = (*other_store.pools[0]).clone(); + let mut set = (*pool.disk_set[0]).clone(); + let refresh_calls = Arc::new(CapacityLeaseLossControl { + calls: AtomicUsize::new(0), + fail_refresh: std::sync::atomic::AtomicBool::new(false), + }); + set.lockers = (0..set.lockers.len().max(1)) + .map(|_| { + Arc::new(CapacityLeaseLossClient { + target: rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, POOL_META_NAME), + control: Arc::clone(&refresh_calls), + active: tokio::sync::Mutex::new(HashMap::new()), + }) as Arc + }) + .collect(); + pool.disk_set[0] = Arc::new(set); + let mut pools = other_store.pools.clone(); + pools[0] = Arc::new(pool); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure) + .await; + *ctx.local_disk_map().write().await = other_store.ctx.local_disk_map().read().await.clone(); + let endpoints = other_store + .ctx + .endpoints() + .expect("lease-loss test store should have endpoint topology"); + ctx.set_endpoints(endpoints.clone()); + if let Some(deployment_id) = other_store.ctx.deployment_id() { + ctx.set_deployment_id(deployment_id); + } + let store = Arc::new(crate::store::ECStore { + id: uuid::Uuid::new_v4(), + disk_map: other_store.disk_map.clone(), + pools, + peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&ctx)), + pool_meta: tokio::sync::RwLock::new(other_store.pool_meta.read().await.clone()), + rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()), + decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]), + start_gate: tokio::sync::Mutex::new(()), + pool_meta_save_gate: tokio::sync::Mutex::new( + other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(), + ), + ctx, + bucket_fence_registry: Arc::default(), + }); + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + (store, refresh_calls) + } + + async fn assert_lock_order(mutation: ExternalObjectMutation) { + assert_lock_order_with_tail(mutation, false).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn no_active_external_commit_probe_blocks_decommission_activation_until_commit_owns_it() { + let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("no-active-probe"); + let object = "probe-target.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create no-active probe bucket"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024), + DecommissionPoolCapacityInfo::for_test(1, layout, 1024, 1024, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 1024, 1024, 0), + ]], + ); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.pause_external_object_capacity_probe(); + let put_store = Arc::clone(&store); + let put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(b"staged no-active commit".to_vec()); + put_store.pools[2] + .put_object( + &bucket, + object, + &mut data, + &ObjectOptions { + decommission_capacity_admission: Some(Arc::clone(&put_store)), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_external_object_capacity_probe_acquired().await; + + let activation_store = Arc::clone(&store); + let mut activation = tokio::spawn(async move { + activation_store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut activation) + .await + .is_err(), + "decommission activation must wait behind the no-active commit probe" + ); + + barrier.release_external_object_capacity_probe(); + tokio::time::timeout(std::time::Duration::from_secs(30), put) + .await + .expect("the staged external PUT should finish after probe release") + .expect("the staged external PUT should not panic") + .expect("the staged external PUT should commit"); + drop(barrier); + tokio::time::timeout(std::time::Duration::from_secs(30), activation) + .await + .expect("decommission activation should proceed after the commit releases its probe") + .expect("decommission activation should not panic") + .expect("decommission activation should commit after the probe release"); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_upload_part_holds_decommission_capacity_until_rename() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-part-probe"); + let object = "public-part.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public UploadPart probe bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load public UploadPart bucket incarnation"); + let upload = new_multipart_upload( + &store, + 2, + &bucket, + object, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create public UploadPart probe upload"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024), + DecommissionPoolCapacityInfo::for_test(1, layout, 1024, 1024, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 1024, 1024, 0), + ]], + ); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission); + let put_store = Arc::clone(&other_store); + let put_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(b"public staged part".to_vec()); + put_store + .put_object_part( + &put_bucket, + object, + &upload_id, + 1, + &mut data, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let activation_store = Arc::clone(&other_store); + let mut activation = tokio::spawn(async move { + activation_store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut activation) + .await + .is_err(), + "decommission activation must wait behind the public UploadPart capacity guard" + ); + + barrier.release(); + let part = tokio::time::timeout(std::time::Duration::from_secs(30), put) + .await + .expect("public UploadPart should finish after the commit barrier release") + .expect("public UploadPart task should not panic") + .expect("public UploadPart should commit"); + drop(barrier); + tokio::time::timeout(std::time::Duration::from_secs(30), activation) + .await + .expect("decommission activation should proceed after public UploadPart commit") + .expect("decommission activation task should not panic") + .expect("decommission activation should commit after public UploadPart"); + + let listed = other_store + .list_object_parts( + &bucket, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("public UploadPart result should remain readable"); + assert_eq!(listed.parts.len(), 1, "public UploadPart should publish exactly one part"); + assert_eq!(listed.parts[0].part_num, part.part_num); + assert_eq!(listed.parts[0].etag, part.etag); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_upload_part_stages_before_decommission_capacity_admission() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-part-staged"); + let object = "public-part-staged.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create staged public UploadPart bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load staged public UploadPart bucket incarnation"); + let upload = new_multipart_upload( + &store, + 2, + &bucket, + object, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create staged public UploadPart upload"); + + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost); + let put_store = Arc::clone(&other_store); + let put_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(b"public staged before admission".to_vec()); + put_store + .put_object_part( + &put_bucket, + object, + &upload_id, + 1, + &mut data, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let probe_store = Arc::clone(&other_store); + tokio::time::timeout(std::time::Duration::from_secs(1), async move { + probe_store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("pool metadata mutation probe should commit before UploadPart admission"); + }) + .await + .expect("public UploadPart must not hold a capacity read guard during staging"); + + barrier.release(); + let part = tokio::time::timeout(std::time::Duration::from_secs(30), put) + .await + .expect("staged public UploadPart should finish after the commit barrier release") + .expect("staged public UploadPart task should not panic") + .expect("staged public UploadPart should commit"); + drop(barrier); + + let listed = other_store + .list_object_parts( + &bucket, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("staged public UploadPart result should remain readable"); + assert_eq!(listed.parts.len(), 1, "staged public UploadPart should publish exactly one part"); + assert_eq!(listed.parts[0].part_num, part.part_num); + assert_eq!(listed.parts[0].etag, part.etag); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_put_holds_decommission_capacity_through_early_ack_tail() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-put-tail"); + let object = "public-put-tail.bin"; + let body = vec![0x71; 256 * 1024]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public PUT tail bucket"); + + let mut initial = PutObjReader::from_vec(vec![0x69; body.len()]); + store.pools[2] + .put_object(&bucket, object, &mut initial, &ObjectOptions::default()) + .await + .expect("seed the unreserved public PUT target"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the public PUT tail reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let reservation_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("public PUT tail reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reservation_target, 1, "the public PUT target must remain unreserved"); + + let tail_barrier = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let put_store = Arc::clone(&other_store); + let put_bucket = bucket.clone(); + let put_body = body.clone(); + let expected_body = body.clone(); + let mut put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(put_body); + put_store + .put_object(&put_bucket, object, &mut data, &ObjectOptions::default()) + .await + }); + tail_barrier.wait_until_paused().await; + tokio::time::timeout(Duration::from_secs(30), &mut put) + .await + .expect("public PUT should return its early quorum acknowledgement") + .expect("public PUT task should not panic") + .expect("public PUT should commit its quorum"); + + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the public PUT capacity probe"); + let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe) + .await + .is_err(), + "public PUT early-ACK tail must retain its staged capacity read fence" + ); + + tail_barrier.release(); + drop(tail_barrier); + let capacity_guard = tokio::time::timeout(Duration::from_secs(30), capacity_probe) + .await + .expect("public PUT capacity probe should finish after its tail") + .expect("public PUT capacity probe should not panic") + .expect("public PUT capacity probe should acquire after its tail"); + drop(capacity_guard); + + let mut reader = other_store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("public PUT result should remain readable"); + let mut committed = Vec::new(); + reader + .stream + .read_to_end(&mut committed) + .await + .expect("public PUT result body should be readable"); + assert_eq!(committed, expected_body, "public PUT should publish the requested bytes"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn public_put_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-put-lease-loss"); + let object = "public-put-lease-loss.bin"; + let body = vec![0x72; 256 * 1024]; + let old_body = vec![0x69; body.len()]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public PUT lease-loss bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load public PUT lease-loss bucket incarnation"); + let mut old_target = PutObjReader::from_vec(old_body.clone()); + store.pools[2] + .put_object(&bucket, object, &mut old_target, &ObjectOptions::default()) + .await + .expect("seed the public PUT lease-loss target"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the public PUT lease-loss reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::BeforeQuotaRename); + tokio::time::pause(); + let put_store = Arc::clone(&lossy_store); + let put_bucket = bucket.clone(); + let put_body = body.clone(); + let put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(put_body); + put_store + .put_object( + &put_bucket, + object, + &mut data, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the public PUT capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("public PUT should finish after the commit barrier release") + .expect("public PUT task should not panic") + .expect_err("lost public PUT capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "public PUT lease loss should remain observable"); + drop(barrier); + + let mut target = lossy_store + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("the old public PUT target should remain readable"); + let mut committed = Vec::new(); + target + .stream + .read_to_end(&mut committed) + .await + .expect("the old public PUT target should drain"); + assert_eq!(committed, old_body, "lost public PUT capacity lease must preserve the prior target"); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_cross_object_copy_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-copy-lease-loss"); + let source_object = "public-copy-source.bin"; + let target_object = "public-copy-target.bin"; + let source_body = vec![0x73; 256 * 1024]; + let old_target_body = vec![0x69; source_body.len()]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public CopyObject lease-loss bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load public CopyObject lease-loss bucket incarnation"); + + let mut source = PutObjReader::from_vec(source_body.clone()); + store.pools[2] + .put_object(&bucket, source_object, &mut source, &ObjectOptions::default()) + .await + .expect("seed the public CopyObject source"); + let mut old_target = PutObjReader::from_vec(old_target_body.clone()); + store.pools[2] + .put_object(&bucket, target_object, &mut old_target, &ObjectOptions::default()) + .await + .expect("seed the public CopyObject target"); + + let mut copy_reader = store.pools[2] + .get_object_reader( + &bucket, + source_object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("read the public CopyObject source"); + let mut copy_info = copy_reader.object_info.clone(); + let mut copy_body = Vec::new(); + copy_reader + .stream + .read_to_end(&mut copy_body) + .await + .expect("stage the public CopyObject source bytes"); + assert_eq!(copy_body, source_body); + copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body)); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = source_body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the public CopyObject capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let barrier = PutObjectCommitBarrier::install(&bucket, target_object, PutObjectCommitPause::BeforeQuotaRename); + tokio::time::pause(); + let copy_store = Arc::clone(&lossy_store); + let copy_bucket = bucket.clone(); + let copy = tokio::spawn(async move { + copy_store + .copy_object( + ©_bucket, + source_object, + ©_bucket, + target_object, + &mut copy_info, + &ObjectOptions::default(), + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the public CopyObject capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), copy) + .await + .expect("public CopyObject should finish after the commit barrier release") + .expect("public CopyObject task should not panic") + .expect_err("lost public CopyObject capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "public CopyObject lease loss should remain observable"); + drop(barrier); + tokio::time::resume(); + + let mut target_reader = lossy_store.pools[2] + .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("the old public CopyObject target should remain readable"); + let mut committed = Vec::new(); + target_reader + .stream + .read_to_end(&mut committed) + .await + .expect("the old public CopyObject target body should remain readable"); + assert_eq!(committed, old_target_body, "lost CopyObject capacity lease must not publish new bytes"); + } + + async fn assert_same_object_copy_fences_capacity_lease_loss(transitioned: bool) { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-self-copy-loss"); + let object = "public-self-copy-lease-loss.bin"; + let source_body = vec![0x74; 256 * 1024]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public self-copy lease-loss bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load public self-copy lease-loss bucket incarnation"); + + let (mut copy_info, source_opts, destination_opts, expected_body, expected_tier, expected_current_version) = + if transitioned { + let mut original = PutObjReader::from_vec(source_body.clone()); + let original_info = store.pools[2] + .put_object(&bucket, object, &mut original, &ObjectOptions::default()) + .await + .expect("seed the transitioned self-copy source"); + let tier_name = format!("SELFCOPY{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let _backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await; + store.pools[2] + .transition_object( + &bucket, + object, + &ObjectOptions { + no_lock: true, + transition: crate::bucket::lifecycle::lifecycle::TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original_info.etag.clone().expect("transition source should have an ETag"), + ..Default::default() + }, + mod_time: original_info.mod_time, + ..Default::default() + }, + ) + .await + .expect("seed the transitioned self-copy source"); + let mut copy_info = store.pools[2] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("read the transitioned self-copy source metadata"); + assert_eq!(copy_info.transitioned_object.tier, tier_name); + assert!(!copy_info.metadata_only, "transitioned self-copy must use the reader PUT path"); + let mut reader = store.pools[2] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("read the transitioned self-copy source body"); + let mut copy_body = Vec::new(); + reader + .stream + .read_to_end(&mut copy_body) + .await + .expect("stage the transitioned self-copy source body"); + assert_eq!(copy_body, source_body); + copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body)); + ( + copy_info, + ObjectOptions::default(), + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + source_body.clone(), + Some(tier_name), + None, + ) + } else { + let historical_version = uuid::Uuid::new_v4().to_string(); + let current_version = uuid::Uuid::new_v4().to_string(); + let current_body = vec![0x68; source_body.len()]; + let mut historical = PutObjReader::from_vec(source_body.clone()); + store.pools[2] + .put_object( + &bucket, + object, + &mut historical, + &ObjectOptions { + versioned: true, + version_id: Some(historical_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the historical self-copy source"); + let mut current = PutObjReader::from_vec(current_body.clone()); + store.pools[2] + .put_object( + &bucket, + object, + &mut current, + &ObjectOptions { + versioned: true, + version_id: Some(current_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the current self-copy target"); + let mut copy_info = store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(historical_version.clone()), + ..Default::default() + }, + ) + .await + .expect("read the historical self-copy source metadata"); + let mut reader = store.pools[2] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + versioned: true, + version_id: Some(historical_version.clone()), + ..Default::default() + }, + ) + .await + .expect("read the historical self-copy source body"); + let mut copy_body = Vec::new(); + reader + .stream + .read_to_end(&mut copy_body) + .await + .expect("stage the historical self-copy source body"); + assert_eq!(copy_body, source_body); + copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_body)); + ( + copy_info, + ObjectOptions { + versioned: true, + version_id: Some(historical_version), + ..Default::default() + }, + ObjectOptions { + versioned: true, + expected_current_version_id: Some(current_version.clone()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + current_body, + None, + Some(current_version), + ) + }; + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = source_body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the same-object CopyObject capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::BeforeQuotaRename); + tokio::time::pause(); + let copy_store = Arc::clone(&lossy_store); + let copy_bucket = bucket.clone(); + let copy = tokio::spawn(async move { + copy_store + .copy_object( + ©_bucket, + object, + ©_bucket, + object, + &mut copy_info, + &source_opts, + &destination_opts, + ) + .await + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the same-object CopyObject capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), copy) + .await + .expect("same-object CopyObject should finish after the commit barrier release") + .expect("same-object CopyObject task should not panic") + .expect_err("lost same-object CopyObject capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "same-object CopyObject lease loss should remain observable"); + drop(barrier); + tokio::time::resume(); + + let target_info = lossy_store.pools[2] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("the failed same-object CopyObject target should remain readable"); + if let Some(expected_tier) = expected_tier { + assert_eq!( + target_info.transitioned_object.tier, expected_tier, + "lost transitioned self-copy capacity lease must not publish a local target" + ); + } + if let Some(expected_current_version) = expected_current_version { + assert_eq!( + target_info.version_id.map(|version| version.to_string()), + Some(expected_current_version), + "lost historical self-copy capacity lease must not publish a new current version" + ); + } + let mut target_reader = store.pools[2] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("the failed same-object CopyObject target body should remain readable"); + let mut committed = Vec::new(); + target_reader + .stream + .read_to_end(&mut committed) + .await + .expect("the failed same-object CopyObject target body should drain"); + assert_eq!(committed, expected_body, "lost same-object CopyObject lease must preserve target bytes"); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_historical_self_copy_fences_capacity_lease_loss_before_publication() { + assert_same_object_copy_fences_capacity_lease_loss(false).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn public_transitioned_self_copy_fences_capacity_lease_loss_before_publication() { + assert_same_object_copy_fences_capacity_lease_loss(true).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_restart_reconciles_published_capacity_before_new_upload() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-restart"); + let object = "published-multipart-before-capacity-save.bin"; + let next_object = "next-after-published-multipart-reconcile.bin"; + let first_part = vec![0x61; 5 * 1024 * 1024]; + let second_part = vec![0x62; 1]; + let mut body = first_part.clone(); + body.extend_from_slice(&second_part); + let source_version = uuid::Uuid::new_v4().to_string(); + let next_version = uuid::Uuid::new_v4().to_string(); + let source_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(41); + + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the multipart restart reconciliation bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the multipart restart reconciliation bucket incarnation"); + + let source_upload = new_multipart_upload( + &store, + 0, + &bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + mod_time: Some(source_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the real source multipart upload"); + let source_multipart_opts = multipart_options( + &store, + &bucket, + ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + mod_time: Some(source_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await; + let mut source_parts = Vec::with_capacity(2); + for (part_num, part_body) in [(1, first_part.as_slice()), (2, second_part.as_slice())] { + let mut part_reader = PutObjReader::from_vec(part_body.to_vec()); + let part = store.pools[0] + .put_object_part( + &bucket, + object, + &source_upload.upload_id, + part_num, + &mut part_reader, + &source_multipart_opts, + ) + .await + .expect("stage the real source multipart part"); + source_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + let source_info = store.pools[0] + .clone() + .complete_multipart_upload(&bucket, object, &source_upload.upload_id, source_parts, &source_multipart_opts) + .await + .expect("complete the real source multipart object"); + assert!(source_info.is_multipart(), "the source fixture must take the multipart migration path"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the source multipart capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the real source multipart object for migration"); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test(lossy_store.id, (0..40).map(|_| capacity_snapshot()).collect()); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AfterObjectPublication); + let migration = tokio::spawn({ + let migration_store = Arc::clone(&lossy_store); + let migration_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + Some(incarnation), + "multipart_restart_reconcile_initial", + Some(owner), + ) + .await + } + }); + barrier.wait_until_paused().await; + tokio::time::pause(); + let published = lossy_store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the multipart target must publish before capacity progress save"); + assert!(published.is_multipart(), "published target must retain multipart identity"); + assert_eq!(published.version_id.map(|version| version.to_string()), Some(source_version.clone())); + let mut published_reader = lossy_store.pools[2] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the published multipart target should be readable before reconciliation"); + let mut published_body = Vec::new(); + published_reader + .stream + .read_to_end(&mut published_body) + .await + .expect("the published multipart target body should be readable"); + assert_eq!(published_body, body, "the published multipart target must retain the full source body"); + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the final multipart capacity save must observe its lost lease" + ); + barrier.release(); + tokio::time::resume(); + let initial_err = tokio::time::timeout(Duration::from_secs(30), migration) + .await + .expect("the initial multipart migration should finish after publication release") + .expect("the initial multipart migration task should not panic") + .expect_err("the published multipart target must report its failed capacity progress save"); + assert!( + !initial_err.to_string().is_empty(), + "the failed multipart capacity save must remain observable" + ); + drop(barrier); + + let mut failed_persisted = crate::core::pools::PoolMeta::default(); + failed_persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the published multipart failure must leave readable durable metadata"); + let failed_reservation = failed_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the failed multipart capacity intent must remain durable"); + assert!(failed_reservation.pending_target_physical_bytes > 0); + assert_eq!(failed_reservation.consumed_target_physical_bytes, 0); + *other_store.pool_meta.write().await = failed_persisted.clone(); + let retry_owner = decommission_capacity_owner(&failed_persisted); + + let new_upload_observation = NewMultipartUploadCommitObservation::install(&bucket, object); + let retry_reader = other_store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("restart should reread the exact multipart source version"); + tokio::time::timeout( + Duration::from_secs(30), + data_movement::migrate_decommission_object( + Arc::clone(&other_store), + 0, + bucket.clone(), + retry_reader, + Some(incarnation), + "multipart_restart_reconcile_retry", + Some(retry_owner), + ), + ) + .await + .expect("published multipart retry should not deadlock") + .expect("published multipart retry should reconcile its durable capacity intent"); + assert!( + !new_upload_observation.committed(), + "published multipart retry must reconcile before starting a new MPU" + ); + + let mut reconciled = crate::core::pools::PoolMeta::default(); + reconciled + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("reconciled multipart capacity metadata should remain durable"); + let reconciled_reservation = reconciled.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("reconciled multipart capacity reservation should remain present"); + assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0); + assert_eq!(reconciled_reservation.inflight_target_physical_bytes, 0); + assert_eq!(reconciled_reservation.consumed_target_physical_bytes, body.len()); + assert_eq!(reconciled_reservation.committed_data_bytes, body.len()); + + let mut next_data = PutObjReader::from_vec(body.clone()); + other_store.pools[0] + .put_object( + &bucket, + next_object, + &mut next_data, + &ObjectOptions { + versioned: true, + version_id: Some(next_version.clone()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the next source object for the multipart capacity oracle"); + let next_reader = other_store.pools[0] + .get_object_reader( + &bucket, + next_object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(next_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the next source object for the multipart capacity oracle"); + let next_result = tokio::time::timeout( + Duration::from_secs(30), + data_movement::migrate_decommission_object( + Arc::clone(&other_store), + 0, + bucket.clone(), + next_reader, + Some(incarnation), + "multipart_restart_reconcile_next", + Some(retry_owner), + ), + ) + .await + .expect("the next-object multipart capacity oracle should finish promptly"); + assert!(next_result.is_err(), "consumed multipart capacity must reject the next exact-fit object"); + let next_target_err = other_store.pools[2] + .get_object_info( + &bucket, + next_object, + &ObjectOptions { + versioned: true, + version_id: Some(next_version), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect_err("the next object must not publish after multipart reconciliation"); + assert!( + crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err), + "capacity rejection must leave the next multipart target absent: {next_target_err}" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_restart_cleans_partial_upload_before_exact_fit_retry() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-part-restart"); + let object = "published-part-before-capacity-save.bin"; + let first_part = vec![0x71; 5 * 1024 * 1024]; + let second_part = vec![0x72; 1]; + let mut body = first_part.clone(); + body.extend_from_slice(&second_part); + let source_version = uuid::Uuid::new_v4().to_string(); + let source_mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(71); + + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the partial multipart restart bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the partial multipart restart bucket incarnation"); + let source_upload = new_multipart_upload( + &store, + 0, + &bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + mod_time: Some(source_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the partial-restart source upload"); + let source_multipart_opts = multipart_options( + &store, + &bucket, + ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + mod_time: Some(source_mod_time), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await; + let mut source_parts = Vec::with_capacity(2); + for (part_num, part_body) in [(1, first_part.as_slice()), (2, second_part.as_slice())] { + let mut reader = PutObjReader::from_vec(part_body.to_vec()); + let part = store.pools[0] + .put_object_part(&bucket, object, &source_upload.upload_id, part_num, &mut reader, &source_multipart_opts) + .await + .expect("stage the partial-restart source part"); + source_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + let source_info = store.pools[0] + .clone() + .complete_multipart_upload(&bucket, object, &source_upload.upload_id, source_parts, &source_multipart_opts) + .await + .expect("complete the partial-restart source object"); + assert!(source_info.is_multipart(), "the fixture must use multipart data movement"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source_estimate = body.len().saturating_add(1); + let target_total = source_estimate.saturating_mul(2); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_estimate, source_estimate), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the exact-fit partial multipart reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the partial-restart source object"); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test(lossy_store.id, (0..40).map(|_| capacity_snapshot()).collect()); + let part_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartAfterRename); + let abort_barrier = data_movement::DataMovementMultipartAbortBarrier::install(&bucket, object); + tokio::time::pause(); + let migration = tokio::spawn({ + let migration_store = Arc::clone(&lossy_store); + let migration_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + Some(incarnation), + "multipart_partial_restart_initial", + Some(owner), + ) + .await + } + }); + part_barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the published UploadPart capacity save must lose its lease" + ); + part_barrier.release(); + abort_barrier.wait_until_paused().await; + migration.abort(); + let _ = migration.await; + drop(abort_barrier); + drop(part_barrier); + tokio::time::resume(); + + let upload_identity = format!("v1:{source_version}:{}", source_mod_time.unix_timestamp_nanos()); + let stale_uploads = lossy_store.pools[2] + .get_disks_by_key(object) + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity) + .await + .expect("discover the published partial upload after the simulated crash"); + assert_eq!(stale_uploads.len(), 1, "the failed UploadPart must leave exactly one recoverable upload"); + + let mut failed_persisted = crate::core::pools::PoolMeta::default(); + failed_persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("load the durable partial UploadPart capacity intent"); + let failed_reservation = failed_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the partial UploadPart reservation should remain durable"); + assert!(failed_reservation.pending_target_physical_bytes > 0); + assert_eq!(failed_reservation.consumed_target_physical_bytes, 0); + *other_store.pool_meta.write().await = failed_persisted.clone(); + set_decommission_capacity_info_overrides_for_test(other_store.id, (0..80).map(|_| capacity_snapshot()).collect()); + let retry_owner = decommission_capacity_owner(&failed_persisted); + let retry_reader = other_store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("restart should reread the exact partial multipart source"); + let new_upload = NewMultipartUploadCommitObservation::install(&bucket, object); + let retry_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::NewUploadBeforeLockLost); + let retry = tokio::spawn({ + let retry_store = Arc::clone(&other_store); + let retry_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + retry_store, + 0, + retry_bucket, + retry_reader, + Some(incarnation), + "multipart_partial_restart_retry", + Some(retry_owner), + ) + .await + } + }); + retry_barrier.wait_until_paused().await; + let uploads_before_retry = other_store.pools[2] + .get_disks_by_key(object) + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity) + .await + .expect("scan for the old partial upload before the replacement upload commits"); + assert!( + uploads_before_retry.is_empty(), + "restart must delete the old partial upload before publishing a replacement upload" + ); + retry_barrier.release(); + tokio::time::timeout(Duration::from_secs(30), retry) + .await + .expect("the exact-fit partial multipart retry should not deadlock") + .expect("the exact-fit partial multipart retry task should not panic") + .expect("the exact-fit partial multipart retry should converge"); + drop(retry_barrier); + assert!(new_upload.committed(), "restart must create a fresh upload after cleaning the old one"); + let remaining_uploads = other_store.pools[2] + .get_disks_by_key(object) + .data_movement_multipart_upload_ids(&bucket, object, Some(incarnation), &upload_identity) + .await + .expect("scan for residual partial uploads after restart convergence"); + assert!( + remaining_uploads.is_empty(), + "restart must remove the original partial upload before retrying" + ); + + let mut reconciled = crate::core::pools::PoolMeta::default(); + reconciled + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("reload exact-fit multipart progress after restart"); + let reservation = reconciled.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the exact-fit multipart reservation should remain readable"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, body.len()); + assert_eq!(reservation.committed_data_bytes, body.len()); + + let mut target = other_store.pools[2] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the exact-fit retry target should be readable"); + let mut target_body = Vec::new(); + target + .stream + .read_to_end(&mut target_body) + .await + .expect("read the exact-fit retry target body"); + assert_eq!(target_body, body); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_abort_fences_capacity_lease_loss_before_delete() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-abort-fence"); + let object = "abort-capacity-lease-before-delete.bin"; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 2, 2), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4, 4), + DecommissionPoolCapacityInfo::for_test(2, layout, 4, 4, 0), + ] + }; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the abort fence bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the abort fence incarnation"); + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the abort fence reservation"); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); + let mut upload_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + rustfs_utils::http::insert_str( + &mut upload_opts.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "v1:abort-fence:1".to_string(), + ); + let upload = new_multipart_upload(&store, 2, &bucket, object, upload_opts) + .await + .expect("create the abort fence staging upload"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test(lossy_store.id, vec![capacity_snapshot(), capacity_snapshot()]); + let mut abort_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut abort_opts); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AbortBeforeDelete); + tokio::time::pause(); + let abort = tokio::spawn({ + let abort_store = Arc::clone(&lossy_store); + let abort_bucket = bucket.clone(); + let abort_upload_id = upload.upload_id.clone(); + let abort_opts = abort_opts.clone(); + async move { + abort_store + .abort_multipart_upload_for_data_movement(2, &abort_bucket, object, &abort_upload_id, &abort_opts) + .await + } + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the abort delete must observe capacity lease refresh loss" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), abort) + .await + .expect("the fenced abort should finish after release") + .expect("the fenced abort task should not panic") + .expect_err("lost capacity lease must reject the staging delete"); + assert!(!err.to_string().is_empty()); + drop(barrier); + tokio::time::resume(); + + let upload_info = lossy_store.pools[2] + .list_object_parts( + &bucket, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + data_movement: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("the fenced abort must leave the staging upload readable"); + assert!(upload_info.parts.is_empty()); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-abort-restart"); + let object = "deleted-upload-before-release-save.bin"; + let body = vec![0x79; 256 * 1024]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let source_estimate = body.len().saturating_add(1); + let target_total = source_estimate.saturating_mul(2); + let capacity_snapshot = |target_free| { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_estimate, source_estimate), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_free, target_total, target_total - target_free), + ] + }; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the abort restart bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the abort restart incarnation"); + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the abort restart reservation"); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4()); + let upload_identity = format!("v1:{}:{}", uuid::Uuid::new_v4(), 91); + let mut new_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + rustfs_utils::http::insert_str( + &mut new_opts.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + upload_identity, + ); + owner.apply_to(&mut new_opts); + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![ + capacity_snapshot(target_total), + capacity_snapshot(target_total), + capacity_snapshot(target_total - 1), + ], + ); + let (upload, target_pool_idx, _) = store + .handle_new_multipart_upload_with_pool_idx(&bucket, object, &new_opts, None) + .await + .expect("create the capacity-accounted data movement upload"); + assert_eq!(target_pool_idx, 2); + + let mut part_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + part_number: Some(1), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut part_opts); + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![ + capacity_snapshot(target_total - 1), + capacity_snapshot(target_total - 1), + capacity_snapshot(target_total - 1 - body.len()), + ], + ); + let mut part_reader = PutObjReader::from_vec(body.clone()); + store + .put_object_part_for_data_movement(target_pool_idx, &bucket, object, &upload.upload_id, &mut part_reader, &part_opts) + .await + .expect("persist the scoped temporary multipart inflight bytes"); + let persisted_before_abort = store.pool_meta.read().await.clone(); + let before_reservation = persisted_before_abort.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the abort fixture reservation should exist"); + assert_eq!(before_reservation.pending_target_physical_bytes, 0); + assert_eq!(before_reservation.inflight_target_physical_bytes, body.len() + 1); + assert_eq!(before_reservation.targets[0].temporary_mutations.len(), 1); + assert_eq!( + before_reservation.targets[0].temporary_mutations[0].mutation_id, + owner + .mutation_id + .expect("the abort fixture owner should have a mutation identity") + ); + *other_store.pool_meta.write().await = persisted_before_abort; + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![ + capacity_snapshot(target_total - 1 - body.len()), + capacity_snapshot(target_total - 1 - body.len()), + ], + ); + let mut abort_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut abort_opts); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::AbortAfterDelete); + tokio::time::pause(); + let abort = tokio::spawn({ + let abort_store = Arc::clone(&lossy_store); + let abort_bucket = bucket.clone(); + let abort_object = object.to_string(); + let abort_upload_id = upload.upload_id.clone(); + let abort_opts = abort_opts.clone(); + async move { + abort_store + .abort_multipart_upload_for_data_movement( + target_pool_idx, + &abort_bucket, + &abort_object, + &abort_upload_id, + &abort_opts, + ) + .await + } + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the abort release save must observe capacity lease loss after deletion" + ); + barrier.release(); + let abort_err = tokio::time::timeout(Duration::from_secs(30), abort) + .await + .expect("the fenced abort should finish after release") + .expect("the fenced abort task should not panic") + .expect_err("the deleted upload must report its lost final capacity save"); + assert!(!abort_err.to_string().is_empty()); + drop(barrier); + tokio::time::resume(); + + let missing = lossy_store.pools[target_pool_idx] + .list_object_parts( + &bucket, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect_err("the first abort must durably delete the staging upload"); + assert!(crate::error::is_err_invalid_upload_id(&missing)); + let mut failed_persisted = crate::core::pools::PoolMeta::default(); + failed_persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("reload inflight capacity after the abort release save loss"); + let failed_reservation = failed_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the failed abort reservation should remain durable"); + assert_eq!(failed_reservation.inflight_target_physical_bytes, body.len() + 1); + assert_eq!(failed_reservation.targets[0].temporary_mutations.len(), 1); + assert_eq!( + failed_reservation.targets[0].temporary_mutations[0].mutation_id, + owner + .mutation_id + .expect("the failed abort owner should retain its mutation identity") + ); + + *other_store.pool_meta.write().await = failed_persisted; + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![ + capacity_snapshot(target_total - 1 - body.len()), + capacity_snapshot(target_total - 1 - body.len()), + ], + ); + other_store + .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, object, &upload.upload_id, &abort_opts) + .await + .expect("restart must treat the missing upload as a confirmed scoped release"); + let mut reconciled = crate::core::pools::PoolMeta::default(); + reconciled + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("reload capacity after idempotent abort reconciliation"); + let reservation = reconciled.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the reconciled abort reservation should remain readable"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert!(reservation.targets[0].temporary_mutations.is_empty()); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_complete_multipart_fences_inner_capacity_lease_loss() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-complete-loss"); + let object = "public-complete-lease-loss.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public CompleteMultipartUpload lease-loss bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load public CompleteMultipartUpload bucket incarnation"); + let upload = new_multipart_upload( + &store, + 2, + &bucket, + object, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create public CompleteMultipartUpload lease-loss upload"); + let multipart_opts = multipart_options( + &store, + &bucket, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await; + let body = b"public complete staged part".to_vec(); + let mut data = PutObjReader::from_vec(body.clone()); + let part = store.pools[2] + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &multipart_opts) + .await + .expect("stage public CompleteMultipartUpload lease-loss part"); + let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the public CompleteMultipartUpload capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename); + tokio::time::pause(); + let complete_store = Arc::clone(&lossy_store); + let complete_bucket = bucket.clone(); + let complete_upload_id = upload.upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload( + &complete_bucket, + object, + &complete_upload_id, + completed_parts, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the inner CompleteMultipartUpload capacity guard must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), complete) + .await + .expect("public CompleteMultipartUpload should finish after the commit barrier release") + .expect("public CompleteMultipartUpload task should not panic") + .expect_err("lost inner capacity lease must fence CompleteMultipartUpload before rename"); + assert!( + err.to_string().contains("quota_reservation"), + "inner capacity lease loss should report a quota reservation fence error: {err}" + ); + drop(barrier); + + let object_err = lossy_store + .get_object_info( + &bucket, + object, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect_err("failed CompleteMultipartUpload must not publish the object"); + assert!( + crate::error::is_err_object_not_found(&object_err), + "failed CompleteMultipartUpload should leave the target object absent: {object_err}" + ); + lossy_store + .get_multipart_info( + &bucket, + object, + &upload.upload_id, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("failed CompleteMultipartUpload must leave the MPU retryable"); + let listed = lossy_store + .list_object_parts( + &bucket, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("failed CompleteMultipartUpload should preserve the staged part"); + assert_eq!(listed.parts.len(), 1, "failed CompleteMultipartUpload must preserve the MPU part"); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_complete_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let object = "data-movement-complete-lease-loss.bin"; + let body = vec![0x4d; 64]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096), + DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the data-movement capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reserved_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reserved_target, 2, "the exact-fit data-movement target must be pool 2"); + + let upload = new_multipart_upload( + &store, + 2, + RUSTFS_META_BUCKET, + object, + ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }, + ) + .await + .expect("create the staged data-movement upload"); + let multipart_opts = multipart_options( + &store, + RUSTFS_META_BUCKET, + ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }, + ) + .await; + let mut data = PutObjReader::from_vec(body.clone()); + let part = store.pools[2] + .put_object_part(RUSTFS_META_BUCKET, object, &upload.upload_id, 1, &mut data, &multipart_opts) + .await + .expect("stage the data-movement completion part"); + let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }]; + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let mut complete_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + owner.apply_to(&mut complete_opts); + let barrier = MultipartCommitBarrier::install(RUSTFS_META_BUCKET, object, MultipartCommitPause::BeforeQuotaRename); + let complete_store = Arc::clone(&lossy_store); + let complete_upload_id = upload.upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload_for_data_movement( + (2, None), + RUSTFS_META_BUCKET, + object, + &complete_upload_id, + completed_parts, + &complete_opts, + ) + .await + }); + barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the data-movement capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), complete) + .await + .expect("data-movement CompleteMultipartUpload should finish after the commit barrier release") + .expect("data-movement CompleteMultipartUpload task should not panic") + .expect_err("a lost data-movement capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "data-movement lease loss should remain observable"); + drop(barrier); + + let object_err = lossy_store.pools[2] + .get_object_info( + RUSTFS_META_BUCKET, + object, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect_err("lost data-movement capacity lease must not publish the object"); + assert!( + crate::error::is_err_object_not_found(&object_err), + "data-movement lease loss should leave the target object absent: {object_err}" + ); + lossy_store.pools[2] + .get_multipart_info( + RUSTFS_META_BUCKET, + object, + &upload.upload_id, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("data-movement lease loss must leave the staged MPU retryable"); + let listed = lossy_store.pools[2] + .list_object_parts( + RUSTFS_META_BUCKET, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("data-movement lease loss should preserve the staged part"); + assert_eq!(listed.parts.len(), 1, "data-movement lease loss must preserve the MPU part"); + let meta = lossy_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement reservation should remain durable after the fenced failure"); + assert_eq!( + reservation.consumed_target_physical_bytes, 0, + "a pre-rename data-movement failure must not consume durable target capacity" + ); + drop(meta); + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the fenced data-movement failure should leave readable pool metadata"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the fenced data-movement reservation should remain persisted"); + assert_eq!( + persisted_reservation.consumed_target_physical_bytes, 0, + "the pre-rename data-movement failure must not persist consumed target capacity" + ); + } + + async fn assert_data_movement_complete_waits_for_capacity_tail() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let object = format!("data-movement-complete-tail-{}.bin", uuid::Uuid::new_v4()); + let body = vec![0x5a; 64]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096), + DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the data-movement tail capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reserved_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement tail reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reserved_target, 2, "the data-movement tail target must be pool 2"); + + let upload = new_multipart_upload( + &store, + 2, + RUSTFS_META_BUCKET, + &object, + ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }, + ) + .await + .expect("create the staged data-movement tail upload"); + let multipart_opts = multipart_options( + &store, + RUSTFS_META_BUCKET, + ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }, + ) + .await; + let mut data = PutObjReader::from_vec(body.clone()); + let part = store.pools[2] + .put_object_part(RUSTFS_META_BUCKET, &object, &upload.upload_id, 1, &mut data, &multipart_opts) + .await + .expect("stage the data-movement tail part"); + let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }]; + + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let mut complete_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + http_preconditions: Some(data_movement::data_movement_target_precondition()), + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + owner.apply_to(&mut complete_opts); + + let tail_barrier = + crate::set_disk::rename_fanout_barrier::arm(&object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let complete_store = Arc::clone(&other_store); + let complete_object = object.clone(); + let complete_upload_id = upload.upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload_for_data_movement( + (2, None), + RUSTFS_META_BUCKET, + &complete_object, + &complete_upload_id, + completed_parts, + &complete_opts, + ) + .await + }); + tail_barrier.wait_until_paused().await; + + { + let meta = other_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("paused data-movement completion reservation should remain present"); + assert!( + reservation.pending_target_physical_bytes > 0 || reservation.inflight_target_physical_bytes > 0, + "paused data-movement completion must retain pending or inflight capacity" + ); + assert_eq!( + reservation.consumed_target_physical_bytes, 0, + "paused data-movement completion must not persist consumed capacity" + ); + } + + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the data-movement tail capacity probe"); + let mut capacity_probe = + tokio::spawn(async move { capacity_lock.get_write_lock(std::time::Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut capacity_probe) + .await + .is_err(), + "data-movement completion must retain the capacity write lock while its tail is paused" + ); + + tail_barrier.release(); + drop(tail_barrier); + tokio::time::timeout(std::time::Duration::from_secs(30), complete) + .await + .expect("data-movement CompleteMultipartUpload should finish after its tail resumes") + .expect("data-movement CompleteMultipartUpload task should not panic") + .expect("data-movement CompleteMultipartUpload should commit after its tail resumes"); + let capacity_guard = tokio::time::timeout(std::time::Duration::from_secs(30), capacity_probe) + .await + .expect("capacity probe should finish after data-movement tail completion") + .expect("capacity probe should not panic") + .expect("capacity probe should acquire after data-movement tail completion"); + drop(capacity_guard); + + let meta = other_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("completed data-movement tail reservation should remain present"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert!( + reservation.consumed_target_physical_bytes > 0, + "data-movement completion should consume capacity only after its tail" + ); + drop(meta); + + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("data-movement tail completion should persist readable pool metadata"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("persisted data-movement tail reservation should remain present"); + assert_eq!(persisted_reservation.pending_target_physical_bytes, 0); + assert_eq!(persisted_reservation.inflight_target_physical_bytes, 0); + assert!(persisted_reservation.consumed_target_physical_bytes > 0); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_complete_does_not_release_capacity_before_early_ack_tail() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_data_movement_complete_waits_for_capacity_tail().await + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_put_fences_capacity_lease_loss_before_publication() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_data_movement_put_fences_capacity_lease_loss_before_publication().await + }) + .await; + } + + async fn assert_data_movement_put_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let object = "data-movement-put-lease-loss.bin"; + let body = vec![0x52; 256 * 1024]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the data-movement PUT reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reservation_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement PUT reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reservation_target, 2, "data-movement PUT should use the reserved target"); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let version_id = uuid::Uuid::new_v4().to_string(); + let mut put_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.clone()), + src_pool_idx: 0, + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + owner.apply_to(&mut put_opts); + let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&lossy_store); + let put_opts = put_opts.clone(); + let put_body = body.clone(); + let put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(put_body); + put_store + .put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut data, &put_opts, None) + .await + }); + barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the data-movement PUT capacity lease must lose refresh quorum" + ); + barrier.release(); + tokio::time::resume(); + let (_target_idx, result) = tokio::time::timeout(Duration::from_secs(2), put) + .await + .expect("data-movement PUT should finish after the commit barrier release") + .expect("data-movement PUT task should not panic") + .expect("data-movement PUT should select its reserved target"); + let err = result.expect_err("a lost data-movement PUT capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "data-movement PUT lease loss should remain observable"); + drop(barrier); + + let object_err = lossy_store.pools[2] + .get_object_info( + RUSTFS_META_BUCKET, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.clone()), + ..Default::default() + }, + ) + .await + .expect_err("lost data-movement PUT capacity lease must not publish the object"); + assert!( + crate::error::is_err_object_not_found(&object_err), + "data-movement PUT lease loss should leave the target object absent: {object_err}" + ); + + let meta = lossy_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement PUT reservation should remain after the fenced failure"); + assert!( + reservation.pending_target_physical_bytes > 0, + "failed data-movement PUT must retain its durable pending reservation" + ); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + let live_progress = ( + reservation.pending_target_physical_bytes, + reservation.inflight_target_physical_bytes, + reservation.consumed_target_physical_bytes, + reservation.observed_target_physical_bytes, + reservation.committed_data_bytes, + ); + drop(meta); + + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the fenced data-movement PUT failure should leave readable pool metadata"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the fenced data-movement PUT reservation should remain persisted"); + assert_eq!( + ( + persisted_reservation.pending_target_physical_bytes, + persisted_reservation.inflight_target_physical_bytes, + persisted_reservation.consumed_target_physical_bytes, + persisted_reservation.observed_target_physical_bytes, + persisted_reservation.committed_data_bytes, + ), + live_progress, + "fenced data-movement PUT progress must match its durable replica" + ); + + *other_store.pool_meta.write().await = persisted.clone(); + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![ + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + ], + ); + let retry_owner = decommission_capacity_owner(&persisted); + let mut retry_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.clone()), + src_pool_idx: 0, + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + retry_owner.apply_to(&mut retry_opts); + let mut retry_data = PutObjReader::from_vec(body.clone()); + let (retry_target, retry_result) = other_store + .put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut retry_data, &retry_opts, None) + .await + .expect("same-mutation retry should reach the staged target write"); + assert_eq!(retry_target, 2); + retry_result.expect("same-mutation retry should recover the pending intent and publish"); + let retry_info = other_store.pools[2] + .get_object_info( + RUSTFS_META_BUCKET, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version_id), + ..Default::default() + }, + ) + .await + .expect("same-mutation retry should publish a readable target"); + assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64")); + + let next_object = "data-movement-put-after-retry.bin"; + let next_version_id = uuid::Uuid::new_v4().to_string(); + let mut next_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(next_version_id.clone()), + src_pool_idx: 0, + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + retry_owner.apply_to(&mut next_opts); + let mut next_data = PutObjReader::from_vec(body); + let next_result = other_store + .put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None) + .await; + assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity"); + let next_target_err = other_store.pools[2] + .get_object_info( + RUSTFS_META_BUCKET, + next_object, + &ObjectOptions { + versioned: true, + version_id: Some(next_version_id), + ..Default::default() + }, + ) + .await + .expect_err("the next mutation must remain unpublished after A recovery"); + assert!(crate::error::is_err_object_not_found(&next_target_err)); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_put_holds_capacity_through_successful_early_ack_tail() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_data_movement_put_holds_capacity_through_successful_tail().await + }) + .await; + } + + async fn assert_data_movement_put_holds_capacity_through_successful_tail() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let object = "data-movement-put-success-tail.bin"; + let body = vec![0x53; 256 * 1024]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the successful data-movement PUT reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reservation_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("successful data-movement PUT reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reservation_target, 2, "successful data-movement PUT should use the reserved target"); + + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let version_id = uuid::Uuid::new_v4().to_string(); + let mut put_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.clone()), + src_pool_idx: 0, + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + owner.apply_to(&mut put_opts); + let tail_barrier = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let put_store = Arc::clone(&other_store); + let put_opts = put_opts.clone(); + let put_body = body.clone(); + let mut put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(put_body); + put_store + .put_object_for_data_movement(RUSTFS_META_BUCKET, object, &mut data, &put_opts, None) + .await + }); + tail_barrier.wait_until_paused().await; + + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(), + "owner-bearing data-movement PUT must wait for the rename tail instead of early-ACKing" + ); + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the data-movement PUT tail capacity probe"); + let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe) + .await + .is_err(), + "data-movement PUT tail must retain its capacity fence" + ); + { + let meta = other_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("paused successful data-movement PUT reservation should remain present"); + assert!( + reservation.pending_target_physical_bytes > 0, + "paused successful data-movement PUT must retain pending capacity" + ); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + } + + tail_barrier.release(); + drop(tail_barrier); + let (target_pool_idx, put_result) = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("data-movement PUT should finish after its rename tail resumes") + .expect("data-movement PUT task should not panic") + .expect("data-movement PUT should select its reserved target"); + let committed = put_result.expect("data-movement PUT should commit after its rename tail resumes"); + assert_eq!(target_pool_idx, 2); + assert_eq!(committed.version_id.map(|id| id.to_string()), Some(version_id.clone())); + let target = other_store.pools[2] + .get_object_info( + RUSTFS_META_BUCKET, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version_id), + ..Default::default() + }, + ) + .await + .expect("successful data-movement PUT should remain readable after its tail"); + assert!(!target.delete_marker); + + let capacity_guard = tokio::time::timeout(Duration::from_secs(30), capacity_probe) + .await + .expect("capacity probe should finish after the data-movement PUT tail") + .expect("capacity probe should not panic") + .expect("capacity probe should acquire after the data-movement PUT tail"); + drop(capacity_guard); + + let meta = other_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("completed data-movement PUT reservation should remain present"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert!( + reservation.consumed_target_physical_bytes > 0, + "data-movement PUT should consume capacity only after its tail" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("equivalent-target"); + let object = "published-before-capacity-save.bin"; + let interleaved_object = "interleaved-before-capacity-reconcile.bin"; + let next_object = "next-after-capacity-reconcile.bin"; + let body = vec![0x54; 256 * 1024]; + let source_version = uuid::Uuid::new_v4().to_string(); + let interleaved_source_version = uuid::Uuid::new_v4().to_string(); + let next_source_version = uuid::Uuid::new_v4().to_string(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the equivalent-target reconciliation bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the equivalent-target reconciliation bucket incarnation"); + + let mut source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_data, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the source object whose target will publish before the save"); + let mut interleaved_source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + interleaved_object, + &mut interleaved_source_data, + &ObjectOptions { + versioned: true, + version_id: Some(interleaved_source_version.clone()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the interleaved source object"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the source capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the source object for the first migration attempt"); + let barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::AfterRenameHandoff); + let migration = tokio::spawn({ + let migration_store = Arc::clone(&lossy_store); + let migration_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + Some(incarnation), + "equivalent_target_reconcile_initial", + Some(owner), + ) + .await + } + }); + barrier.wait_until_paused().await; + tokio::time::pause(); + let published = lossy_store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the target must be published before capacity progress save"); + assert_eq!(published.version_id.map(|version| version.to_string()), Some(source_version.clone())); + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the initial capacity progress save must observe its lost lease" + ); + barrier.release(); + tokio::time::resume(); + let initial_err = tokio::time::timeout(Duration::from_secs(30), migration) + .await + .expect("the initial migration should finish after the commit barrier release") + .expect("the initial migration task should not panic") + .expect_err("the published target must report its failed capacity progress save"); + assert!(!initial_err.to_string().is_empty(), "the failed capacity save must remain observable"); + drop(barrier); + + let mut failed_persisted = crate::core::pools::PoolMeta::default(); + failed_persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the published target failure must leave readable durable metadata"); + let failed_reservation = failed_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the failed target capacity intent must remain durable"); + assert!(failed_reservation.pending_target_physical_bytes > 0); + assert!( + failed_reservation + .targets + .iter() + .any(|target| target.pending_mutation_id.is_some()) + ); + assert_eq!(failed_reservation.consumed_target_physical_bytes, 0); + + *other_store.pool_meta.write().await = failed_persisted.clone(); + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![ + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + ], + ); + let retry_owner = decommission_capacity_owner(&failed_persisted); + let interleaved_reader = other_store.pools[0] + .get_object_reader( + &bucket, + interleaved_object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(interleaved_source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the interleaved source object while the first intent is pending"); + let interleaved_result = tokio::time::timeout( + Duration::from_secs(30), + data_movement::migrate_decommission_object( + Arc::clone(&other_store), + 0, + bucket.clone(), + interleaved_reader, + Some(incarnation), + "equivalent_target_reconcile_interleaved", + Some(retry_owner), + ), + ) + .await + .expect("a different mutation must be rejected without waiting on the pending intent"); + assert!(interleaved_result.is_err(), "B must not reuse A's pending capacity intent"); + assert!( + interleaved_result + .as_ref() + .expect_err("B must not succeed while A's pending intent is unresolved") + .to_string() + .contains("unresolved target capacity intent") + ); + let interleaved_target_err = other_store.pools[2] + .get_object_info( + &bucket, + interleaved_object, + &ObjectOptions { + versioned: true, + version_id: Some(interleaved_source_version), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect_err("B must not publish while A's pending intent is unresolved"); + assert!( + crate::error::is_err_object_not_found(&interleaved_target_err) + || crate::error::is_err_version_not_found(&interleaved_target_err) + ); + let retry_reader = other_store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("restart retry should reread the exact source version"); + tokio::time::timeout( + Duration::from_secs(30), + data_movement::migrate_decommission_object( + Arc::clone(&other_store), + 0, + bucket.clone(), + retry_reader, + Some(incarnation), + "equivalent_target_reconcile_retry", + Some(retry_owner), + ), + ) + .await + .expect("equivalent-target retry should not deadlock") + .expect("equivalent target retry should reconcile the pending capacity intent"); + + let mut reconciled = crate::core::pools::PoolMeta::default(); + reconciled + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("reconciled capacity metadata should remain durable"); + let reconciled_reservation = reconciled.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("reconciled capacity reservation should remain present"); + assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0); + assert_eq!(reconciled_reservation.inflight_target_physical_bytes, 0); + assert_eq!(reconciled_reservation.consumed_target_physical_bytes, body.len()); + assert_eq!(reconciled_reservation.committed_data_bytes, body.len()); + + let mut next_source_data = PutObjReader::from_vec(body.clone()); + store.pools[0] + .put_object( + &bucket, + next_object, + &mut next_source_data, + &ObjectOptions { + versioned: true, + version_id: Some(next_source_version.clone()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the next source object for the capacity oracle"); + let next_reader = other_store.pools[0] + .get_object_reader( + &bucket, + next_object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(next_source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the next source object for the capacity oracle"); + let next_result = tokio::time::timeout( + Duration::from_secs(30), + data_movement::migrate_decommission_object( + Arc::clone(&other_store), + 0, + bucket.clone(), + next_reader, + Some(incarnation), + "equivalent_target_reconcile_next_object", + Some(retry_owner), + ), + ) + .await + .expect("the next object capacity oracle should finish promptly"); + assert!(next_result.is_err(), "consumed capacity must reject the next exact-fit object"); + let next_target_err = other_store.pools[2] + .get_object_info( + &bucket, + next_object, + &ObjectOptions { + versioned: true, + version_id: Some(next_source_version), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect_err("the next object must not publish after the reservation is consumed"); + assert!( + crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err), + "capacity rejection must leave the next target absent: {next_target_err}" + ); + let mut final_persisted = crate::core::pools::PoolMeta::default(); + final_persisted + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("the next-object capacity rejection must preserve durable reconciliation"); + let final_reservation = final_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the final capacity reservation should remain present"); + assert_eq!(final_reservation.pending_target_physical_bytes, 0); + assert_eq!(final_reservation.consumed_target_physical_bytes, body.len()); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_equivalent_delete_marker_reconciles_pending_capacity_after_restart() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("equivalent-delete-marker"); + let object = "published-delete-marker.bin"; + let next_object = "next-delete-marker.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the equivalent delete-marker bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load the equivalent delete-marker bucket incarnation"); + + let mut source_body = PutObjReader::from_vec(b"delete marker source".to_vec()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source_body, + &ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the source object"); + let source_marker = store.pools[0] + .delete_object( + &bucket, + object, + ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the source delete marker"); + let source_version_id = source_marker.version_id.expect("source marker must have a version id"); + let source_mod_time = source_marker.mod_time.expect("source marker must have a modification time"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 1, 1), + DecommissionPoolCapacityInfo::for_test(2, layout, 2, 2, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the delete-marker capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let mut move_opts = ObjectOptions { + data_movement: true, + delete_marker: true, + versioned: true, + version_id: Some(source_version_id.to_string()), + mod_time: Some(source_mod_time), + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut move_opts); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![ + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + capacity_snapshot(), + ], + ); + let barrier = DeleteObjectCommitBarrier::install_after_publish(&bucket, object); + let publish_store = Arc::clone(&lossy_store); + let publish_bucket = bucket.clone(); + let publish_opts = move_opts.clone(); + let publish = tokio::spawn(async move { publish_store.delete_object(&publish_bucket, object, publish_opts).await }); + barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + barrier.release(); + let publish_err = tokio::time::timeout(Duration::from_secs(30), publish) + .await + .expect("the published marker should finish after the deterministic barrier release") + .expect("the published marker task should not panic") + .expect_err("the final capacity progress save must observe the lost lease"); + assert!(!publish_err.to_string().is_empty()); + tokio::time::resume(); + drop(barrier); + let published = lossy_store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version_id.to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("the target marker must be visible before the simulated restart"); + assert!(published.delete_marker); + + let mut failed_persisted = crate::core::pools::PoolMeta::default(); + failed_persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("published marker metadata should remain readable"); + let failed_reservation = failed_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the source reservation should remain durable"); + assert_eq!(failed_reservation.pending_target_physical_bytes, 1); + assert_eq!(failed_reservation.consumed_target_physical_bytes, 0); + assert!( + failed_reservation + .targets + .iter() + .any(|target| target.pending_mutation_id.is_some()) + ); + *other_store.pool_meta.write().await = failed_persisted.clone(); + let mut restart_persisted = crate::core::pools::PoolMeta::default(); + restart_persisted + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("the pending marker intent should survive restart"); + assert_eq!( + restart_persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the restarted reservation should exist") + .pending_target_physical_bytes, + 1 + ); + *other_store.pool_meta.write().await = restart_persisted.clone(); + + let mut next_source_body = PutObjReader::from_vec(b"next delete marker".to_vec()); + store.pools[0] + .put_object( + &bucket, + next_object, + &mut next_source_body, + &ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the next source object"); + let next_source_marker = store.pools[0] + .delete_object( + &bucket, + next_object, + ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the next source delete marker"); + let next_source_version_id = next_source_marker.version_id.expect("next marker version id"); + let next_owner = decommission_capacity_owner(&restart_persisted); + let mut next_opts = ObjectOptions { + data_movement: true, + delete_marker: true, + versioned: true, + version_id: Some(next_source_version_id.to_string()), + mod_time: next_source_marker.mod_time, + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + next_owner.apply_to(&mut next_opts); + let next_before_reconcile = other_store + .delete_object(&bucket, next_object, next_opts.clone()) + .await + .expect_err("a next mutation must not reuse the pending marker intent"); + assert!(next_before_reconcile.to_string().contains("decommission_capacity_blocked")); + + other_store + .delete_object(&bucket, object, move_opts) + .await + .expect("the equivalent target retry must reconcile the pending marker intent"); + let mut reconciled = crate::core::pools::PoolMeta::default(); + reconciled + .load_no_lock_from_replicas(other_store.pools.clone()) + .await + .expect("reconciled marker metadata should remain durable"); + let reconciled_reservation = reconciled.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the reconciled reservation should exist"); + assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0); + assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1); + + let next_after_reconcile = other_store + .delete_object(&bucket, next_object, next_opts) + .await + .expect_err("the consumed exact-fit capacity must reject the next marker"); + assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked")); + let next_target_err = other_store.pools[2] + .get_object_info( + &bucket, + next_object, + &ObjectOptions { + versioned: true, + version_id: Some(next_source_version_id.to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect_err("the rejected next marker must remain unpublished"); + assert!( + crate::error::is_err_object_not_found(&next_target_err) || crate::error::is_err_version_not_found(&next_target_err) + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_delete_marker_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("delete-marker-loss"); + let object = "delete-marker-lease-loss.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create delete-marker lease-loss bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load delete-marker lease-loss bucket incarnation"); + let mut body = PutObjReader::from_vec(b"delete-marker source".to_vec()); + store.pools[0] + .put_object( + &bucket, + object, + &mut body, + &ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the source object for delete-marker movement"); + let source_marker = store.pools[0] + .delete_object( + &bucket, + object, + ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the source delete marker"); + assert!(source_marker.delete_marker, "the source version must be a delete marker"); + let source_version_id = source_marker + .version_id + .expect("the source delete marker must have a version id"); + let source_mod_time = source_marker + .mod_time + .expect("the source delete marker must have a modification time"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1024, 1024), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 4096, 4096), + DecommissionPoolCapacityInfo::for_test(2, layout, 4096, 4096, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the delete-marker capacity reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + assert_eq!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("delete-marker reservation should exist") + .targets[0] + .pool_index, + 2 + ); + + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let mut move_opts = ObjectOptions { + data_movement: true, + delete_marker: true, + versioned: true, + version_id: Some(source_version_id.to_string()), + mod_time: Some(source_mod_time), + src_pool_idx: 0, + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + owner.apply_to(&mut move_opts); + let barrier = DeleteObjectCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&lossy_store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { delete_store.delete_object(&delete_bucket, object, move_opts).await }); + barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the delete-marker capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), delete) + .await + .expect("delete-marker movement should finish after the commit barrier release") + .expect("delete-marker movement task should not panic") + .expect_err("lost delete-marker capacity lease must fence before publication"); + assert!(!err.to_string().is_empty(), "delete-marker lease loss should remain observable"); + drop(barrier); + + let target_err = lossy_store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version_id.to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect_err("lost delete-marker capacity lease must not publish the target marker"); + assert!( + crate::error::is_err_object_not_found(&target_err) || crate::error::is_err_version_not_found(&target_err), + "target delete marker should remain absent after the fenced failure: {target_err}" + ); + let source = lossy_store.pools[0] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version_id.to_string()), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("the source delete marker must remain intact"); + assert!(source.delete_marker); + assert_eq!(source.mod_time, Some(source_mod_time)); + + let meta = lossy_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("delete-marker reservation should remain after the fenced failure"); + assert!(reservation.pending_target_physical_bytes > 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + let live_progress = ( + reservation.pending_target_physical_bytes, + reservation.inflight_target_physical_bytes, + reservation.consumed_target_physical_bytes, + reservation.observed_target_physical_bytes, + reservation.committed_data_bytes, + ); + drop(meta); + + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the fenced delete-marker failure should leave readable pool metadata"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the fenced delete-marker reservation should remain persisted"); + assert_eq!( + ( + persisted_reservation.pending_target_physical_bytes, + persisted_reservation.inflight_target_physical_bytes, + persisted_reservation.consumed_target_physical_bytes, + persisted_reservation.observed_target_physical_bytes, + persisted_reservation.committed_data_bytes, + ), + live_progress, + "fenced delete-marker progress must match its durable replica" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_upload_part_fences_capacity_lease_loss_before_publication() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let object = "data-movement-part-lease-loss.bin"; + let body = vec![0x63; 256 * 1024]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = || { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the data-movement UploadPart reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reservation_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement UploadPart reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reservation_target, 2, "data-movement UploadPart should use the reserved target"); + + let upload = new_multipart_upload( + &store, + 2, + RUSTFS_META_BUCKET, + object, + ObjectOptions { + data_movement: true, + src_pool_idx: 0, + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + ) + .await + .expect("create the data-movement UploadPart target upload"); + let (lossy_store, refresh_calls) = store_with_capacity_lease_loss(&other_store).await; + set_decommission_capacity_info_overrides_for_test( + lossy_store.id, + vec![capacity_snapshot(), capacity_snapshot(), capacity_snapshot()], + ); + let mut part_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + part_number: Some(1), + ..ObjectOptions::with_capacity_expected_data_bytes(Some(body.len())) + }; + owner.apply_to(&mut part_opts); + let barrier = MultipartCommitBarrier::install(RUSTFS_META_BUCKET, object, MultipartCommitPause::PutPartBeforeLockLost); + let part_store = Arc::clone(&lossy_store); + let part_upload_id = upload.upload_id.clone(); + let part_opts = part_opts.clone(); + let part_body = body.clone(); + let part = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(part_body); + part_store + .put_object_part_for_data_movement(2, RUSTFS_META_BUCKET, object, &part_upload_id, &mut data, &part_opts) + .await + }); + barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::task::yield_now().await; + refresh_calls.arm(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::Acquire) > 0, + "the data-movement UploadPart capacity lease must lose refresh quorum" + ); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), part) + .await + .expect("data-movement UploadPart should finish after the commit barrier release") + .expect("data-movement UploadPart task should not panic") + .expect_err("a lost data-movement UploadPart capacity lease must fence before publication"); + assert!( + !err.to_string().is_empty(), + "data-movement UploadPart lease loss should remain observable" + ); + drop(barrier); + + let listed = lossy_store.pools[2] + .list_object_parts( + RUSTFS_META_BUCKET, + object, + &upload.upload_id, + None, + 100, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("the fenced data-movement UploadPart should leave the MPU readable"); + assert!( + listed.parts.is_empty(), + "lost data-movement UploadPart capacity lease must not publish a part" + ); + + let meta = lossy_store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("data-movement UploadPart reservation should remain after the fenced failure"); + assert!(reservation.pending_target_physical_bytes > 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + let live_progress = ( + reservation.pending_target_physical_bytes, + reservation.inflight_target_physical_bytes, + reservation.consumed_target_physical_bytes, + reservation.observed_target_physical_bytes, + reservation.committed_data_bytes, + ); + drop(meta); + + let mut persisted = crate::core::pools::PoolMeta::default(); + persisted + .load_no_lock_from_replicas(lossy_store.pools.clone()) + .await + .expect("the fenced data-movement UploadPart failure should leave readable pool metadata"); + let persisted_reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the fenced data-movement UploadPart reservation should remain persisted"); + assert_eq!( + ( + persisted_reservation.pending_target_physical_bytes, + persisted_reservation.inflight_target_physical_bytes, + persisted_reservation.consumed_target_physical_bytes, + persisted_reservation.observed_target_physical_bytes, + persisted_reservation.committed_data_bytes, + ), + live_progress, + "fenced data-movement UploadPart progress must match its durable replica" + ); + } + + async fn assert_lock_order_with_tail(mutation: ExternalObjectMutation, pause_restore_tail: bool) { + let (_temp_dirs, store, other_store) = if pause_restore_tail { + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await + } else { + test_three_pool_stores_with_isolated_node_contexts(None).await + }; + let bucket = test_bucket(&format!("{}-order", mutation.label())); + let object = "shared-object.bin"; + let copy_source_object = "copy-source.bin"; + let source_version = uuid::Uuid::new_v4().to_string(); + let source_body = b"decommission source body".to_vec(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create lock-order bucket"); + let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation"); + + let mut existing = PutObjReader::from_vec(b"existing unreserved target".to_vec()); + let existing_info = store.pools[2] + .put_object(&bucket, object, &mut existing, &ObjectOptions::default()) + .await + .expect("seed the ordinary PUT target"); + let mut source = PutObjReader::from_vec(source_body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut source, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the decommission source version"); + + let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) { + let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await; + store.pools[2] + .transition_object( + &bucket, + object, + &ObjectOptions { + no_lock: true, + transition: crate::bucket::lifecycle::lifecycle::TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: existing_info.etag.clone().expect("the restore target should have an ETag"), + ..Default::default() + }, + mod_time: existing_info.mod_time, + ..Default::default() + }, + ) + .await + .expect("seed the transitioned restore target"); + Some(backend.arm_get_barrier().await) + } else { + None + }; + + let delete_marker_version = if matches!(mutation, ExternalObjectMutation::Delete) { + let marker = store.pools[2] + .delete_object( + &bucket, + object, + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("seed the ordinary target delete marker"); + Some( + marker + .version_id + .expect("the versioned target delete should return its marker version") + .to_string(), + ) + } else { + None + }; + + let prepared_multipart = if matches!(mutation, ExternalObjectMutation::CompleteMultipart) { + let upload = new_multipart_upload( + &store, + 2, + &bucket, + object, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("create the ordinary multipart upload"); + let multipart_opts = multipart_options( + &store, + &bucket, + ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await; + let mut data = PutObjReader::from_vec(b"ordinary multipart replacement".to_vec()); + let part = store.pools[2] + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &multipart_opts) + .await + .expect("stage the ordinary multipart part"); + Some(( + upload.upload_id, + vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + )) + } else { + None + }; + let copy_body_size = if matches!(mutation, ExternalObjectMutation::Copy) { + if pause_restore_tail { + 256 * 1024 + } else { + b"ordinary copy replacement".len() + } + } else { + 0 + }; + let prepared_copy = if matches!(mutation, ExternalObjectMutation::Copy) { + let copy_body = vec![0x6a; copy_body_size]; + let mut copy_source = PutObjReader::from_vec(copy_body.clone()); + store.pools[2] + .put_object(&bucket, copy_source_object, &mut copy_source, &ObjectOptions::default()) + .await + .expect("seed the ordinary copy source"); + let mut copy_reader = store.pools[2] + .get_object_reader( + &bucket, + copy_source_object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("read the ordinary copy source"); + let mut copy_info = copy_reader.object_info.clone(); + let mut copy_data = Vec::new(); + copy_reader + .stream + .read_to_end(&mut copy_data) + .await + .expect("drain the ordinary copy source"); + assert_eq!(copy_data, copy_body); + copy_info.put_object_reader = Some(PutObjReader::from_vec(copy_data)); + Some(copy_info) + } else { + None + }; + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = source_body.len().max(copy_body_size) * 4; + let capacity_snapshot = |target_free| { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, source_body.len(), source_body.len()), + DecommissionPoolCapacityInfo::for_test( + 1, + layout, + target_free, + target_total, + target_total.saturating_sub(target_free), + ), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the source reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![ + capacity_snapshot(target_total), + capacity_snapshot(target_total), + capacity_snapshot(target_total - source_body.len()), + ], + ); + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.clone()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the exact source version"); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id); + let mut migration = tokio::spawn({ + let migration_store = Arc::clone(&store); + let migration_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + Some(incarnation), + "decommission_object_lock_order", + Some(owner), + ) + .await + } + }); + barrier.wait_until_owner_paused().await; + + let mut ordinary_mutation = tokio::spawn({ + let ordinary_store = Arc::clone(&other_store); + let ordinary_bucket = bucket.clone(); + async move { + match mutation { + ExternalObjectMutation::Put => { + let mut data = PutObjReader::from_vec(b"ordinary replacement".to_vec()); + ordinary_store + .put_object(&ordinary_bucket, object, &mut data, &ObjectOptions::default()) + .await + .map(|_| ()) + } + ExternalObjectMutation::CompleteMultipart => { + let (upload_id, completed_parts) = + prepared_multipart.expect("multipart mutation should have a staged upload"); + ordinary_store + .complete_multipart_upload( + &ordinary_bucket, + object, + &upload_id, + completed_parts, + &ObjectOptions::default(), + ) + .await + .map(|_| ()) + } + ExternalObjectMutation::Copy => { + let mut copy_info = prepared_copy.expect("copy mutation should have a source reader"); + ordinary_store + .copy_object( + &ordinary_bucket, + copy_source_object, + &ordinary_bucket, + object, + &mut copy_info, + &ObjectOptions::default(), + &ObjectOptions::default(), + ) + .await + .map(|_| ()) + } + ExternalObjectMutation::PutTags => ordinary_store + .put_object_tags(&ordinary_bucket, object, "ordinary=tag", &ObjectOptions::default()) + .await + .map(|_| ()), + ExternalObjectMutation::DeleteTags => ordinary_store + .delete_object_tags(&ordinary_bucket, object, &ObjectOptions::default()) + .await + .map(|_| ()), + ExternalObjectMutation::PutMetadata => ordinary_store + .put_object_metadata(&ordinary_bucket, object, &ObjectOptions::default()) + .await + .map(|_| ()), + ExternalObjectMutation::Delete => { + let version_id = delete_marker_version.expect("delete mutation should have a marker version"); + ordinary_store + .delete_object( + &ordinary_bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(version_id.clone()), + expected_current_version_id: Some(version_id), + ..Default::default() + }, + ) + .await + .map(|_| ()) + } + ExternalObjectMutation::Restore => { + let mut opts = ObjectOptions::default(); + opts.transition.restore_request.days = Some(1); + ordinary_store + .restore_transitioned_object(&ordinary_bucket, object, &opts) + .await + } + } + } + }); + if let Some(get_barrier) = restore_get_barrier.as_ref() { + get_barrier.wait_until_paused().await; + let read_opts = ObjectOptions { + skip_decommissioned: true, + ..Default::default() + }; + tokio::time::timeout( + std::time::Duration::from_secs(1), + other_store.get_object_info(&bucket, object, &read_opts), + ) + .await + .expect("HEAD must not wait for the paused tier restore download") + .expect("HEAD should remain readable during the paused tier restore download"); + let mut reader = tokio::time::timeout( + std::time::Duration::from_secs(1), + other_store.get_object_reader(&bucket, object, None, HeaderMap::new(), &read_opts), + ) + .await + .expect("GET must not wait for the paused tier restore download") + .expect("GET should remain readable during the paused tier restore download"); + let mut body = Vec::new(); + tokio::time::timeout(std::time::Duration::from_secs(1), reader.stream.read_to_end(&mut body)) + .await + .expect("GET body must not wait for the paused tier restore download") + .expect("GET body should remain readable during the paused tier restore download"); + assert_eq!(body, b"existing unreserved target"); + get_barrier.release(); + } + if matches!( + mutation, + ExternalObjectMutation::Restore + | ExternalObjectMutation::Put + | ExternalObjectMutation::CompleteMultipart + | ExternalObjectMutation::Copy + ) { + if pause_restore_tail { + barrier.pause_external_object_commit_phase(); + } + barrier.wait_until_external_object_commit_phase_started().await; + } else { + barrier.wait_until_external_capacity_released().await; + } + assert!(!ordinary_mutation.is_finished()); + let migration_result = if pause_restore_tail { + barrier.release_owner(); + tokio::time::timeout(std::time::Duration::from_secs(30), &mut migration) + .await + .expect("migration must finish before the early-ACK tail is released") + .expect("migration task should join") + } else { + barrier.release_owner(); + tokio::time::timeout(std::time::Duration::from_secs(30), migration) + .await + .expect("migration must not deadlock with the ordinary object mutation") + .expect("migration task should join") + }; + migration_result.expect("migration should commit to its reserved target"); + + let ordinary_result = if pause_restore_tail { + let handoff_barrier = PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::AfterRenameHandoff); + let tail_barrier = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + barrier.release_external_object_commit_phase(); + tokio::time::timeout(std::time::Duration::from_secs(30), handoff_barrier.wait_until_paused()) + .await + .expect("restore should reach the quorum handoff before the tail probe"); + handoff_barrier.release(); + drop(handoff_barrier); + tokio::time::timeout(std::time::Duration::from_secs(30), tail_barrier.wait_until_paused()) + .await + .expect("restore should pause a tail rename after quorum publication"); + let object_lock = other_store + .new_ns_lock(&bucket, object) + .await + .expect("create the shared-domain object guard probe"); + let mut object_probe = + tokio::spawn(async move { object_lock.get_write_lock(std::time::Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut object_probe) + .await + .is_err(), + "early-ACK tail must retain the decommission namespace guard" + ); + + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the decommission capacity guard probe"); + let mut capacity_probe = + tokio::spawn(async move { capacity_lock.get_write_lock(std::time::Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut capacity_probe) + .await + .is_err(), + "early-ACK tail must retain the decommission capacity guard" + ); + + tail_barrier.release(); + drop(tail_barrier); + let object_guard = tokio::time::timeout(std::time::Duration::from_secs(30), object_probe) + .await + .expect("object guard probe should finish after the tail") + .expect("object guard probe should join") + .expect("object guard probe should acquire after the tail"); + drop(object_guard); + let capacity_guard = tokio::time::timeout(std::time::Duration::from_secs(30), capacity_probe) + .await + .expect("capacity guard probe should finish after the tail") + .expect("capacity guard probe should join") + .expect("capacity guard probe should acquire after the tail"); + drop(capacity_guard); + tokio::time::timeout(std::time::Duration::from_secs(30), &mut ordinary_mutation) + .await + .expect("restore should return its early quorum acknowledgement") + .expect("restore task should join") + } else { + tokio::time::timeout(std::time::Duration::from_secs(30), ordinary_mutation) + .await + .expect("ordinary object mutation must finish after the migration releases its object fence") + .expect("ordinary object mutation task should join") + }; + drop(barrier); + ordinary_result.expect("ordinary object mutation should commit to the unreserved target"); + store.pools[1] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version), + ..Default::default() + }, + ) + .await + .expect("migration should preserve the source version on the reserved target"); + store.pools[2] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("ordinary object mutation should stay on the unreserved target"); + if matches!(mutation, ExternalObjectMutation::Restore) { + let mut restored = store.pools[2] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("restored target should be readable"); + let mut restored_body = Vec::new(); + restored + .stream + .read_to_end(&mut restored_body) + .await + .expect("restored target body should be readable"); + assert_eq!(restored_body, b"existing unreserved target"); + } + } + + async fn assert_same_object_historical_copy_lock_order() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = test_bucket("same-object-copy-order"); + let object = "same-object.bin"; + let migration_version = uuid::Uuid::new_v4().to_string(); + let historical_version = uuid::Uuid::new_v4().to_string(); + let current_version = uuid::Uuid::new_v4().to_string(); + let migration_body = b"decommission migration body".to_vec(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create same-object copy lock-order bucket"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("load same-object copy bucket incarnation"); + + let mut migration_source = PutObjReader::from_vec(migration_body.clone()); + store.pools[0] + .put_object( + &bucket, + object, + &mut migration_source, + &ObjectOptions { + versioned: true, + version_id: Some(migration_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the migration source version"); + + let mut historical_source = PutObjReader::from_vec(b"historical copy body".to_vec()); + store.pools[2] + .put_object( + &bucket, + object, + &mut historical_source, + &ObjectOptions { + versioned: true, + version_id: Some(historical_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the historical copy version"); + let mut current_source = PutObjReader::from_vec(b"current copy body".to_vec()); + store.pools[2] + .put_object( + &bucket, + object, + &mut current_source, + &ObjectOptions { + versioned: true, + version_id: Some(current_version.clone()), + ..Default::default() + }, + ) + .await + .expect("seed the current copy version"); + + let mut copy_info = store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + version_id: Some(historical_version.clone()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("historical copy version should be readable"); + assert_eq!( + store.pools[2] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("current copy version should be readable") + .version_id + .map(|version| version.to_string()), + Some(current_version.clone()) + ); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = migration_body.len() * 4; + let capacity_snapshot = |target_free| { + vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, migration_body.len(), migration_body.len()), + DecommissionPoolCapacityInfo::for_test( + 1, + layout, + target_free, + target_total, + target_total.saturating_sub(target_free), + ), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total), + ] + }; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot(target_total)]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the same-object copy source reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![ + capacity_snapshot(target_total), + capacity_snapshot(target_total), + capacity_snapshot(target_total - migration_body.len()), + ], + ); + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(migration_version), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read the migration source version"); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id); + let migration = tokio::spawn({ + let migration_store = Arc::clone(&store); + let migration_bucket = bucket.clone(); + async move { + data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + Some(incarnation), + "same_object_copy_lock_order", + Some(owner), + ) + .await + } + }); + barrier.wait_until_owner_paused().await; + + let copy_task = tokio::spawn({ + let copy_store = Arc::clone(&other_store); + let copy_bucket = bucket.clone(); + let source_opts = ObjectOptions { + versioned: true, + version_id: Some(historical_version), + ..Default::default() + }; + let destination_opts = ObjectOptions { + expected_current_version_id: Some(current_version), + versioned: true, + ..Default::default() + }; + async move { + copy_store + .copy_object( + ©_bucket, + object, + ©_bucket, + object, + &mut copy_info, + &source_opts, + &destination_opts, + ) + .await + .map(|_| ()) + } + }); + barrier.wait_until_external_capacity_released().await; + assert!( + !copy_task.is_finished(), + "historical same-object CopyObject must wait behind the migration object fence" + ); + barrier.release_owner(); + drop(barrier); + + tokio::time::timeout(std::time::Duration::from_secs(30), migration) + .await + .expect("migration must not deadlock with historical same-object CopyObject") + .expect("migration task should join") + .expect("migration should commit to its reserved target"); + tokio::time::timeout(std::time::Duration::from_secs(30), copy_task) + .await + .expect("historical same-object CopyObject must finish after the migration releases its object fence") + .expect("CopyObject task should join") + .expect("CopyObject should preserve the expected current-version precondition"); + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_unreserved_put_locks() { + assert_lock_order(ExternalObjectMutation::Put).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_multipart_complete_locks() { + assert_lock_order(ExternalObjectMutation::CompleteMultipart).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_cross_object_copy_locks() { + assert_lock_order(ExternalObjectMutation::Copy).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn public_cross_object_copy_holds_decommission_capacity_through_early_ack_tail() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_lock_order_with_tail(ExternalObjectMutation::Copy, true).await + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_same_object_historical_copy_locks() { + assert_same_object_historical_copy_lock_order().await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_put_object_tag_locks() { + assert_lock_order(ExternalObjectMutation::PutTags).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_delete_object_tag_locks() { + assert_lock_order(ExternalObjectMutation::DeleteTags).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_put_object_metadata_locks() { + assert_lock_order(ExternalObjectMutation::PutMetadata).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_delete_object_locks() { + assert_lock_order(ExternalObjectMutation::Delete).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_migration_does_not_invert_capacity_and_restore_locks() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + assert_lock_order_with_tail(ExternalObjectMutation::Restore, true).await + }) + .await; + } +} diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 9f8fe3dc3..371a3f326 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -1351,11 +1351,20 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc) pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( ctx: Arc, pool_idx: usize, +) -> (Vec, Arc) { + make_local_two_set_sets_for_pool_with_drive_count_and_ctx(ctx, pool_idx, 2).await +} + +#[cfg(any(test, feature = "test-util"))] +pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx( + ctx: Arc, + pool_idx: usize, + set_drive_count: usize, ) -> (Vec, Arc) { use crate::layout::endpoint::Endpoint; use rustfs_lock::client::local::LocalClient; - let format = FormatV3::new(2, 2); + let format = FormatV3::new(2, set_drive_count); let mut temp_dirs = Vec::new(); let mut all_endpoints = Vec::new(); let mut disk_sets = Vec::new(); @@ -1363,7 +1372,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( for set_index in 0..2 { let mut endpoints = Vec::new(); let mut disks = Vec::new(); - for disk_index in 0..2 { + for disk_index in 0..set_drive_count { let temp_dir = tempfile::tempdir().expect("tempdir should be created"); let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8")) .expect("endpoint should parse"); @@ -1389,7 +1398,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( endpoints.push(endpoint); disks.push(Some(disk)); } - let lockers = (0..2) + let lockers = (0..set_drive_count) .map(|_| { Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new( rustfs_lock::FastObjectLockManager::new(), @@ -1400,7 +1409,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( SetDisks::new_with_instance_ctx( "test-owner".to_string(), Arc::new(RwLock::new(disks)), - 2, + set_drive_count, 1, set_index, pool_idx, @@ -1420,7 +1429,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( endpoints: PoolEndpoints { legacy: false, set_count: 2, - drives_per_set: 2, + drives_per_set: set_drive_count, endpoints: Endpoints::from(all_endpoints), cmd_line: String::new(), platform: String::new(), @@ -1428,7 +1437,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx( format, parity_count: 1, set_count: 2, - set_drive_count: 2, + set_drive_count, default_parity_count: 1, distribution_algo: DistributionAlgoVersion::V1, exit_signal: None, diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 2a552c186..e1b2ee1ac 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -16,13 +16,14 @@ pub(crate) mod backpressure; +use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id}; use crate::error::{ Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found, }; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::storage_api_contracts::{ - multipart::{CompletePart, MultipartOperations as _}, + multipart::CompletePart, namespace::NamespaceLocking as _, object::{HTTPPreconditions, ObjectOperations as _}, }; @@ -160,6 +161,99 @@ pub fn mark_multipart_upload_completed(flag: &Arc) { flag.store(false, Ordering::Relaxed); } +#[cfg(test)] +struct DataMovementMultipartAbortBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct DataMovementMultipartAbortBarrier { + state: Arc, +} + +#[cfg(test)] +static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DataMovementMultipartAbortBarrier { + pub(crate) fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(DataMovementMultipartAbortBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart abort barrier mutex should not poison"); + assert!(slot.is_none(), "data movement multipart abort barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified()) + .await + .expect("data movement multipart failure should reach abort cleanup"); + } +} + +#[cfg(test)] +impl Drop for DataMovementMultipartAbortBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart abort barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) { + let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart abort barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + +fn data_movement_abort_opts( + src_pool_idx: usize, + expected_bucket_incarnation_id: Option, + lock_lost_signal: Option<&Arc>, + capacity_owner: Option, +) -> ObjectOptions { + let mut opts = ObjectOptions { + data_movement: true, + src_pool_idx, + expected_bucket_incarnation_id, + ..Default::default() + }; + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut opts); + } + if let Some(signal) = lock_lost_signal { + opts.add_namespace_lock_lost_signal(Arc::clone(signal)); + } + opts +} + fn insert_data_movement_checksum(user_defined: &mut HashMap, object_info: &ObjectInfo) { rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC); if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) { @@ -192,7 +286,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz preserve_etag: object_info.etag.clone(), src_pool_idx, data_movement: true, - ..Default::default() + ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok()) } } @@ -363,7 +457,7 @@ fn data_movement_complete_multipart_opts( preserve_etag: object_info.etag.clone(), user_defined, src_pool_idx, - ..Default::default() + ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok()) }) } @@ -533,6 +627,7 @@ fn schedule_data_movement_multipart_abort_cleanup( bucket: String, object: String, upload_id: String, + opts: ObjectOptions, op_label: &str, ) { let op_label = op_label.to_string(); @@ -540,23 +635,32 @@ fn schedule_data_movement_multipart_abort_cleanup( for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS { tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await; - let Some(pool) = store.pools.get(target_pool_idx).cloned() else { + if store.pools.get(target_pool_idx).is_none() { error!( "{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range" ); return; + } + + let mut cleanup_opts = opts.clone(); + let _multipart_mutation_fence = match DecommissionCapacityOwner::from_options(&cleanup_opts) { + Some(owner) => match store.acquire_decommission_multipart_mutation_fence(owner).await { + Ok(fence) => { + fence.add_namespace_lock_fence(&mut cleanup_opts); + Some(fence) + } + Err(err) => { + error!( + "{op_label}: background abort_multipart_upload cleanup could not fence {bucket}/{object} upload {upload_id} on attempt {attempt}: {err:?}" + ); + continue; + } + }, + None => None, }; - match pool - .abort_multipart_upload( - &bucket, - &object, - &upload_id, - &ObjectOptions { - data_movement: true, - ..Default::default() - }, - ) + match store + .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object, &upload_id, &cleanup_opts) .await { Ok(()) => { @@ -1334,27 +1438,43 @@ fn resolve_data_movement_overwrite_resume_result_for( Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target)) } +#[derive(Clone, Copy)] +struct DataMovementOverwriteCapacity { + owner: Option, + expected_data_bytes: Option, +} + async fn should_treat_data_movement_overwrite_as_complete( store: &ECStore, - src_pool_idx: usize, - target_pool_idx: usize, + pool_indices: (usize, usize), bucket: &str, object_info: &ObjectInfo, err: &Error, compare_part_checksums: bool, + capacity: DataMovementOverwriteCapacity, ) -> Result { if !should_check_data_movement_overwrite_resume(err) { return Ok(false); } + let (src_pool_idx, target_pool_idx) = pool_indices; - resolve_data_movement_overwrite_resume_result_for( + let equivalent = resolve_data_movement_overwrite_resume_result_for( err, find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await, object_info, src_pool_idx, target_pool_idx, compare_part_checksums, - ) + )?; + if equivalent && let Some(owner) = capacity.owner { + let expected_data_bytes = capacity + .expected_data_bytes + .ok_or_else(|| Error::other("equivalent data-movement target cannot reconcile unknown committed data size"))?; + store + .reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes) + .await?; + } + Ok(equivalent) } fn data_movement_part_stage_error( @@ -1395,6 +1515,7 @@ pub(crate) async fn migrate_decommission_object( rd: GetObjectReader, source_bucket_incarnation_id: Option, op_label: &str, + capacity_owner: Option, ) -> Result<()> { let source = rd.object_info.clone(); let _mutation_fence = store @@ -1415,6 +1536,7 @@ pub(crate) async fn migrate_decommission_object( source_bucket_incarnation_id, op_label, None, + capacity_owner, Some(&_mutation_fence), ) .await @@ -1451,6 +1573,7 @@ pub(crate) async fn migrate_object_with_lock_lost_signal( op_label, lock_lost_signal, None, + None, ) .await } @@ -1464,22 +1587,98 @@ async fn migrate_object_inner( source_bucket_incarnation_id: Option, op_label: &str, lock_lost_signal: Option>, + capacity_owner: Option, mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<()> { let object_info = rd.object_info.clone(); + let capacity_owner = capacity_owner.map(|owner| { + let version_id = object_info.version_id.map(|version_id| version_id.to_string()); + let mutation_id = owner.mutation_id.unwrap_or_else(|| { + decommission_capacity_mutation_id( + owner, + &bucket, + &object_info.name, + version_id.as_deref(), + object_info.delete_marker, + object_info.mod_time, + ) + }); + owner.with_mutation_id(mutation_id) + }); let has_part_checksums = object_info .parts .iter() .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty())); let preserve_part_checksums = data_movement_part_checksum_writer_enabled(); + let capacity_expected_data_bytes = usize::try_from(object_info.size).ok(); if should_use_multipart_data_movement(&object_info, has_part_checksums) { + let multipart_mutation_fence = match capacity_owner { + Some(owner) => Some(store.acquire_decommission_multipart_mutation_fence(owner).await?), + None => None, + }; let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx); + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut new_multipart_opts); + } new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; if let Some(signal) = lock_lost_signal.as_ref() { new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal)); } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut new_multipart_opts); + } + if let Some(owner) = capacity_owner { + let existing_target_pool_idx = store + .select_data_movement_pool_idx(&bucket, &object_info.name, -1, &new_multipart_opts, false) + .await?; + if existing_target_pool_idx != pool_idx + && let Some(target) = + find_data_movement_target_info(store.as_ref(), existing_target_pool_idx, &bucket, &object_info).await? + && is_equivalent_data_movement_object_identity(&object_info, &target, true, preserve_part_checksums) + { + let expected_data_bytes = capacity_expected_data_bytes + .ok_or_else(|| Error::other("equivalent multipart target cannot reconcile unknown committed data size"))?; + store + .reconcile_decommission_capacity_after_equivalent_target(owner, existing_target_pool_idx, expected_data_bytes) + .await?; + info!( + "{op_label}: multipart upload restart reconciled equivalent target for {}/{}", + bucket.as_str(), + object_info.name.as_str() + ); + return Ok(()); + } + let mut cleanup_opts = + data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + if let Some(fence) = mutation_fence { + fence.add_namespace_lock_fence(&mut cleanup_opts); + } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut cleanup_opts); + } + for target_pool_idx in store.decommission_capacity_cleanup_target_indices(owner).await? { + store + .reconcile_multipart_uploads_for_data_movement( + target_pool_idx, + &bucket, + &object_info.name, + &data_movement_upload_identity(&object_info), + &cleanup_opts, + ) + .await + .map_err(|err| { + data_movement_stage_error( + op_label, + "reconcile_multipart_upload", + bucket.as_str(), + object_info.name.as_str(), + err, + ) + })?; + } + } let (res, target_pool_idx, expected_bucket_incarnation_id) = match store .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence) .await @@ -1532,9 +1731,15 @@ async fn migrate_object_inner( expected_bucket_incarnation_id, ..Default::default() }; + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut part_opts); + } if let Some(signal) = lock_lost_signal.as_ref() { part_opts.add_namespace_lock_lost_signal(Arc::clone(signal)); } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut part_opts); + } let pi = match store .put_object_part_for_data_movement( target_pool_idx, @@ -1578,10 +1783,16 @@ async fn migrate_object_inner( err, ) })?; + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut complete_multipart_opts); + } complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; if let Some(signal) = lock_lost_signal.as_ref() { complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal)); } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut complete_multipart_opts); + } if let Err(err) = store .clone() .complete_multipart_upload_for_data_movement( @@ -1596,12 +1807,15 @@ async fn migrate_object_inner( { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), - pool_idx, - target_pool_idx, + (pool_idx, target_pool_idx), bucket.as_str(), &object_info, &err, preserve_part_checksums, + DataMovementOverwriteCapacity { + owner: capacity_owner, + expected_data_bytes: capacity_expected_data_bytes, + }, ) .await? { @@ -1629,31 +1843,37 @@ async fn migrate_object_inner( .await; if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) { + let mut abort_opts = + data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + if let Some(fence) = mutation_fence { + fence.add_namespace_lock_fence(&mut abort_opts); + } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut abort_opts); + } let abort_result = store - .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{ - let mut opts = ObjectOptions { - data_movement: true, - src_pool_idx: pool_idx, - expected_bucket_incarnation_id, - ..Default::default() - }; - if let Some(signal) = lock_lost_signal.as_ref() { - opts.add_namespace_lock_lost_signal(Arc::clone(signal)); - } - opts - }) + .abort_multipart_upload_for_data_movement( + target_pool_idx, + &bucket, + &object_info.name, + &res.upload_id, + &abort_opts, + ) .await; match abort_result { Ok(()) => return Ok(()), Err(abort_err) if is_err_invalid_upload_id(&abort_err) => { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), - pool_idx, - target_pool_idx, + (pool_idx, target_pool_idx), bucket.as_str(), &object_info, &abort_err, preserve_part_checksums, + DataMovementOverwriteCapacity { + owner: capacity_owner, + expected_data_bytes: capacity_expected_data_bytes, + }, ) .await? { @@ -1683,6 +1903,7 @@ async fn migrate_object_inner( bucket.clone(), object_info.name.clone(), res.upload_id.clone(), + abort_opts, op_label, ); return Err(data_movement_stage_error( @@ -1698,19 +1919,24 @@ async fn migrate_object_inner( if let Err(primary_err) = multipart_result { if should_abort_multipart_upload(&abort_multipart_flag) { + #[cfg(test)] + pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await; + let mut abort_opts = + data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner); + if let Some(fence) = mutation_fence { + fence.add_namespace_lock_fence(&mut abort_opts); + } + if let Some(fence) = multipart_mutation_fence.as_ref() { + fence.add_namespace_lock_fence(&mut abort_opts); + } return match store - .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{ - let mut opts = ObjectOptions { - data_movement: true, - src_pool_idx: pool_idx, - expected_bucket_incarnation_id, - ..Default::default() - }; - if let Some(signal) = lock_lost_signal.as_ref() { - opts.add_namespace_lock_lost_signal(Arc::clone(signal)); - } - opts - }) + .abort_multipart_upload_for_data_movement( + target_pool_idx, + &bucket, + &object_info.name, + &res.upload_id, + &abort_opts, + ) .await { Ok(()) => Err(primary_err), @@ -1722,6 +1948,7 @@ async fn migrate_object_inner( bucket.clone(), object_info.name.clone(), res.upload_id.clone(), + abort_opts, op_label, ); Err(resolve_data_movement_abort_result( @@ -1744,6 +1971,9 @@ async fn migrate_object_inner( let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?; let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx); + if let Some(capacity_owner) = capacity_owner { + capacity_owner.apply_to(&mut put_opts); + } put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; if let Some(signal) = lock_lost_signal { put_opts.add_namespace_lock_lost_signal(signal); @@ -1755,12 +1985,15 @@ async fn migrate_object_inner( if let Err(err) = put_result { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), - pool_idx, - target_pool_idx, + (pool_idx, target_pool_idx), bucket.as_str(), &object_info, &err, preserve_part_checksums, + DataMovementOverwriteCapacity { + owner: capacity_owner, + expected_data_bytes: capacity_expected_data_bytes, + }, ) .await? { diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index c6002250c..3f5035634 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -183,6 +183,10 @@ pub enum StorageError { DecommissionNotStarted, #[error("Decommission already running")] DecommissionAlreadyRunning, + #[error("Decommission capacity error: {0}")] + DecommissionCapacity(String), + #[error("decommission_capacity_blocked: Storage reached its minimum free drive threshold.: {message}")] + DecommissionCapacityBlocked { message: String }, #[error("Rebalance already running")] RebalanceAlreadyRunning, #[error("{operation}: stale pool metadata update rejected for pool {pool_index}; {reason}")] @@ -569,6 +573,10 @@ impl Clone for StorageError { StorageError::EntityTooLarge(a, b) => StorageError::EntityTooLarge(*a, *b), StorageError::DoneForNow => StorageError::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning, + StorageError::DecommissionCapacity(message) => StorageError::DecommissionCapacity(message.clone()), + StorageError::DecommissionCapacityBlocked { message } => StorageError::DecommissionCapacityBlocked { + message: message.clone(), + }, StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning, StorageError::StalePoolMetadataUpdate { operation, @@ -681,6 +689,8 @@ impl StorageError { StorageError::InvalidPart(_, _, _) => StorageErrorCode::InvalidPart, StorageError::DoneForNow => StorageErrorCode::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning, + StorageError::DecommissionCapacity(_) => StorageErrorCode::InvalidArgument, + StorageError::DecommissionCapacityBlocked { .. } => StorageErrorCode::StorageFull, StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning, StorageError::StalePoolMetadataUpdate { .. } => StorageErrorCode::InvalidArgument, StorageError::OperationCanceled => StorageErrorCode::OperationCanceled, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 694fbf0c0..88b648b49 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -681,6 +681,16 @@ impl Drop for ScannerPublicationCommitScopeInner { } } +#[derive(Clone, Default)] +#[doc(hidden)] +pub struct DecommissionCapacityOptions { + pub(crate) expected_data_bytes: Option, + pub(crate) operation_id: Option, + pub(crate) generation: Option, + pub(crate) owner_nonce: Option, + pub(crate) mutation_id: Option, +} + #[derive(Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -725,6 +735,12 @@ pub struct ObjectOptions { pub data_movement: bool, pub raw_data_movement_read: bool, + /// Durable reservation identity carried only by decommission writes. Other + /// data-movement users, including rebalance, leave it unset. Keep this + /// context boxed because `ObjectOptions` is passed by value through deep + /// storage futures. + #[doc(hidden)] + pub decommission_capacity: Option>, /// Materialize the data-movement per-part checksum sidecar for APIs that /// return part checksums. Ordinary object reads leave it encoded. pub include_part_checksums: bool, @@ -788,6 +804,36 @@ pub struct ObjectOptions { /// Storage-owned journal writer used by the atomic delete path. This is /// populated only by the `ECStore` wrapper that holds the namespace locks. pub tier_delete_journal_api: Option>, + /// Internal staged-mutation admission supplied by `ECStore`; each local + /// publish is fenced namespace-first and then by decommission capacity. + #[doc(hidden)] + pub decommission_capacity_admission: Option>, +} + +impl ObjectOptions { + pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option) -> Self { + Self { + decommission_capacity: expected_data_bytes.map(|expected_data_bytes| { + Box::new(DecommissionCapacityOptions { + expected_data_bytes: Some(expected_data_bytes), + ..Default::default() + }) + }), + ..Default::default() + } + } + + pub(crate) fn capacity_expected_data_bytes(&self) -> Option { + self.decommission_capacity + .as_deref() + .and_then(|capacity| capacity.expected_data_bytes) + } + + pub(crate) fn has_decommission_capacity_reservation(&self) -> bool { + self.decommission_capacity + .as_deref() + .is_some_and(|capacity| capacity.operation_id.is_some()) + } } impl std::fmt::Debug for ObjectOptions { diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index d9939702d..b351ebbdb 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -1336,7 +1336,11 @@ mod tests { use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated}; async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc) { - let mut write_state = PoolMetaWriteState::for_startup(store.id, false); + let deployment_id = store + .ctx + .deployment_id() + .expect("test store should have a deployment identity"); + let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false); persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true) .await .expect("initialized pool metadata identity should persist"); diff --git a/crates/ecstore/src/services/rebalance/mod.rs b/crates/ecstore/src/services/rebalance/mod.rs index e36699500..352c1138e 100644 --- a/crates/ecstore/src/services/rebalance/mod.rs +++ b/crates/ecstore/src/services/rebalance/mod.rs @@ -97,7 +97,7 @@ pub(crate) async fn test_two_pool_stores( std::sync::Arc, std::sync::Arc, ) { - test_two_pool_stores_with_contexts(rebalance_meta, false).await + test_pool_stores_with_contexts(rebalance_meta, false, 2, 2).await } #[cfg(test)] @@ -108,7 +108,7 @@ pub(crate) async fn test_two_pool_stores_with_isolated_node_contexts( std::sync::Arc, std::sync::Arc, ) { - test_two_pool_stores_with_contexts(rebalance_meta, true).await + test_pool_stores_with_contexts(rebalance_meta, true, 2, 2).await } #[cfg(test)] @@ -123,26 +123,58 @@ pub(crate) async fn promote_test_pool_meta_to_v2(store: &std::sync::Arc, +) -> ( + Vec, + std::sync::Arc, + std::sync::Arc, +) { + test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await +} + +#[cfg(test)] +pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts( + rebalance_meta: Option, +) -> ( + Vec, + std::sync::Arc, + std::sync::Arc, +) { + test_pool_stores_with_contexts(rebalance_meta, true, 3, 3).await +} + +#[cfg(test)] +async fn test_pool_stores_with_contexts( rebalance_meta: Option, isolate_node_contexts: bool, + pool_count: usize, + set_drive_count: usize, ) -> ( Vec, std::sync::Arc, std::sync::Arc, ) { crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test(); - use crate::core::pools::PoolMeta; + use crate::core::pools::{POOL_META_VERSION, PoolMeta, PoolMetaWriteState, persist_pool_meta_identity_for_startup}; use crate::layout::endpoints::{EndpointServerPools, SetupType}; let ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new()); ctx.update_erasure_type(SetupType::DistErasure).await; - let (mut temp_dirs, first_pool) = - crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 0).await; - let (second_temp_dirs, second_pool) = - crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 1).await; - temp_dirs.extend(second_temp_dirs); - let pools = vec![first_pool, second_pool]; + let deployment_id = uuid::Uuid::new_v4(); + ctx.set_deployment_id(deployment_id); + let mut temp_dirs = Vec::new(); + let mut pools = Vec::with_capacity(pool_count); + for pool_index in 0..pool_count { + let (pool_temp_dirs, pool) = crate::core::sets::make_local_two_set_sets_for_pool_with_drive_count_and_ctx( + std::sync::Arc::clone(&ctx), + pool_index, + set_drive_count, + ) + .await; + temp_dirs.extend(pool_temp_dirs); + pools.push(pool); + } { let local_disk_map = ctx.local_disk_map(); let mut local_disk_map = local_disk_map.write().await; @@ -154,11 +186,24 @@ async fn test_two_pool_stores_with_contexts( } } } - let pool_meta = PoolMeta::new(&pools, &PoolMeta::default()); + let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default()); + pool_meta.version = POOL_META_VERSION; pool_meta .save_for_startup(pools.clone()) .await .expect("baseline pool metadata should be persisted"); + let mut pool_meta_write_state = PoolMetaWriteState::for_startup(deployment_id, true); + persist_pool_meta_identity_for_startup(pools.clone(), &mut pool_meta_write_state, false) + .await + .expect("pending pool metadata identity should be persisted"); + let replica_state = pool_meta + .load_no_lock_from_replicas_observing(pools.clone(), &mut pool_meta_write_state) + .await + .expect("baseline pool metadata should remain readable"); + pool_meta_write_state.observe_replicas(replica_state); + persist_pool_meta_identity_for_startup(pools.clone(), &mut pool_meta_write_state, true) + .await + .expect("initialized pool metadata identity should be persisted"); if let Some(meta) = rebalance_meta.as_ref() { meta.save(pools[0].clone()) .await @@ -169,6 +214,7 @@ async fn test_two_pool_stores_with_contexts( let other_ctx = if isolate_node_contexts { let other_ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new()); other_ctx.update_erasure_type(SetupType::DistErasure).await; + other_ctx.set_deployment_id(deployment_id); *other_ctx.local_disk_map().write().await = ctx.local_disk_map().read().await.clone(); other_ctx.set_endpoints(endpoint_pools.clone()); other_ctx @@ -183,9 +229,9 @@ async fn test_two_pool_stores_with_contexts( peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)), pool_meta: tokio::sync::RwLock::new(pool_meta.clone()), rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()), - decommission_cancelers: tokio::sync::RwLock::new(vec![None, None]), + decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]), start_gate: tokio::sync::Mutex::new(()), - pool_meta_save_gate: tokio::sync::Mutex::default(), + pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()), ctx: store_ctx, bucket_fence_registry: std::sync::Arc::default(), }) @@ -195,6 +241,8 @@ async fn test_two_pool_stores_with_contexts( if isolate_node_contexts { crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&store), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&other_store), Vec::new()).await; + } else { + crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&store), Vec::new()).await; } (temp_dirs, store, other_store) } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index c79bd6868..2c8064c0e 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -475,7 +475,7 @@ const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS"; const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200; const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total"; -pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"; +pub(crate) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"; /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215, /// #1159, #923). /// @@ -6849,7 +6849,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { /// Cross-process/black-box fault injection (toxiproxy, blackhole peers, 2-pool) /// is a later cluster-harness block, not this one. #[cfg(test)] -pub(in crate::set_disk) mod rename_fanout_barrier { +pub(crate) mod rename_fanout_barrier { use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 4fac5362a..b7794d3f3 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -857,7 +857,7 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock = OnceLock::new(); mod core; #[cfg(test)] -pub(crate) use core::io_primitives::disk_call_counters; +pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier}; mod ctx; mod metadata; mod ops; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 6a9545199..20a1911f2 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -27,18 +27,18 @@ use super::super::MetadataCacheInvalidationProbe; #[cfg(test)] use super::super::capacity_scope_from_disks; use super::super::{ - AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DiskError, DiskStore, EVENT_SET_DISK_MULTIPART, Error, FileInfo, - GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE, - LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART, - MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult, - MultipartWriteQuorumContext, NamespaceLockFence, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions, - ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, - RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, - SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, - SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, Uuid, WriteLayout, - check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding, complete_multipart_part_error, - complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part, contains_key_str, - create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str, + AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore, + EVENT_SET_DISK_MULTIPART, Error, FileInfo, GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, + HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, + MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART, MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, + MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult, MultipartWriteQuorumContext, NamespaceLockFence, + OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo, + PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, + Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, + SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, + Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding, + complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part, + contains_key_str, create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, @@ -183,6 +183,45 @@ pub(crate) struct StaleMultipartCleanupGuard { lock_guard: ObjectLockDiagGuard, } +pub(crate) struct DataMovementMultipartAbortGuard { + upload_path: String, + write_quorum: Option, + lock_guard: ObjectLockDiagGuard, +} + +impl DataMovementMultipartAbortGuard { + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + opts.add_namespace_lock_guard(&self.lock_guard.guard); + } + + pub(crate) async fn delete(&self, set: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(bucket, object, MultipartCommitPause::AbortBeforeDelete).await; + fence_commit_on_lock_loss(Some(&self.lock_guard), "abort_multipart_upload_commit", &self.upload_path)?; + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "abort_multipart_upload_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; + if let Some(write_quorum) = self.write_quorum { + set.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &self.upload_path, write_quorum) + .await?; + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(bucket, object, MultipartCommitPause::AbortAfterDelete).await; + } + Ok(()) + } +} + impl StaleMultipartCleanupGuard { pub(crate) fn file_info(&self) -> &FileInfo { &self.file_info @@ -210,7 +249,10 @@ pub enum MultipartCommitPause { NewUploadBeforeLockLost, PutPartBeforeLockAcquire, PutPartBeforeLockLost, + PutPartAfterCapacityAdmission, PutPartAfterRename, + AbortBeforeDelete, + AbortAfterDelete, BeforeLockLost, BeforeQuotaRename, BeforeTransactionEpochVerify, @@ -673,12 +715,28 @@ fn is_corrupt_upload_metadata_error(err: &DiskError) -> bool { ) } -async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result> { +async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_prefix: &str) -> disk::error::Result> { if !disk.is_online().await { return Err(DiskError::DiskNotFound); } - let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, "", -1).await { + if !root_prefix.is_empty() { + let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()), + Err(err) => return Err(err), + }; + return Ok(upload_dirs + .into_iter() + .filter_map(|upload_dir| { + let upload_dir = upload_dir.trim_end_matches('/'); + (!upload_dir.is_empty() && upload_dir != "." && upload_dir != ".." && !upload_dir.contains(['/', '\\'])) + .then(|| format!("{root_prefix}/{upload_dir}")) + }) + .collect()); + } + + let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await { Ok(entries) => entries, Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()), Err(err) => return Err(err), @@ -778,6 +836,7 @@ impl SetDisks { &self, orig_bucket: &str, error_path: &str, + root_prefix: &str, ) -> Result<(Vec>, Vec, usize)> { let disks = self.disks.read().await.clone(); if disks.is_empty() { @@ -794,9 +853,10 @@ impl SetDisks { for (index, disk) in disks.iter().enumerate() { let disk = disk.clone(); let orig_bucket = orig_bucket.to_string(); + let root_prefix = root_prefix.to_string(); discovery_tasks.spawn(async move { let result = match disk { - Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket).await, + Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket, &root_prefix).await, None => Err(DiskError::DiskNotFound), }; (index, result) @@ -832,11 +892,64 @@ impl SetDisks { pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result> { let (_, paths, _) = self - .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET) + .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "") .await?; Ok(paths.into_iter().next()) } + pub(crate) async fn data_movement_multipart_upload_ids( + &self, + bucket: &str, + object: &str, + expected_incarnation_id: Option, + upload_identity: &str, + ) -> Result> { + let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object)); + let (_, candidate_paths, _) = self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?; + let mut upload_ids = Vec::new(); + for upload_path in candidate_paths { + let Some((parent, raw_upload_id)) = upload_path.rsplit_once('/') else { + continue; + }; + if parent != expected_parent || raw_upload_id.is_empty() { + continue; + } + let upload_id = runtime_sources::deployment_upload_id(raw_upload_id); + let file_info = match self + .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false) + .await + { + Ok((file_info, _)) => file_info, + Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => { + continue; + } + Err(err) => return Err(err), + }; + if file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).map(String::as_str) != Some(bucket) + || file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).map(String::as_str) != Some(object) + { + return Err(Error::other("data movement multipart upload target metadata is inconsistent")); + } + if expected_incarnation_id + .is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected)) + { + return Err(Error::other("data movement multipart upload bucket incarnation is inconsistent")); + } + let Some(actual_upload_identity) = + rustfs_utils::http::get_consistent_str(&file_info.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) + else { + return Err(Error::other("data movement multipart upload identity is inconsistent")); + }; + if actual_upload_identity != upload_identity { + continue; + } + upload_ids.push(upload_id); + } + upload_ids.sort_unstable(); + upload_ids.dedup(); + Ok(upload_ids) + } + async fn acquire_multipart_upload_read_lock( &self, op: &'static str, @@ -873,6 +986,55 @@ impl SetDisks { .map(Some) } + pub(crate) async fn lock_data_movement_multipart_abort( + &self, + bucket: &str, + object: &str, + upload_id: &str, + expected_upload_identity: Option<&str>, + opts: &ObjectOptions, + ) -> Result> { + let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_id, true); + let lock_guard = self + .acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path) + .await?; + let file_info = match self + .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true) + .await + { + Ok((file_info, _)) => file_info, + Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => { + return Ok(Some(DataMovementMultipartAbortGuard { + upload_path, + write_quorum: None, + lock_guard, + })); + } + Err(err) => return Err(err), + }; + ensure_data_movement_upload_access(&file_info, bucket, object, upload_id, opts)?; + let upload_identity = + rustfs_utils::http::get_consistent_str(&file_info.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD); + if upload_identity.is_none() || expected_upload_identity.is_some_and(|expected| upload_identity != Some(expected)) { + return Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())); + } + ensure_multipart_bucket_incarnation( + &self.ctx, + &file_info, + bucket, + object, + upload_id, + opts.expected_bucket_incarnation_id, + ) + .await?; + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; + Ok(Some(DataMovementMultipartAbortGuard { + upload_path, + write_quorum: Some(file_info.write_quorum(self.default_write_quorum())), + lock_guard, + })) + } + pub(super) async fn list_parts( disks: &[Option], part_path: &str, @@ -1028,7 +1190,7 @@ impl SetDisks { max_uploads: usize, expected_incarnation_id: Option, ) -> Result { - let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix).await?; + let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix, "").await?; let listed_uploads = stream::iter(candidate_paths) .map(|upload_path| { let disks = &disks; @@ -1624,7 +1786,27 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { admitted_multipart_size(current_size, candidate_size, limit)?; } - let _ = self + let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() { + Some( + store + .acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation") + .await?, + ) + } else { + None + }; + #[cfg(test)] + pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await; + if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "put_object_part_decommission_capacity", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + let rename_result = self .rename_part( &shuffle_disks, RUSTFS_META_TMP_BUCKET, @@ -1641,7 +1823,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { part_number: Some(part_id), }), ) - .await?; + .await; + drop(decommission_capacity_guard); + let _ = rename_result?; #[cfg(test)] observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost); @@ -2112,6 +2296,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock; let mut object_lock_guard = None; + let mut decommission_object_lock_guard = None; + let mut decommission_target_lock_covered = false; + let mut decommission_capacity_guard = None; if opts.http_preconditions.is_some() { if !opts.no_lock { @@ -2126,7 +2313,25 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } - if !opts.no_lock && object_lock_guard.is_none() { + if let Some(store) = opts.decommission_capacity_admission.as_ref() { + #[cfg(test)] + { + crate::core::pools::notify_decommission_external_object_commit_phase_started(store.id); + crate::core::pools::wait_for_decommission_external_object_commit_phase_release(store.id).await; + } + let (object_guard, target_lock_covered, capacity_guard) = store + .acquire_external_decommission_commit_guards( + self.pool_index, + bucket, + object, + opts.no_lock || object_lock_guard.is_some(), + ) + .await?; + decommission_object_lock_guard = object_guard; + decommission_target_lock_covered = target_lock_covered; + decommission_capacity_guard = capacity_guard; + } + if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered { object_lock_guard = Some( self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object) .await?, @@ -2728,7 +2933,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // so a lost lock leaves the upload intact and retryable. #[cfg(test)] pause_multipart_commit(bucket, object, MultipartCommitPause::BeforeLockLost).await; - if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) + { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "complete_multipart_upload_commit", bucket: bucket.to_string(), @@ -2805,7 +3014,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { Err(err) => return Err(err), } } - if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) + { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "complete_multipart_upload_commit", bucket: bucket.to_string(), @@ -2838,6 +3051,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; + // Complete has already acquired the object and upload-id namespaces. + // Recheck decommission capacity only after those locks, and retain the + // guard through the durable rename below. + if decommission_capacity_guard.is_none() + && let Some(store) = opts.decommission_capacity_admission.as_ref() + { + decommission_capacity_guard = Some( + store + .acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation") + .await?, + ); + } + let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { return Err(Error::other("object transaction fencing requires a live fleet capability proof")); @@ -2896,11 +3122,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let commit_is_versioned = opts.versioned || opts.version_suspended; let commit_capacity_scope_token = opts.capacity_scope_token; let commit_object_lock_guard = object_lock_guard.take(); - let commit_allows_early_ack = commit_object_lock_guard.is_some(); + let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); + let commit_decommission_capacity_guard = decommission_capacity_guard.take(); + let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) + && (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some()); let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence; let commit = async move { let mut _object_lock_guard = commit_object_lock_guard; + let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; let mut _upload_guard = upload_guard; + let mut _decommission_capacity_guard = commit_decommission_capacity_guard; let mut quota_reservation = quota_reservation; let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); @@ -2919,6 +3150,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || commit_namespace_lock_fence .as_ref() @@ -2926,6 +3160,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { || commit_bucket_lifecycle_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) + || _decommission_capacity_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -2967,6 +3204,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || commit_namespace_lock_fence .as_ref() @@ -2974,6 +3214,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { || commit_bucket_lifecycle_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) + || _decommission_capacity_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -3037,6 +3280,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .map(|version_id| version_id.to_string()); let object_lock_guard = _object_lock_guard.take(); let upload_guard = _upload_guard.take(); + let decommission_object_lock_guard = _decommission_object_lock_guard.take(); + let decommission_capacity_guard = _decommission_capacity_guard.take(); let cleanup_bucket = commit_bucket.clone(); let cleanup_object = commit_object.clone(); let heal_set = commit_set.clone(); @@ -3054,7 +3299,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { tokio::spawn(finish_rename_tail_heal( rename_tail_drain, guard_release_rx, - (object_lock_guard, upload_guard), + ( + object_lock_guard, + upload_guard, + decommission_object_lock_guard, + decommission_capacity_guard, + ), request, move || async move { if quota_mutation_fence { @@ -3068,7 +3318,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .await; } }, - move |(object_lock_guard, upload_guard), targets| async move { + move |(object_lock_guard, upload_guard, decommission_object_lock_guard, decommission_capacity_guard), + targets| async move { drop(object_lock_guard); cleanup_set.cleanup_multipart_path(&cleanup_parts).await; cleanup_set @@ -3093,11 +3344,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ); } drop(upload_guard); + drop(decommission_object_lock_guard); + drop(decommission_capacity_guard); }, |request| async move { heal_set.submit_rename_tail_heal(request).await }, )); } } + if !tail_owns_staging_cleanup { + drop(_decommission_capacity_guard.take()); + } if quota_mutation_fence && !tail_owns_staging_cleanup { let _ = SetDisks::release_quota_mutation_fences( &commit_disks, diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 0e2f27178..6a680431b 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -2760,6 +2760,9 @@ impl SetDisks { let disks = self.get_disks_internal().await; let mut object_lock_guard = None; + let mut decommission_object_lock_guard = None; + let mut decommission_target_lock_covered = false; + let mut decommission_capacity_guard = None; let mut bucket_lifecycle_guard = None; // This pre-body check is advisory fast-fail only: the authoritative @@ -3212,7 +3215,25 @@ impl SetDisks { .await?; } - if !opts.no_lock && object_lock_guard.is_none() { + if let Some(store) = opts.decommission_capacity_admission.as_ref() { + #[cfg(test)] + { + crate::core::pools::notify_decommission_external_object_commit_phase_started(store.id); + crate::core::pools::wait_for_decommission_external_object_commit_phase_release(store.id).await; + } + let (object_guard, target_lock_covered, capacity_guard) = store + .acquire_external_decommission_commit_guards( + self.pool_index, + bucket, + object, + opts.no_lock || object_lock_guard.is_some(), + ) + .await?; + decommission_object_lock_guard = object_guard; + decommission_target_lock_covered = target_lock_covered; + decommission_capacity_guard = capacity_guard; + } + if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered { #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id @@ -3389,6 +3410,33 @@ impl SetDisks { }); } + if decommission_capacity_guard.is_none() + && let Some(store) = opts.decommission_capacity_admission.as_ref() + { + decommission_capacity_guard = Some( + store + .acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation") + .await?, + ); + } + + // The object namespace is acquired above, after the input stream + // has been fully staged. Only then admit the local publication + // against the decommission capacity ledger; holding this guard + // through rename_data keeps the namespace -> capacity order. + if decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "put_object_external_namespace", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { return Err(Error::other("object transaction fencing requires a live fleet capability proof")); @@ -3523,13 +3571,17 @@ impl SetDisks { let commit_object = object.to_owned(); let commit_tmp_dir = tmp_dir.clone(); let commit_object_lock_guard = object_lock_guard.take(); + let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); + let commit_decommission_capacity_guard = decommission_capacity_guard.take(); let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone(); // A scanner publication scope owns the movement permit until the // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases // remote leases. - let commit_allows_early_ack = commit_object_lock_guard.is_some() && commit_scanner_publication_scope.is_none(); + let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) + && (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some()) + && commit_scanner_publication_scope.is_none(); let detach_commit_owner = commit_scanner_publication_scope.is_some() || commit_allows_early_ack || commit_bucket_lifecycle_guard.is_some() @@ -3549,7 +3601,9 @@ impl SetDisks { let commit = move |cancellation: Option| async move { let mut _object_lock_guard = commit_object_lock_guard; + let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard; + let mut _decommission_capacity_guard = commit_decommission_capacity_guard; let mut quota_reservation = quota_reservation; let rename_stage_start = Instant::now(); let pre_rename = async { @@ -3561,6 +3615,9 @@ impl SetDisks { if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) || commit_namespace_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) @@ -3568,6 +3625,9 @@ impl SetDisks { .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_capacity_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -3611,6 +3671,9 @@ impl SetDisks { if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) || commit_namespace_lock_fence .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) @@ -3618,6 +3681,9 @@ impl SetDisks { .as_ref() .is_some_and(NamespaceLockFence::is_lock_lost) || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _decommission_capacity_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -3727,6 +3793,8 @@ impl SetDisks { .map(|version_id| version_id.to_string()); let object_lock_guard = _object_lock_guard.take(); let bucket_lifecycle_guard = _bucket_lifecycle_guard.take(); + let decommission_object_lock_guard = _decommission_object_lock_guard.take(); + let decommission_capacity_guard = _decommission_capacity_guard.take(); let cleanup_bucket = commit_bucket.clone(); let cleanup_object = commit_object.clone(); let heal_set = commit_set.clone(); @@ -3741,7 +3809,12 @@ impl SetDisks { tokio::spawn(finish_rename_tail_heal( rename_tail_drain, guard_release_rx, - (object_lock_guard, bucket_lifecycle_guard), + ( + object_lock_guard, + bucket_lifecycle_guard, + decommission_object_lock_guard, + decommission_capacity_guard, + ), request, move || async move { if quota_mutation_fence { @@ -3755,7 +3828,13 @@ impl SetDisks { .await; } }, - move |(object_lock_guard, bucket_lifecycle_guard), targets| async move { + move |( + object_lock_guard, + bucket_lifecycle_guard, + decommission_object_lock_guard, + decommission_capacity_guard, + ), + targets| async move { drop(object_lock_guard); drop(bucket_lifecycle_guard); cleanup_set @@ -3776,11 +3855,16 @@ impl SetDisks { "issue3031_put_object_tmp_cleanup_done" ); } + drop(decommission_object_lock_guard); + drop(decommission_capacity_guard); }, |request| async move { heal_set.submit_rename_tail_heal(request).await }, )); } } + if !tail_owns_tmp_cleanup { + drop(_decommission_capacity_guard.take()); + } #[cfg(any(test, feature = "test-util"))] if rename_result.is_ok() { pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameHandoff).await; @@ -5687,6 +5771,7 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) { struct DeleteObjectCommitBarrierState { bucket: String, object: String, + pause_after_publish: bool, arrived: tokio::sync::Notify, release: tokio::sync::Notify, } @@ -5703,9 +5788,18 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock Self { + Self::install_with_mode(bucket, object, false) + } + + pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self { + Self::install_with_mode(bucket, object, true) + } + + fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self { let state = Arc::new(DeleteObjectCommitBarrierState { bucket: bucket.to_string(), object: object.to_string(), + pause_after_publish, arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), }); @@ -5750,7 +5844,22 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) { .lock() .expect("delete object commit barrier mutex should not poison") .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + +#[cfg(test)] +async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) { + let barrier = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish) .cloned(); if let Some(barrier) = barrier { barrier.arrived.notify_one(); @@ -7527,6 +7636,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts)) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; + #[cfg(test)] + pause_delete_object_commit_after_publish(bucket, object).await; let disks = self.disk_inventory().await; self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); @@ -7567,6 +7678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.delete_object_version(bucket, object, &dfi, opts.delete_marker) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; + #[cfg(test)] + pause_delete_object_commit_after_publish(bucket, object).await; let disks = self.disk_inventory().await; self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); @@ -8319,19 +8432,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { part_checksums.to_string(), ); } - // The restore copy-back re-writes this same object via put_object / - // new_multipart_upload / complete_multipart_upload, each of which takes - // the object write lock in its commit phase. The caller - // (handle_restore_transitioned_object, #4877) already holds that write - // lock for the whole restore and forwards no_lock=true, so the inner - // writes must inherit it or they self-deadlock on the lock we already - // hold and time out. put_restore_opts builds fresh options that default - // no_lock=false, so propagate it explicitly here. + // Keep the public ECStore capacity admission attached to each local + // commit. The tier reads below must remain outside the object write + // lock so HEAD/GET do not wait for a slow remote copy-back. ropts.no_lock = opts.no_lock; ropts.expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id; ropts.bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); ropts.namespace_lock_fence = opts.namespace_lock_fence.clone(); ropts.object_lock_config_snapshot = opts.object_lock_config_snapshot.clone(); + ropts.decommission_capacity_admission = opts.decommission_capacity_admission.clone(); if oi.parts.len() == 1 { let mut opts = opts.clone(); opts.part_number = Some(1); @@ -8462,25 +8571,19 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { .expect("multipart restore must contain at least one uploaded part") .etag = Some("injected-invalid-complete-etag".to_string()); } + let complete_opts = ObjectOptions { + mod_time: oi.mod_time, + version_id: oi.version_id.map(|version| version.to_string()), + expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id, + bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(), + user_defined: restore_commit_metadata, + no_lock: opts.no_lock, + decommission_capacity_admission: opts.decommission_capacity_admission.clone(), + ..Default::default() + }; self_ .clone() - .complete_multipart_upload( - bucket, - object, - &res.upload_id, - uploaded_parts, - &ObjectOptions { - mod_time: oi.mod_time, - version_id: oi.version_id.map(|version| version.to_string()), - expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id, - bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(), - user_defined: restore_commit_metadata, - // Inherit the restore write lock (see ropts.no_lock above): - // the commit phase re-acquires this object's write lock. - no_lock: opts.no_lock, - ..Default::default() - }, - ) + .complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &complete_opts) .await } .await; diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index c1c0eeaf7..0ba4ea2f0 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -157,7 +157,15 @@ impl SetDisks { .clone() .unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref())); let version_id = expected.version_id.map(|v| v.to_string()); - let lock_guard = if !opts.no_lock { + let (decommission_object_lock_guard, decommission_target_lock_covered, mut decommission_capacity_guard) = + if let Some(store) = opts.decommission_capacity_admission.as_ref() { + store + .acquire_external_decommission_commit_guards(self.pool_index, bucket, object, opts.no_lock) + .await? + } else { + (None, false, None) + }; + let lock_guard = if !opts.no_lock && !decommission_target_lock_covered { Some( self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object) .await?, @@ -165,6 +173,15 @@ impl SetDisks { } else { None }; + if decommission_capacity_guard.is_none() + && let Some(store) = opts.decommission_capacity_admission.as_ref() + { + decommission_capacity_guard = Some( + store + .acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation") + .await?, + ); + } let read_opts = ObjectOptions { version_id, versioned: opts.versioned, @@ -198,7 +215,12 @@ impl SetDisks { ); self.invalidate_get_object_metadata_cache(bucket, object).await; ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?; - if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) + || decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { return Err(Error::other("restore finalization lock lost before metadata update")); } self.update_object_meta_with_opts( @@ -233,7 +255,15 @@ impl SetDisks { .clone() .unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref())); let version_id = expected.version_id.map(|v| v.to_string()); - let lock_guard = if !opts.no_lock { + let (decommission_object_lock_guard, decommission_target_lock_covered, mut decommission_capacity_guard) = + if let Some(store) = opts.decommission_capacity_admission.as_ref() { + store + .acquire_external_decommission_commit_guards(self.pool_index, bucket, object, opts.no_lock) + .await? + } else { + (None, false, None) + }; + let lock_guard = if !opts.no_lock && !decommission_target_lock_covered { Some( self.acquire_write_lock_diag("restore_cleanup_metadata", bucket, object) .await?, @@ -241,6 +271,15 @@ impl SetDisks { } else { None }; + if decommission_capacity_guard.is_none() + && let Some(store) = opts.decommission_capacity_admission.as_ref() + { + decommission_capacity_guard = Some( + store + .acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation") + .await?, + ); + } let read_opts = ObjectOptions { version_id, versioned: opts.versioned, @@ -269,7 +308,12 @@ impl SetDisks { &mut fi.metadata, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, ); - if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || decommission_object_lock_guard + .as_ref() + .is_some_and(|guard| guard.is_lock_lost()) + || decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { return Err(Error::other("restore cleanup lock lost before metadata update".to_string())); } self.invalidate_get_object_metadata_cache(bucket, object).await; diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 9159d38b3..71708aea2 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -470,6 +470,27 @@ impl ECStore { let object = encode_dir_object(object); let pools = self.get_pools_for_heal_object(opts)?; + if let Some(set_idx) = opts.set { + for pool in &pools { + if set_idx >= pool.disk_set.len() { + let err = StorageError::InvalidArgument( + "heal".to_string(), + "set".to_string(), + format!( + "invalid heal set index {set_idx} for pool {} with {} sets", + pool.pool_idx, + pool.disk_set.len() + ), + ); + if opts.pool.is_some() { + return Err(err); + } + return Ok((HealResultItem::default(), Some(err))); + } + } + } + #[cfg(test)] + let store_id = self.id; let mut futures = Vec::with_capacity(pools.len()); for pool in pools.iter() { @@ -499,7 +520,17 @@ impl ECStore { } continue; } - futures.push(pool.heal_object(bucket, &object, version_id, opts)); + let pool_idx = pool.pool_idx; + let pool = Arc::clone(pool); + let pool_object = object.clone(); + let opts = *opts; + futures.push( + self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move { + #[cfg(test)] + crate::core::pools::notify_decommission_external_heal_operation_started(store_id); + pool.heal_object(bucket, &pool_object, version_id, &opts).await + }), + ); } let results = join_all(futures).await; @@ -594,14 +625,18 @@ mod tests { use crate::cluster::rpc::PeerS3Client; use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config}; use crate::core::pools::{ - POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test, + DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, + PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test, + set_decommission_capacity_info_overrides_for_test, }; use crate::core::sets::HealFormatAfterSaveBarrier; use crate::disk::error::Result as DiskResult; use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::runtime::instance::InstanceContext; - use crate::services::rebalance::{RebalanceInfo, RebalanceStats}; + use crate::services::rebalance::{ + RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores, + }; use crate::storage_api_contracts::bucket::{ BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, }; @@ -1207,6 +1242,292 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() { + let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await; + let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple()); + let object = "targeted-heal.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("heal capacity bucket should be created"); + + let target_set = store.pools[1].get_disks(0); + let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec()); + target_set + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("targeted heal fixture should be written"); + let missing_disk = target_set.disks.read().await[0] + .clone() + .expect("targeted heal fixture disk should be online"); + missing_disk + .delete( + &bucket, + object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("targeted heal fixture shard should be removed"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_err(), + "targeted heal fixture must start with one missing metadata copy" + ); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("the exact-fit decommission reservation should activate"); + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("the active decommission reservation should be durable"); + assert_eq!(reservation.targets.len(), 1); + assert_eq!(reservation.targets[0].pool_index, 1); + assert_eq!(reservation.targets[0].reserved_physical_bytes, 60); + } + + let (_, err) = store + .handle_heal_object( + &bucket, + object, + "", + &HealOpts { + pool: Some(1), + set: Some(0), + ..Default::default() + }, + ) + .await + .expect("targeted heal should return a mapped capacity result"); + assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_err(), + "capacity-blocked targeted heal must not rewrite the missing shard" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn targeted_heal_keeps_target_lock_for_different_lock_domain() { + let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple()); + let object = "different-domain-heal.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("heal lock-domain bucket should be created"); + + let target_set = other_store.pools[1].get_disks(0); + let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec()); + target_set + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("heal lock-domain fixture should be written"); + let missing_disk = target_set.disks.read().await[0] + .clone() + .expect("heal lock-domain fixture disk should be online"); + missing_disk + .delete( + &bucket, + object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("heal lock-domain fixture shard should be removed"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_err(), + "heal lock-domain fixture must start with one missing metadata copy" + ); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30), + DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100), + DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("heal lock-domain reservation should activate"); + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("heal lock-domain reservation should be durable"); + assert_eq!(reservation.targets.len(), 1); + assert_eq!(reservation.targets[0].pool_index, 2); + } + let pool_meta = store.pool_meta.read().await.clone(); + *other_store.pool_meta.write().await = pool_meta; + + let fixed_set = other_store.pools[0].disk_set[0].clone(); + assert!( + !fixed_set.shares_namespace_lock_domain(&target_set).await, + "heal fixture must use different fixed and target lock domains" + ); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id); + let heal_store = Arc::clone(&other_store); + let heal_bucket = bucket.clone(); + let heal_object = object.to_string(); + let mut heal = tokio::spawn(async move { + heal_store + .handle_heal_object( + &heal_bucket, + &heal_object, + "", + &HealOpts { + pool: Some(1), + set: Some(0), + ..Default::default() + }, + ) + .await + }); + + barrier.wait_until_external_capacity_released().await; + barrier.wait_until_external_heal_target_lock_attempted().await; + + let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal) + .await + .expect("targeted heal should finish after target lock release") + .expect("targeted heal task should not panic") + .expect("targeted heal should complete"); + assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_ok(), + "targeted heal should rewrite the missing shard after the target lock is released" + ); + drop(barrier); + } + + #[tokio::test] + #[serial_test::serial] + async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() { + let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await; + let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple()); + let object = "shared-domain-heal.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("shared heal lock-domain bucket should be created"); + + let target_set = store.pools[0].get_disks(0); + let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec()); + target_set + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("shared heal lock-domain fixture should be written"); + let missing_disk = target_set.disks.read().await[0] + .clone() + .expect("shared heal lock-domain fixture disk should be online"); + missing_disk + .delete( + &bucket, + object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("shared heal lock-domain fixture shard should be removed"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_err(), + "shared heal lock-domain fixture must start with one missing metadata copy" + ); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100), + DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[2], Vec::new()) + .await + .expect("shared heal lock-domain reservation should activate"); + { + let pool_meta = store.pool_meta.read().await; + let reservation = pool_meta.pools[2] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("shared heal lock-domain reservation should be durable"); + assert_eq!(reservation.targets.len(), 1); + assert_eq!(reservation.targets[0].pool_index, 1); + assert_eq!(reservation.targets[0].reserved_physical_bytes, 60); + } + + let fixed_set = store.pools[0].disk_set[0].clone(); + assert!( + fixed_set.shares_namespace_lock_domain(&target_set).await, + "shared heal fixture must use one fixed and target lock domain" + ); + + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + let heal_store = Arc::clone(&store); + let heal_bucket = bucket.clone(); + let heal_object = object.to_string(); + let mut heal = tokio::spawn(async move { + heal_store + .handle_heal_object( + &heal_bucket, + &heal_object, + "", + &HealOpts { + pool: Some(0), + set: Some(0), + ..Default::default() + }, + ) + .await + }); + + barrier.wait_until_external_capacity_released().await; + barrier.wait_until_external_heal_operation_started().await; + let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal) + .await + .expect("shared-domain heal must not reenter the fixed namespace lock") + .expect("shared-domain heal task should not panic") + .expect("shared-domain heal should complete"); + assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}"); + assert!( + missing_disk.read_xl(&bucket, object, false).await.is_ok(), + "shared-domain heal should rewrite the missing shard" + ); + drop(barrier); + } + #[tokio::test] async fn scoped_heal_object_defers_when_requested_pool_is_suspended() { let mut store = minimal_heal_store().await; @@ -1697,8 +2018,10 @@ mod tests { .expect("quorum boundary heal should return a mapped result"); *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks; assert!( - matches!(quorum_err, Some(Error::ErasureReadQuorum)), - "quorum-boundary heal must preserve quorum error, got {quorum_err:?}" + quorum_err.as_ref().is_some_and(|err| err + .to_string() + .contains("pool metadata writes remain blocked after a recovery-required replica state")), + "heal must fail closed when capacity admission cannot verify pool metadata, got {quorum_err:?}" ); shutdown.cancel(); } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index fdaaf6d88..95cb27505 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -2142,8 +2142,16 @@ mod tests { }, ) .await?; - crate::data_movement::migrate_decommission_object(migration_store, 0, migration_bucket, source_reader, None, op_label) - .await + crate::data_movement::migrate_decommission_object( + migration_store, + 0, + migration_bucket, + source_reader, + None, + op_label, + None, + ) + .await }); barrier.wait_until_paused().await; barrier.release(); diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index fd4a4750b..a2eed1658 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id}; use crate::multipart_listing::paginate_multipart_listing; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::multipart::MultipartOperations as _; @@ -196,6 +197,21 @@ async fn list_pool_multipart_uploads_for_incarnation( } impl ECStore { + pub(crate) async fn acquire_decommission_multipart_mutation_fence( + &self, + owner: DecommissionCapacityOwner, + ) -> Result { + let mutation_id = owner + .mutation_id + .ok_or_else(|| Error::other("decommission multipart mutation identity is missing"))?; + let object = format!( + "decommission-multipart/{}/{}/{}/{}", + owner.source_pool_index, owner.operation_id, owner.generation, mutation_id + ); + self.acquire_object_write_lock("decommission_multipart_mutation", crate::disk::RUSTFS_META_MULTIPART_BUCKET, &object) + .await + } + async fn existing_multipart_pool_order(&self) -> Vec { // A draining source must not hide a valid UploadID in an active target, // while physical order within each phase preserves fail-closed errors. @@ -215,6 +231,24 @@ impl ECStore { active } + async fn multipart_upload_pool_idx( + &self, + bucket: &str, + object: &str, + upload_id: &str, + opts: &ObjectOptions, + ) -> Result { + for pool_idx in self.existing_multipart_pool_order().await { + match self.pools[pool_idx].get_multipart_info(bucket, object, upload_id, opts).await { + Ok(_) => return Ok(pool_idx), + Err(err) if is_err_invalid_upload_id(&err) => continue, + Err(err) => return Err(err), + } + } + + Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) + } + #[allow(clippy::too_many_arguments)] pub async fn list_multipart_uploads_for_bucket_incarnation( &self, @@ -433,8 +467,10 @@ impl ECStore { if self.single_pool() { self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence) .await; - return self.pools[0] - .new_multipart_upload(bucket, object, &opts) + return self + .run_decommission_capacity_admitted_mutation(0, None, None, || async { + self.pools[0].new_multipart_upload(bucket, object, &opts).await + }) .await .map(|res| (res, 0, opts.expected_bucket_incarnation_id)); } @@ -450,7 +486,14 @@ impl ECStore { } self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) .await; - let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; + let res = self + .run_decommission_capacity_temporary_mutation( + idx, + DecommissionCapacityOwner::from_options(&opts), + None, + || async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await }, + ) + .await?; return Ok((res, idx, opts.expected_bucket_incarnation_id)); } @@ -475,8 +518,19 @@ impl ECStore { if !res.uploads.is_empty() { self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) .await; - let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; - return Ok((res, idx, opts.expected_bucket_incarnation_id)); + let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id; + let lock_object = encode_dir_object(object); + let res = self + .run_external_decommission_capacity_object_mutation( + idx, + bucket, + &lock_object, + object, + opts, + |opts| async move { self.pools[idx].new_multipart_upload(bucket, object, &opts).await }, + ) + .await?; + return Ok((res, idx, expected_bucket_incarnation_id)); } } let idx = self.get_pool_idx(bucket, object, -1).await?; @@ -490,8 +544,23 @@ impl ECStore { self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) .await; - let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; - Ok((res, idx, opts.expected_bucket_incarnation_id)) + let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id; + let res = if opts.data_movement { + self.run_decommission_capacity_temporary_mutation( + idx, + DecommissionCapacityOwner::from_options(&opts), + None, + || async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await }, + ) + .await? + } else { + let lock_object = encode_dir_object(object); + self.run_external_decommission_capacity_object_mutation(idx, bucket, &lock_object, object, opts, |opts| async move { + self.pools[idx].new_multipart_upload(bucket, object, &opts).await + }) + .await? + }; + Ok((res, idx, expected_bucket_incarnation_id)) } #[instrument(skip(self))] @@ -529,7 +598,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_put_object_part_args(bucket, object, upload_id)?; - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; let opts = &opts; if self.single_pool() { @@ -538,26 +608,10 @@ impl ECStore { .await; } - for pool_idx in self.existing_multipart_pool_order().await { - let pool = &self.pools[pool_idx]; - let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await { - Ok(res) => return Ok(res), - Err(err) => { - if is_err_invalid_upload_id(&err) { - None - } else { - Some(err) - } - } - }; - - if let Some(err) = err { - error!("put_object_part err: {:?}", err); - return Err(err); - } - } - - Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) + let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?; + self.pools[pool_idx] + .put_object_part(bucket, object, upload_id, part_id, data, opts) + .await } pub(crate) async fn put_object_part_for_data_movement( @@ -576,12 +630,24 @@ impl ECStore { if !opts.data_movement { return Err(Error::other("targeted multipart upload requires data_movement options")); } - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; - let pool = self - .pools - .get(target_pool_idx) - .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; - pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + ensure_decommission_capacity_mutation_id(bucket, object, &mut opts); + let pool = self.pools.get(target_pool_idx).ok_or_else(|| { + Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string()) + })?; + let expected_data_bytes = usize::try_from(data.size()).ok(); + self.run_decommission_capacity_temporary_mutation_with_capacity_lease( + target_pool_idx, + DecommissionCapacityOwner::from_options(&opts), + expected_data_bytes, + |capacity_lease| async move { + if let Some(capacity_lease) = capacity_lease { + opts.add_namespace_lock_lost_signal(capacity_lease); + } + pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await + }, + ) + .await } #[instrument(skip(self))] @@ -662,16 +728,95 @@ impl ECStore { upload_id: &str, opts: &ObjectOptions, ) -> Result<()> { - check_abort_multipart_args(bucket, object, upload_id)?; - if !opts.data_movement { - return Err(Error::other("targeted multipart abort requires data_movement options")); - } - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[upload_id.to_owned()], None, opts) + .await + } + + pub(crate) async fn reconcile_multipart_uploads_for_data_movement( + &self, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_identity: &str, + opts: &ObjectOptions, + ) -> Result<()> { let pool = self .pools .get(target_pool_idx) .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; - pool.abort_multipart_upload(bucket, object, upload_id, &opts).await + let owner = DecommissionCapacityOwner::from_options(opts); + let has_capacity_state = match owner { + Some(owner) => { + self.has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner) + .await + } + None => false, + }; + if !has_capacity_state { + return Ok(()); + } + let set = pool.get_disks_by_key(object); + let upload_ids = set + .data_movement_multipart_upload_ids(bucket, object, opts.expected_bucket_incarnation_id, upload_identity) + .await?; + self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &upload_ids, Some(upload_identity), opts) + .await + } + + async fn abort_multipart_uploads_for_data_movement( + &self, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_ids: &[String], + expected_upload_identity: Option<&str>, + opts: &ObjectOptions, + ) -> Result<()> { + check_new_multipart_args(bucket, object)?; + for upload_id in upload_ids { + check_abort_multipart_args(bucket, object, upload_id)?; + } + if !opts.data_movement { + return Err(Error::other("targeted multipart abort requires data_movement options")); + } + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + ensure_decommission_capacity_mutation_id(bucket, object, &mut opts); + let pool = self + .pools + .get(target_pool_idx) + .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; + let set = pool.get_disks_by_key(object); + let mut guards = Vec::with_capacity(upload_ids.len()); + for upload_id in upload_ids { + if let Some(guard) = set + .lock_data_movement_multipart_abort(bucket, object, upload_id, expected_upload_identity, &opts) + .await? + { + guard.add_namespace_lock_fence(&mut opts); + guards.push(guard); + } + } + opts.no_lock = true; + let capacity_owner = DecommissionCapacityOwner::from_options(&opts); + // Keep every upload namespace guard alive through the final capacity progress save. + let result = self + .run_decommission_capacity_temporary_release_with_capacity_lease(target_pool_idx, capacity_owner, |capacity_lease| { + let mut delete_opts = opts.clone(); + let guards = &guards; + let set = &set; + async move { + if let Some(capacity_lease) = capacity_lease { + delete_opts.add_namespace_lock_lost_signal(capacity_lease); + } + for guard in guards { + guard.delete(set, bucket, object, &delete_opts).await?; + } + Ok(()) + } + }) + .await; + drop(guards); + result } #[instrument(skip(self))] @@ -684,7 +829,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_complete_multipart_args(bucket, object, upload_id)?; - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; let opts = &opts; if self.single_pool() { @@ -694,27 +840,10 @@ impl ECStore { .await; } - for pool_idx in self.existing_multipart_pool_order().await { - let pool = &self.pools[pool_idx]; - - let pool = pool.clone(); - let err = match pool - .complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts) - .await - { - Ok(res) => return Ok(res), - Err(err) => { - // - if is_err_invalid_upload_id(&err) { None } else { Some(err) } - } - }; - - if let Some(er) = err { - return Err(er); - } - } - - Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) + let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?; + let pool = self.pools[pool_idx].clone(); + pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) + .await } pub(crate) async fn complete_multipart_upload_for_data_movement( @@ -732,6 +861,7 @@ impl ECStore { return Err(Error::other("targeted multipart completion requires data_movement options")); } let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + ensure_decommission_capacity_mutation_id(bucket, object, &mut opts); if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) { let expected_incarnation_id = opts .expected_bucket_incarnation_id @@ -764,12 +894,24 @@ impl ECStore { .get(target_pool_idx) .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))? .clone(); - let result = enqueue_transition_after_write( - pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts) - .await, - LcEventSrc::S3CompleteMultipartUpload, - ) - .await; + // Data movement already owns the pool-meta write lease. Forward its + // loss signal into SetDisks so commit fencing observes the same lease + // without trying to reacquire the namespace. + let result = self + .run_decommission_capacity_admitted_mutation_with_capacity_lease( + target_pool_idx, + DecommissionCapacityOwner::from_options(&opts), + opts.capacity_expected_data_bytes(), + |capacity_lease| async move { + if let Some(capacity_lease) = capacity_lease { + opts.add_namespace_lock_lost_signal(capacity_lease); + } + pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts) + .await + }, + ) + .await; + let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await; if result.is_ok() { list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await; } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 2f86d94f3..b3e1c5008 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -34,6 +34,7 @@ use crate::bucket::object_lock::objectlock_sys::{ }; use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge}; use crate::bucket::versioning::VersioningApi; +use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id}; use crate::disk::OldCurrentSize; use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot}; use crate::set_disk::{ @@ -1865,7 +1866,12 @@ impl ECStore { } } - async fn acquire_object_write_lock(&self, op: &'static str, bucket: &str, object: &str) -> Result { + pub(super) async fn acquire_object_write_lock( + &self, + op: &'static str, + bucket: &str, + object: &str, + ) -> Result { let diag_enabled = is_object_lock_diag_enabled(); let ns_lock = self.handle_new_ns_lock(bucket, object).await?; let acquire_start = Instant::now(); @@ -2001,6 +2007,150 @@ impl ECStore { ))) } + pub(super) async fn run_external_decommission_capacity_object_mutation( + &self, + target_pool_idx: usize, + bucket: &str, + lock_object: &str, + target_object: &str, + mut opts: ObjectOptions, + operation: F, + ) -> Result + where + F: FnOnce(ObjectOptions) -> Fut, + Fut: std::future::Future>, + { + let (capacity_guard, has_active_decommission) = self + .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation") + .await?; + let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock { + // Active migration acquires the object namespace before its capacity + // write. Match that order, then recheck capacity admission. + drop(capacity_guard); + #[cfg(test)] + crate::core::pools::notify_decommission_external_object_capacity_released(self.id); + let guard = self + .acquire_object_write_lock("external_capacity_order", bucket, lock_object) + .await?; + self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard)) + .await; + let capacity_guard = self + .acquire_external_decommission_capacity_fence(&[target_pool_idx], "mutation") + .await?; + (capacity_guard, Some(guard)) + } else { + (capacity_guard, None) + }; + + let result = operation(opts).await; + drop(capacity_guard); + drop(object_guard); + result + } + + /// Finish an external object mutation's staged phase by acquiring the + /// fixed namespace before the target-side commit lock. The caller must + /// recheck capacity only after all applicable namespaces are held. + /// + /// Callers must invoke this only after consuming and staging their input + /// stream. The returned namespace guard remains owned by the caller until + /// its local commit completes; `target_lock_covered` tells the set layer + /// whether that guard also covers its target namespace. When no + /// decommission is active, the returned capacity guard is the admission + /// probe and must likewise remain held until the commit completes. + pub(crate) async fn acquire_external_decommission_commit_guards( + &self, + target_pool_idx: usize, + bucket: &str, + object: &str, + no_lock: bool, + ) -> Result<(Option, bool, Option)> { + let (capacity_guard, has_active_decommission) = self + .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation") + .await?; + if !has_active_decommission { + // Keep the read probe through the staged commit. This closes the + // activation gap between admission and publication. + #[cfg(test)] + { + crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id); + crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await; + } + return Ok((None, false, Some(capacity_guard))); + } + drop(capacity_guard); + if no_lock { + return Ok((None, false, None)); + } + + // Active migration holds the fixed object namespace before taking its + // capacity write fence. Drop the probe and acquire that namespace + // before rechecking capacity, so the staged external commit cannot + // form capacity-read -> object-write against the migration path. + let object_guard = self + .acquire_object_write_lock("external_capacity_order", bucket, object) + .await?; + let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()); + let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object)); + let target_lock_covered = match (fixed_set, target_set) { + (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await, + _ => false, + }; + Ok((Some(object_guard), target_lock_covered, None)) + } + + pub(super) async fn run_external_decommission_capacity_heal( + &self, + target_pool_idx: usize, + bucket: &str, + lock_object: &str, + mut opts: HealOpts, + operation: F, + ) -> Result + where + F: FnOnce(HealOpts) -> Fut, + Fut: std::future::Future>, + { + let (capacity_guard, has_active_decommission) = self + .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "heal") + .await?; + let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock { + // Active migration acquires the object namespace before its capacity + // write. Match that order, then recheck capacity admission. + drop(capacity_guard); + #[cfg(test)] + crate::core::pools::notify_decommission_external_object_capacity_released(self.id); + let guard = self + .acquire_object_write_lock("external_capacity_order", bucket, lock_object) + .await?; + let target_set = self + .pools + .get(target_pool_idx) + .ok_or_else(|| Error::other("heal target pool is unavailable"))? + .get_disks_for_heal_object(lock_object, &opts)?; + let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) { + Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await, + None => false, + }; + let capacity_guard = self + .acquire_external_decommission_capacity_fence(&[target_pool_idx], "heal") + .await?; + opts.no_lock = target_lock_covered; + (capacity_guard, Some(guard)) + } else { + (capacity_guard, None) + }; + + #[cfg(test)] + if !opts.no_lock { + crate::core::pools::notify_decommission_external_heal_target_lock_attempted(); + } + let result = operation(opts).await; + drop(capacity_guard); + drop(object_guard); + result + } + pub(crate) async fn acquire_decommission_object_mutation_fence( &self, bucket: &str, @@ -2239,7 +2389,7 @@ impl ECStore { resolve_latest_object_access(bucket, object, info, idx, opts) } - pub(super) async fn select_data_movement_pool_idx( + pub(crate) async fn select_data_movement_pool_idx( &self, bucket: &str, object: &str, @@ -2257,6 +2407,16 @@ impl ECStore { return Err(err); } + if let Some(owner) = DecommissionCapacityOwner::from_options(opts) { + let expected_data_bytes = opts + .capacity_expected_data_bytes() + .or_else(|| usize::try_from(size).ok()) + .unwrap_or_default(); + return self + .select_decommission_capacity_target_pool(owner, expected_data_bytes) + .await; + } + self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull) } } @@ -2293,13 +2453,18 @@ impl ECStore { opts: &ObjectOptions, target_pool_idx: usize, ) -> Result { - resolve_data_movement_delete_marker_resume_result( + let equivalent = resolve_data_movement_delete_marker_resume_result( self.find_data_movement_target_info(bucket, object, target_pool_idx, opts) .await, source, opts.src_pool_idx, target_pool_idx, - ) + )?; + if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) { + self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0) + .await?; + } + Ok(equivalent) } async fn has_equivalent_data_movement_tiered_object( @@ -2310,13 +2475,19 @@ impl ECStore { opts: &ObjectOptions, target_pool_idx: usize, ) -> Result { - resolve_data_movement_tiered_resume_result( + let equivalent = resolve_data_movement_tiered_resume_result( self.find_data_movement_target_info(bucket, object, target_pool_idx, opts) .await, source, opts.src_pool_idx, target_pool_idx, - ) + )?; + if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) { + let expected_data_bytes = usize::try_from(source.size).unwrap_or_default(); + self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes) + .await?; + } + Ok(equivalent) } async fn has_equivalent_data_movement_tier_free_version( @@ -2331,9 +2502,15 @@ impl ECStore { .pools .get(target_pool_idx) .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?; - pool.get_disks_by_key(object) + let equivalent = pool + .get_disks_by_key(object) .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts) - .await + .await?; + if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) { + self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0) + .await?; + } + Ok(equivalent) } fn resolve_decommission_target_pool_idx_result(result: Result, bucket: &str, object: &str) -> Result { @@ -2374,6 +2551,7 @@ impl ECStore { let logical_object = object; let object = encode_dir_object(logical_object); + ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts); if self.single_pool() { return Self::resolve_decommission_tiered_object_result( Err(Error::other("single pool deployments cannot decommission tiered objects")), @@ -2428,9 +2606,15 @@ impl ECStore { crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?; } if opts.data_movement && idx == opts.src_pool_idx { - let resume_target_pool_idx = self - .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx) - .await; + let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) { + Some( + self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default()) + .await?, + ) + } else { + self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx) + .await + }; let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx); if is_free_version && target_pool_idx == opts.src_pool_idx { return Err(Error::DiskFull); @@ -2449,17 +2633,21 @@ impl ECStore { return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id)); } - let result = if is_free_version { - self.pools[idx] - .get_disks_by_key(&object) - .decommission_tier_free_version(bucket, &object, &fi, &opts) - .await - } else { - self.pools[idx] - .get_disks_by_key(&object) - .decommission_tiered_object(bucket, &object, &fi, &opts) - .await - }; + let result = self + .run_decommission_capacity_admitted_mutation(idx, DecommissionCapacityOwner::from_options(&opts), None, || async { + if is_free_version { + self.pools[idx] + .get_disks_by_key(&object) + .decommission_tier_free_version(bucket, &object, &fi, &opts) + .await + } else { + self.pools[idx] + .get_disks_by_key(&object) + .decommission_tiered_object(bucket, &object, &fi, &opts) + .await + } + }) + .await; if matches!(result, Err(Error::PreconditionFailed)) { if self .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx) @@ -2611,15 +2799,29 @@ impl ECStore { return Err(Error::other("data movement PUT requires data_movement options")); } let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?; + ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts); let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence) .await; - let result = self.pools[idx] - .put_object_with_old_current_size(bucket, &object, data, &opts) - .await - .map(|(object_info, _)| object_info); + let expected_data_bytes = usize::try_from(data.size()).ok(); + let result = self + .run_decommission_capacity_admitted_mutation_with_capacity_lease( + idx, + DecommissionCapacityOwner::from_options(&opts), + expected_data_bytes, + |capacity_lease| async move { + if let Some(capacity_lease) = capacity_lease { + opts.add_namespace_lock_lost_signal(capacity_lease); + } + self.pools[idx] + .put_object_with_old_current_size(bucket, &object, data, &opts) + .await + .map(|(object_info, _)| object_info) + }, + ) + .await; let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await; if result.is_ok() { list_objects::observe_list_objects_mutation(self, bucket).await; @@ -2640,11 +2842,10 @@ impl ECStore { let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; - - // Keep PUT atomic-read friendly: SetDisks takes the object write lock only - // around precondition checks and the final rename/commit. + let mut opts = opts; + opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; self.pools[idx] - .put_object_with_old_current_size(bucket, &object, data, &opts) + .put_object_with_old_current_size(bucket, object.as_str(), data, &opts) .await } @@ -2761,8 +2962,20 @@ impl ECStore { && let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) && src_vid == dst_vid { - return self.pools[pool_idx] - .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts) + let capacity_object = dst_object.clone(); + return self + .run_external_decommission_capacity_object_mutation( + pool_idx, + dst_bucket, + &capacity_object, + &capacity_object, + dst_opts.clone(), + |opts| async move { + self.pools[pool_idx] + .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts) + .await + }, + ) .await; } @@ -2777,12 +2990,24 @@ impl ECStore { // metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling // versioned branch below resolves the same risk by rewriting through // put_object (issue #4238). - return self.pools[pool_idx] - .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts) + let capacity_object = dst_object.clone(); + return self + .run_external_decommission_capacity_object_mutation( + pool_idx, + dst_bucket, + &capacity_object, + &capacity_object, + dst_opts.clone(), + |opts| async move { + self.pools[pool_idx] + .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts) + .await + }, + ) .await; } // Transitioned object self-copy: restore from tier into the same pool. - let put_opts = ObjectOptions { + let mut put_opts = ObjectOptions { user_defined: (*src_info.user_defined).clone(), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), @@ -2796,6 +3021,8 @@ impl ECStore { object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; + put_opts.decommission_capacity_admission = + crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; return if let Some(reader) = src_info.put_object_reader.as_mut() { self.pools[pool_idx] .put_object(dst_bucket, &dst_object, reader, &put_opts) @@ -2816,7 +3043,7 @@ impl ECStore { // stays consistent with the new version's metadata. Sharing the source data_dir via // a metadata-only version copy would corrupt SSE/compressed objects (issue #4238). if let Some(reader) = src_info.put_object_reader.as_mut() { - let put_opts = ObjectOptions { + let mut put_opts = ObjectOptions { user_defined: (*src_info.user_defined).clone(), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), @@ -2830,13 +3057,27 @@ impl ECStore { object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; + put_opts.decommission_capacity_admission = + crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; return self.pools[pool_idx] .put_object(dst_bucket, &dst_object, reader, &put_opts) .await; } src_info.version_only = true; - return self.pools[pool_idx] - .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts) + let capacity_object = dst_object.clone(); + return self + .run_external_decommission_capacity_object_mutation( + pool_idx, + dst_bucket, + &capacity_object, + &capacity_object, + dst_opts.clone(), + |opts| async move { + self.pools[pool_idx] + .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts) + .await + }, + ) .await; } } @@ -2846,8 +3087,9 @@ impl ECStore { } else { self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await? }; + let dst_object_name = dst_object.as_str(); - let put_opts = ObjectOptions { + let mut put_opts = ObjectOptions { user_defined: (*src_info.user_defined).clone(), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), @@ -2861,10 +3103,11 @@ impl ECStore { object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; + put_opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; if let Some(put_object_reader) = src_info.put_object_reader.as_mut() { return self.pools[pool_idx] - .put_object(dst_bucket, &dst_object, put_object_reader, &put_opts) + .put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts) .await; } @@ -2982,6 +3225,7 @@ impl ECStore { }; let object = object.as_str(); let mut opts = opts; + ensure_decommission_capacity_mutation_id(bucket, object, &mut opts); let delete_all_configs = if opts.lifecycle_delete_all.is_some() { Some(get_expiry_configs(self, bucket).await?) } else { @@ -3132,11 +3376,21 @@ impl ECStore { let selected_target_pool_idx = match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? { Some(pool_idx) => pool_idx, - None => self.get_pool_idx_no_lock(bucket, object, 0).await?, + None => { + if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) { + self.select_decommission_capacity_target_pool(owner, 0).await? + } else { + self.get_pool_idx_no_lock(bucket, object, 0).await? + } + } }; let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx { - self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx) - .await + if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) { + Some(self.select_decommission_capacity_target_pool(owner, 0).await?) + } else { + self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx) + .await + } } else { None }; @@ -3174,6 +3428,10 @@ impl ECStore { .await?; if let Some(target) = target { if is_equivalent_data_movement_delete_marker(&source, &target) { + if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) { + self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0) + .await?; + } let mut target = target; target.name = decode_dir_object(object); return Ok(target); @@ -3214,7 +3472,20 @@ impl ECStore { } let target_opts = delete_marker_target_opts.unwrap_or(opts); - let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await?; + let mut obj = self + .run_decommission_capacity_admitted_mutation_with_capacity_lease( + target_pool_idx, + DecommissionCapacityOwner::from_options(&target_opts), + None, + |capacity_lease| async move { + let mut target_opts = target_opts; + if let Some(capacity_lease) = capacity_lease { + target_opts.add_namespace_lock_lost_signal(capacity_lease); + } + self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await + }, + ) + .await?; obj.name = decode_dir_object(obj.name.as_str()); return Ok(obj); } @@ -3225,7 +3496,17 @@ impl ECStore { Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum), Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => { let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?; - let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + let pool = self.pools[target_pool_idx].clone(); + let mut obj = self + .run_external_decommission_capacity_object_mutation( + target_pool_idx, + bucket, + object, + object, + opts, + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await?; #[cfg(test)] pause_versioned_delete_marker_after_commit(bucket, object).await; obj.name = decode_dir_object(object); @@ -3288,7 +3569,19 @@ impl ECStore { continue; } - match pool.delete_object(bucket, object, opts.clone()).await { + let pool_idx = pool.pool_idx; + let pool = pool.clone(); + match self + .run_external_decommission_capacity_object_mutation( + pool_idx, + bucket, + object, + object, + opts.clone(), + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await + { Ok(res) => { #[cfg(test)] pause_versioned_delete_marker_after_commit(bucket, object).await; @@ -3474,6 +3767,19 @@ impl ECStore { } } + let _capacity_fence = if latest_marker_objects.iter().any(|creates_marker| *creates_marker) { + let target_pool_indices = (0..self.pools.len()).collect::>(); + match self + .acquire_external_decommission_capacity_fence(&target_pool_indices, "batch_delete") + .await + { + Ok(fence) => Some(fence), + Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), + } + } else { + None + }; + let mut futures = Vec::with_capacity(self.pools.len()); for pool in self.pools.iter() { if self.is_pool_rebalancing(pool.pool_idx).await { @@ -3777,20 +4083,23 @@ impl ECStore { // re-check is tracked separately. Holding the lock here instead // (#4877) blocked HEAD/get_object_info for the whole copy-back and // self-deadlocked on the inner commits. + let object_name = object.as_str(); if self.single_pool() { + opts.decommission_capacity_admission = Some(Arc::clone(&self)); return self.pools[0] .clone() - .restore_transitioned_object(bucket, &object, &opts) + .restore_transitioned_object(bucket, object_name, &opts) .await; } let (_, idx) = self - .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock)) + .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock)) .await?; + opts.decommission_capacity_admission = Some(Arc::clone(&self)); self.pools[idx] .clone() - .restore_transitioned_object(bucket, &object, &opts) + .restore_transitioned_object(bucket, object_name, &opts) .await } @@ -3824,14 +4133,36 @@ impl ECStore { }; if self.single_pool() { - return self.pools[0].put_object_metadata(bucket, object.as_str(), &opts).await; + let pool = self.pools[0].clone(); + let capacity_object = object.clone(); + return self + .run_external_decommission_capacity_object_mutation( + 0, + bucket, + capacity_object.as_str(), + capacity_object.as_str(), + opts, + |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await }, + ) + .await; } let (_, idx) = self .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock)) .await?; - let result = self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await; + let pool = self.pools[idx].clone(); + let capacity_object = object.clone(); + let result = self + .run_external_decommission_capacity_object_mutation( + idx, + bucket, + capacity_object.as_str(), + capacity_object.as_str(), + opts, + |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await }, + ) + .await; drop(bucket_lifecycle_guard); result } @@ -3857,16 +4188,34 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { let object = encode_dir_object(object); + let object_name = object.as_str(); if self.single_pool() { - return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await; + return self + .run_external_decommission_capacity_object_mutation( + 0, + bucket, + object_name, + object_name, + opts.clone(), + |opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await }, + ) + .await; } let (_, idx) = self - .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock)) + .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock)) .await?; - self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await + self.run_external_decommission_capacity_object_mutation( + idx, + bucket, + object_name, + object_name, + opts.clone(), + |opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await }, + ) + .await } #[instrument(skip(self))] @@ -3892,16 +4241,34 @@ impl ECStore { #[instrument(skip(self))] pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { let object = encode_dir_object(object); + let object_name = object.as_str(); if self.single_pool() { - return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await; + return self + .run_external_decommission_capacity_object_mutation( + 0, + bucket, + object_name, + object_name, + opts.clone(), + |opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await }, + ) + .await; } let (_, idx) = self - .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock)) + .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock)) .await?; - self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await + self.run_external_decommission_capacity_object_mutation( + idx, + bucket, + object_name, + object_name, + opts.clone(), + |opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await }, + ) + .await } pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 6c2fe8642..14de70b66 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -925,9 +925,19 @@ impl ECStore { } if let Some(idx) = pe.index { + let pool = self.pools[idx].clone(); results.push(RebalanceDeletePoolResult { pool_idx: idx, - result: self.pools[idx].delete_object(bucket, object, opts.clone()).await, + result: self + .run_external_decommission_capacity_object_mutation( + idx, + bucket, + object, + object, + opts.clone(), + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await, }); } } diff --git a/crates/log-analyzer/src/rules/seed/capacity.rs b/crates/log-analyzer/src/rules/seed/capacity.rs index 589e1a5db..fd036320d 100644 --- a/crates/log-analyzer/src/rules/seed/capacity.rs +++ b/crates/log-analyzer/src/rules/seed/capacity.rs @@ -75,13 +75,13 @@ pub(super) fn rules() -> Vec { }, Rule { evidence_fields: strings(["required", "target_free"]), - anchors: strings(["insufficient target pool capacity"]), + anchors: strings(["insufficient reserved physical target capacity"]), ..base( "decom-capacity-insufficient", P1Unavailable, "capacity", "pool 下线目标容量不足", - contains("insufficient target pool capacity"), + contains("insufficient reserved physical target capacity"), "pool 下线迁移目标容量不足,decommission 无法开始。", "先扩容目标 pool。", ) diff --git a/crates/log-analyzer/src/rules/seed/tests.rs b/crates/log-analyzer/src/rules/seed/tests.rs index 0a283eed8..442546854 100644 --- a/crates/log-analyzer/src/rules/seed/tests.rs +++ b/crates/log-analyzer/src/rules/seed/tests.rs @@ -262,7 +262,9 @@ fn every_rule_has_a_positive_sample() { ), ( "decom-capacity-insufficient", - msg("failed to start decommission: insufficient target pool capacity: required 100 bytes available 50 bytes"), + msg( + "failed to start decommission: insufficient reserved physical target capacity: required 100 bytes available 50 bytes", + ), ), // ops ("decom-object-failed", msg("Decommission object migration failed")), diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index 6d7ee4f80..a2c6df010 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -395,7 +395,7 @@ impl DefaultObjectUsecase { // with that key (for example, copying `bucket/bucket` onto itself). let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; - if cp_src_dst_same && key == bucket && expected_current_version_id.is_none() { + if cp_src_dst_same && key == bucket { dst_opts.object_lock_config_snapshot = Some(store.object_lock_config_snapshot(&bucket).await.map_err(ApiError::from)?); } @@ -405,7 +405,11 @@ impl DefaultObjectUsecase { .map_err(ApiError::from)?, ); - let _self_copy_lock_guard = if cp_src_dst_same && expected_current_version_id.is_none() { + // Hold the self-copy namespace write lock before opening the source reader, including + // expected-current historical copies. With lock optimization disabled, the source + // stream retains its read guard until EOF; taking the write lock later in ECStore + // would self-deadlock before the copy can consume that stream. + let _self_copy_lock_guard = if cp_src_dst_same { let guard = acquire_self_copy_namespace_lock(store.as_ref(), &bucket, &key).await?; src_opts.no_lock = true; src_get_opts.no_lock = true; @@ -1161,6 +1165,129 @@ mod tests { assert_ne!(err.code(), &S3ErrorCode::InvalidRequest); } + #[tokio::test] + #[serial_test::serial] + async fn execute_copy_object_expected_current_historical_self_copy_releases_reader_before_write() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let ambient = current_app_context().expect("expected-current copy test requires an AppContext"); + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + let bucket = format!("copy-current-lock-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("versioned copy test bucket should be created"); + + const STREAMING_TEST_BODY_SIZE: usize = 256 * 1024; + let historical_body = vec![b'h'; STREAMING_TEST_BODY_SIZE]; + let current_body = vec![b'c'; STREAMING_TEST_BODY_SIZE]; + let mut old_reader = PutObjReader::from_vec(historical_body.clone()); + let old = store + .put_object( + &bucket, + object, + &mut old_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("historical source version should be written"); + assert!(!old.is_inline_fast_path_eligible(), "historical source must use the streaming path"); + let mut current_reader = PutObjReader::from_vec(current_body); + let current = store + .put_object( + &bucket, + object, + &mut current_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("current destination version should be written"); + let source_version_id = old.version_id.expect("historical source should have a version id"); + let current_version_id = current.version_id.expect("current destination should have a version id"); + + let input = CopyObjectInput::builder() + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: object.to_string().into(), + version_id: Some(source_version_id.to_string().into()), + }) + .bucket(bucket.clone()) + .key(object.to_string()) + .build() + .expect("expected-current historical copy input should build"); + let mut req = build_request(input, Method::PUT); + req.headers.insert( + RUSTFS_EXPECTED_CURRENT_VERSION_ID, + HeaderValue::from_str(¤t_version_id.to_string()).expect("current version header should be valid"), + ); + let response = temp_env::async_with_vars( + [(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], + tokio::time::timeout( + Duration::from_secs(10), + DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req), + ), + ) + .await + .expect("nonbuffered source reader must not deadlock behind the self-copy write lock") + .expect("expected-current historical self-copy should succeed"); + assert!(response.output.copy_object_result.is_some()); + let source_version_id_text = source_version_id.to_string(); + assert_eq!(response.output.copy_source_version_id.as_deref(), Some(source_version_id_text.as_str())); + let destination_version_id = response + .output + .version_id + .clone() + .expect("versioned self-copy should return its destination version"); + assert_ne!(destination_version_id, source_version_id_text); + assert_ne!(destination_version_id, current_version_id.to_string()); + use tokio::io::AsyncReadExt; + let verify_opts = ObjectOptions { + version_id: Some(destination_version_id.clone()), + versioned: true, + ..Default::default() + }; + let mut verified = store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &verify_opts) + .await + .expect("copied destination version should be readable"); + let mut copied_body = Vec::new(); + verified + .stream + .read_to_end(&mut copied_body) + .await + .expect("copied destination body should be readable"); + assert_eq!(copied_body, historical_body); + assert_eq!(verified.object_info.version_id.map(|id| id.to_string()), Some(destination_version_id)); + + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await + .expect("expected-current copy test bucket should be removed"); + } + #[tokio::test] async fn execute_copy_object_allows_self_copy_of_null_version() { // A "null" source version id is a restore of the null version, not a no-op self-copy. diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index abcae7b50..bfed7fc0e 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -26,7 +26,7 @@ 10|crates/ecstore/src/cluster/rpc/remote_disk.rs 6|crates/ecstore/src/config/com.rs 14|crates/ecstore/src/config/storageclass.rs -185|crates/ecstore/src/core/pools.rs +183|crates/ecstore/src/core/pools.rs 8|crates/ecstore/src/data_movement/mod.rs 2|crates/ecstore/src/data_usage/local_snapshot.rs 12|crates/ecstore/src/data_usage/mod.rs