gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-03 02:38:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): reserve decommission capacity safely (#6917)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-31 15:20:09 +08:00
committed by GitHub
parent ea01cd339c
commit 6c67086d0b
24 changed files with 10523 additions and 488 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/store/heal.rs
+328 -5
View File
@@ -470,6 +470,27 @@ impl ECStore {
let object = encode_dir_object(object);
let pools = self.get_pools_for_heal_object(opts)?;
if let Some(set_idx) = opts.set {
for pool in &pools {
if set_idx >= pool.disk_set.len() {
let err = StorageError::InvalidArgument(
"heal".to_string(),
"set".to_string(),
format!(
"invalid heal set index {set_idx} for pool {} with {} sets",
pool.pool_idx,
pool.disk_set.len()
),
);
if opts.pool.is_some() {
return Err(err);
}
return Ok((HealResultItem::default(), Some(err)));
}
}
}
#[cfg(test)]
let store_id = self.id;
let mut futures = Vec::with_capacity(pools.len());
for pool in pools.iter() {
@@ -499,7 +520,17 @@ impl ECStore {
}
continue;
}
futures.push(pool.heal_object(bucket, &object, version_id, opts));
let pool_idx = pool.pool_idx;
let pool = Arc::clone(pool);
let pool_object = object.clone();
let opts = *opts;
futures.push(
self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move {
#[cfg(test)]
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
pool.heal_object(bucket, &pool_object, version_id, &opts).await
}),
);
}
let results = join_all(futures).await;
@@ -594,14 +625,18 @@ mod tests {
use crate::cluster::rpc::PeerS3Client;
use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
use crate::core::pools::{
POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test,
DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME,
PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test,
set_decommission_capacity_info_overrides_for_test,
};
use crate::core::sets::HealFormatAfterSaveBarrier;
use crate::disk::error::Result as DiskResult;
use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::runtime::instance::InstanceContext;
use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
use crate::services::rebalance::{
RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
};
use crate::storage_api_contracts::bucket::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
@@ -1207,6 +1242,292 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
let object = "targeted-heal.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("heal capacity bucket should be created");
let target_set = store.pools[1].get_disks(0);
let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
target_set
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("targeted heal fixture should be written");
let missing_disk = target_set.disks.read().await[0]
.clone()
.expect("targeted heal fixture disk should be online");
missing_disk
.delete(
&bucket,
object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("targeted heal fixture shard should be removed");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_err(),
"targeted heal fixture must start with one missing metadata copy"
);
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("the exact-fit decommission reservation should activate");
{
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("the active decommission reservation should be durable");
assert_eq!(reservation.targets.len(), 1);
assert_eq!(reservation.targets[0].pool_index, 1);
assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
}
let (_, err) = store
.handle_heal_object(
&bucket,
object,
"",
&HealOpts {
pool: Some(1),
set: Some(0),
..Default::default()
},
)
.await
.expect("targeted heal should return a mapped capacity result");
assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_err(),
"capacity-blocked targeted heal must not rewrite the missing shard"
);
}
#[tokio::test]
#[serial_test::serial]
async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
let object = "different-domain-heal.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("heal lock-domain bucket should be created");
let target_set = other_store.pools[1].get_disks(0);
let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
target_set
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("heal lock-domain fixture should be written");
let missing_disk = target_set.disks.read().await[0]
.clone()
.expect("heal lock-domain fixture disk should be online");
missing_disk
.delete(
&bucket,
object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("heal lock-domain fixture shard should be removed");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_err(),
"heal lock-domain fixture must start with one missing metadata copy"
);
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("heal lock-domain reservation should activate");
{
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[0]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("heal lock-domain reservation should be durable");
assert_eq!(reservation.targets.len(), 1);
assert_eq!(reservation.targets[0].pool_index, 2);
}
let pool_meta = store.pool_meta.read().await.clone();
*other_store.pool_meta.write().await = pool_meta;
let fixed_set = other_store.pools[0].disk_set[0].clone();
assert!(
!fixed_set.shares_namespace_lock_domain(&target_set).await,
"heal fixture must use different fixed and target lock domains"
);
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
let heal_store = Arc::clone(&other_store);
let heal_bucket = bucket.clone();
let heal_object = object.to_string();
let mut heal = tokio::spawn(async move {
heal_store
.handle_heal_object(
&heal_bucket,
&heal_object,
"",
&HealOpts {
pool: Some(1),
set: Some(0),
..Default::default()
},
)
.await
});
barrier.wait_until_external_capacity_released().await;
barrier.wait_until_external_heal_target_lock_attempted().await;
let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
.await
.expect("targeted heal should finish after target lock release")
.expect("targeted heal task should not panic")
.expect("targeted heal should complete");
assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_ok(),
"targeted heal should rewrite the missing shard after the target lock is released"
);
drop(barrier);
}
#[tokio::test]
#[serial_test::serial]
async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
let object = "shared-domain-heal.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("shared heal lock-domain bucket should be created");
let target_set = store.pools[0].get_disks(0);
let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
target_set
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("shared heal lock-domain fixture should be written");
let missing_disk = target_set.disks.read().await[0]
.clone()
.expect("shared heal lock-domain fixture disk should be online");
missing_disk
.delete(
&bucket,
object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("shared heal lock-domain fixture shard should be removed");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_err(),
"shared heal lock-domain fixture must start with one missing metadata copy"
);
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
set_decommission_capacity_info_overrides_for_test(
store.id,
vec![vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
]],
);
store
.save_current_pool_meta_for_decommission_start(&[2], Vec::new())
.await
.expect("shared heal lock-domain reservation should activate");
{
let pool_meta = store.pool_meta.read().await;
let reservation = pool_meta.pools[2]
.decommission
.as_ref()
.and_then(|info| info.capacity_reservation.as_ref())
.expect("shared heal lock-domain reservation should be durable");
assert_eq!(reservation.targets.len(), 1);
assert_eq!(reservation.targets[0].pool_index, 1);
assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
}
let fixed_set = store.pools[0].disk_set[0].clone();
assert!(
fixed_set.shares_namespace_lock_domain(&target_set).await,
"shared heal fixture must use one fixed and target lock domain"
);
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
let heal_store = Arc::clone(&store);
let heal_bucket = bucket.clone();
let heal_object = object.to_string();
let mut heal = tokio::spawn(async move {
heal_store
.handle_heal_object(
&heal_bucket,
&heal_object,
"",
&HealOpts {
pool: Some(0),
set: Some(0),
..Default::default()
},
)
.await
});
barrier.wait_until_external_capacity_released().await;
barrier.wait_until_external_heal_operation_started().await;
let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
.await
.expect("shared-domain heal must not reenter the fixed namespace lock")
.expect("shared-domain heal task should not panic")
.expect("shared-domain heal should complete");
assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
assert!(
missing_disk.read_xl(&bucket, object, false).await.is_ok(),
"shared-domain heal should rewrite the missing shard"
);
drop(barrier);
}
#[tokio::test]
async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
let mut store = minimal_heal_store().await;
@@ -1697,8 +2018,10 @@ mod tests {
.expect("quorum boundary heal should return a mapped result");
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
assert!(
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
quorum_err.as_ref().is_some_and(|err| err
.to_string()
.contains("pool metadata writes remain blocked after a recovery-required replica state")),
"heal must fail closed when capacity admission cannot verify pool metadata, got {quorum_err:?}"
);
shutdown.cancel();
}
crates/ecstore/src/store/init.rs
+10 -2
View File
@@ -2142,8 +2142,16 @@ mod tests {
},
)
.await?;
crate::data_movement::migrate_decommission_object(migration_store, 0, migration_bucket, source_reader, None, op_label)
.await
crate::data_movement::migrate_decommission_object(
migration_store,
0,
migration_bucket,
source_reader,
None,
op_label,
None,
)
.await
});
barrier.wait_until_paused().await;
barrier.release();
crates/ecstore/src/store/multipart.rs
+210 -68
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
use crate::multipart_listing::paginate_multipart_listing;
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
@@ -196,6 +197,21 @@ async fn list_pool_multipart_uploads_for_incarnation(
}
impl ECStore {
pub(crate) async fn acquire_decommission_multipart_mutation_fence(
&self,
owner: DecommissionCapacityOwner,
) -> Result<ObjectLockDiagGuard> {
let mutation_id = owner
.mutation_id
.ok_or_else(|| Error::other("decommission multipart mutation identity is missing"))?;
let object = format!(
"decommission-multipart/{}/{}/{}/{}",
owner.source_pool_index, owner.operation_id, owner.generation, mutation_id
);
self.acquire_object_write_lock("decommission_multipart_mutation", crate::disk::RUSTFS_META_MULTIPART_BUCKET, &object)
.await
}
async fn existing_multipart_pool_order(&self) -> Vec<usize> {
// A draining source must not hide a valid UploadID in an active target,
// while physical order within each phase preserves fail-closed errors.
@@ -215,6 +231,24 @@ impl ECStore {
active
}
async fn multipart_upload_pool_idx(
&self,
bucket: &str,
object: &str,
upload_id: &str,
opts: &ObjectOptions,
) -> Result<usize> {
for pool_idx in self.existing_multipart_pool_order().await {
match self.pools[pool_idx].get_multipart_info(bucket, object, upload_id, opts).await {
Ok(_) => return Ok(pool_idx),
Err(err) if is_err_invalid_upload_id(&err) => continue,
Err(err) => return Err(err),
}
}
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
}
#[allow(clippy::too_many_arguments)]
pub async fn list_multipart_uploads_for_bucket_incarnation(
&self,
@@ -433,8 +467,10 @@ impl ECStore {
if self.single_pool() {
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
.await;
return self.pools[0]
.new_multipart_upload(bucket, object, &opts)
return self
.run_decommission_capacity_admitted_mutation(0, None, None, || async {
self.pools[0].new_multipart_upload(bucket, object, &opts).await
})
.await
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
}
@@ -450,7 +486,14 @@ impl ECStore {
}
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
let res = self
.run_decommission_capacity_temporary_mutation(
idx,
DecommissionCapacityOwner::from_options(&opts),
None,
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
)
.await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
@@ -475,8 +518,19 @@ impl ECStore {
if !res.uploads.is_empty() {
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
let lock_object = encode_dir_object(object);
let res = self
.run_external_decommission_capacity_object_mutation(
idx,
bucket,
&lock_object,
object,
opts,
|opts| async move { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
)
.await?;
return Ok((res, idx, expected_bucket_incarnation_id));
}
}
let idx = self.get_pool_idx(bucket, object, -1).await?;
@@ -490,8 +544,23 @@ impl ECStore {
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
Ok((res, idx, opts.expected_bucket_incarnation_id))
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
let res = if opts.data_movement {
self.run_decommission_capacity_temporary_mutation(
idx,
DecommissionCapacityOwner::from_options(&opts),
None,
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
)
.await?
} else {
let lock_object = encode_dir_object(object);
self.run_external_decommission_capacity_object_mutation(idx, bucket, &lock_object, object, opts, |opts| async move {
self.pools[idx].new_multipart_upload(bucket, object, &opts).await
})
.await?
};
Ok((res, idx, expected_bucket_incarnation_id))
}
#[instrument(skip(self))]
@@ -529,7 +598,8 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
let opts = &opts;
if self.single_pool() {
@@ -538,26 +608,10 @@ impl ECStore {
.await;
}
for pool_idx in self.existing_multipart_pool_order().await {
let pool = &self.pools[pool_idx];
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
Ok(res) => return Ok(res),
Err(err) => {
if is_err_invalid_upload_id(&err) {
None
} else {
Some(err)
}
}
};
if let Some(err) = err {
error!("put_object_part err: {:?}", err);
return Err(err);
}
}
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?;
self.pools[pool_idx]
.put_object_part(bucket, object, upload_id, part_id, data, opts)
.await
}
pub(crate) async fn put_object_part_for_data_movement(
@@ -576,12 +630,24 @@ impl ECStore {
if !opts.data_movement {
return Err(Error::other("targeted multipart upload requires data_movement options"));
}
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
let pool = self.pools.get(target_pool_idx).ok_or_else(|| {
Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string())
})?;
let expected_data_bytes = usize::try_from(data.size()).ok();
self.run_decommission_capacity_temporary_mutation_with_capacity_lease(
target_pool_idx,
DecommissionCapacityOwner::from_options(&opts),
expected_data_bytes,
|capacity_lease| async move {
if let Some(capacity_lease) = capacity_lease {
opts.add_namespace_lock_lost_signal(capacity_lease);
}
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
},
)
.await
}
#[instrument(skip(self))]
@@ -662,16 +728,95 @@ impl ECStore {
upload_id: &str,
opts: &ObjectOptions,
) -> Result<()> {
check_abort_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart abort requires data_movement options"));
}
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[upload_id.to_owned()], None, opts)
.await
}
pub(crate) async fn reconcile_multipart_uploads_for_data_movement(
&self,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_identity: &str,
opts: &ObjectOptions,
) -> Result<()> {
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
pool.abort_multipart_upload(bucket, object, upload_id, &opts).await
let owner = DecommissionCapacityOwner::from_options(opts);
let has_capacity_state = match owner {
Some(owner) => {
self.has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner)
.await
}
None => false,
};
if !has_capacity_state {
return Ok(());
}
let set = pool.get_disks_by_key(object);
let upload_ids = set
.data_movement_multipart_upload_ids(bucket, object, opts.expected_bucket_incarnation_id, upload_identity)
.await?;
self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &upload_ids, Some(upload_identity), opts)
.await
}
async fn abort_multipart_uploads_for_data_movement(
&self,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_ids: &[String],
expected_upload_identity: Option<&str>,
opts: &ObjectOptions,
) -> Result<()> {
check_new_multipart_args(bucket, object)?;
for upload_id in upload_ids {
check_abort_multipart_args(bucket, object, upload_id)?;
}
if !opts.data_movement {
return Err(Error::other("targeted multipart abort requires data_movement options"));
}
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
let set = pool.get_disks_by_key(object);
let mut guards = Vec::with_capacity(upload_ids.len());
for upload_id in upload_ids {
if let Some(guard) = set
.lock_data_movement_multipart_abort(bucket, object, upload_id, expected_upload_identity, &opts)
.await?
{
guard.add_namespace_lock_fence(&mut opts);
guards.push(guard);
}
}
opts.no_lock = true;
let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
// Keep every upload namespace guard alive through the final capacity progress save.
let result = self
.run_decommission_capacity_temporary_release_with_capacity_lease(target_pool_idx, capacity_owner, |capacity_lease| {
let mut delete_opts = opts.clone();
let guards = &guards;
let set = &set;
async move {
if let Some(capacity_lease) = capacity_lease {
delete_opts.add_namespace_lock_lost_signal(capacity_lease);
}
for guard in guards {
guard.delete(set, bucket, object, &delete_opts).await?;
}
Ok(())
}
})
.await;
drop(guards);
result
}
#[instrument(skip(self))]
@@ -684,7 +829,8 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_complete_multipart_args(bucket, object, upload_id)?;
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
let opts = &opts;
if self.single_pool() {
@@ -694,27 +840,10 @@ impl ECStore {
.await;
}
for pool_idx in self.existing_multipart_pool_order().await {
let pool = &self.pools[pool_idx];
let pool = pool.clone();
let err = match pool
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
.await
{
Ok(res) => return Ok(res),
Err(err) => {
//
if is_err_invalid_upload_id(&err) { None } else { Some(err) }
}
};
if let Some(er) = err {
return Err(er);
}
}
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?;
let pool = self.pools[pool_idx].clone();
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
.await
}
pub(crate) async fn complete_multipart_upload_for_data_movement(
@@ -732,6 +861,7 @@ impl ECStore {
return Err(Error::other("targeted multipart completion requires data_movement options"));
}
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
let expected_incarnation_id = opts
.expected_bucket_incarnation_id
@@ -764,12 +894,24 @@ impl ECStore {
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?
.clone();
let result = enqueue_transition_after_write(
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
.await,
LcEventSrc::S3CompleteMultipartUpload,
)
.await;
// Data movement already owns the pool-meta write lease. Forward its
// loss signal into SetDisks so commit fencing observes the same lease
// without trying to reacquire the namespace.
let result = self
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
target_pool_idx,
DecommissionCapacityOwner::from_options(&opts),
opts.capacity_expected_data_bytes(),
|capacity_lease| async move {
if let Some(capacity_lease) = capacity_lease {
opts.add_namespace_lock_lost_signal(capacity_lease);
}
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
.await
},
)
.await;
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
}
crates/ecstore/src/store/object.rs
+424 -57
View File
@@ -34,6 +34,7 @@ use crate::bucket::object_lock::objectlock_sys::{
};
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
use crate::bucket::versioning::VersioningApi;
use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
use crate::disk::OldCurrentSize;
use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot};
use crate::set_disk::{
@@ -1865,7 +1866,12 @@ impl ECStore {
}
}
async fn acquire_object_write_lock(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
pub(super) async fn acquire_object_write_lock(
&self,
op: &'static str,
bucket: &str,
object: &str,
) -> Result<ObjectLockDiagGuard> {
let diag_enabled = is_object_lock_diag_enabled();
let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
let acquire_start = Instant::now();
@@ -2001,6 +2007,150 @@ impl ECStore {
)))
}
pub(super) async fn run_external_decommission_capacity_object_mutation<T, F, Fut>(
&self,
target_pool_idx: usize,
bucket: &str,
lock_object: &str,
target_object: &str,
mut opts: ObjectOptions,
operation: F,
) -> Result<T>
where
F: FnOnce(ObjectOptions) -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let (capacity_guard, has_active_decommission) = self
.acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation")
.await?;
let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
// Active migration acquires the object namespace before its capacity
// write. Match that order, then recheck capacity admission.
drop(capacity_guard);
#[cfg(test)]
crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
let guard = self
.acquire_object_write_lock("external_capacity_order", bucket, lock_object)
.await?;
self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
.await;
let capacity_guard = self
.acquire_external_decommission_capacity_fence(&[target_pool_idx], "mutation")
.await?;
(capacity_guard, Some(guard))
} else {
(capacity_guard, None)
};
let result = operation(opts).await;
drop(capacity_guard);
drop(object_guard);
result
}
/// Finish an external object mutation's staged phase by acquiring the
/// fixed namespace before the target-side commit lock. The caller must
/// recheck capacity only after all applicable namespaces are held.
///
/// Callers must invoke this only after consuming and staging their input
/// stream. The returned namespace guard remains owned by the caller until
/// its local commit completes; `target_lock_covered` tells the set layer
/// whether that guard also covers its target namespace. When no
/// decommission is active, the returned capacity guard is the admission
/// probe and must likewise remain held until the commit completes.
pub(crate) async fn acquire_external_decommission_commit_guards(
&self,
target_pool_idx: usize,
bucket: &str,
object: &str,
no_lock: bool,
) -> Result<(Option<ObjectLockDiagGuard>, bool, Option<rustfs_lock::NamespaceLockGuard>)> {
let (capacity_guard, has_active_decommission) = self
.acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation")
.await?;
if !has_active_decommission {
// Keep the read probe through the staged commit. This closes the
// activation gap between admission and publication.
#[cfg(test)]
{
crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id);
crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await;
}
return Ok((None, false, Some(capacity_guard)));
}
drop(capacity_guard);
if no_lock {
return Ok((None, false, None));
}
// Active migration holds the fixed object namespace before taking its
// capacity write fence. Drop the probe and acquire that namespace
// before rechecking capacity, so the staged external commit cannot
// form capacity-read -> object-write against the migration path.
let object_guard = self
.acquire_object_write_lock("external_capacity_order", bucket, object)
.await?;
let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
let target_lock_covered = match (fixed_set, target_set) {
(Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
_ => false,
};
Ok((Some(object_guard), target_lock_covered, None))
}
pub(super) async fn run_external_decommission_capacity_heal<T, F, Fut>(
&self,
target_pool_idx: usize,
bucket: &str,
lock_object: &str,
mut opts: HealOpts,
operation: F,
) -> Result<T>
where
F: FnOnce(HealOpts) -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let (capacity_guard, has_active_decommission) = self
.acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "heal")
.await?;
let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
// Active migration acquires the object namespace before its capacity
// write. Match that order, then recheck capacity admission.
drop(capacity_guard);
#[cfg(test)]
crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
let guard = self
.acquire_object_write_lock("external_capacity_order", bucket, lock_object)
.await?;
let target_set = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other("heal target pool is unavailable"))?
.get_disks_for_heal_object(lock_object, &opts)?;
let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) {
Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await,
None => false,
};
let capacity_guard = self
.acquire_external_decommission_capacity_fence(&[target_pool_idx], "heal")
.await?;
opts.no_lock = target_lock_covered;
(capacity_guard, Some(guard))
} else {
(capacity_guard, None)
};
#[cfg(test)]
if !opts.no_lock {
crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
}
let result = operation(opts).await;
drop(capacity_guard);
drop(object_guard);
result
}
pub(crate) async fn acquire_decommission_object_mutation_fence(
&self,
bucket: &str,
@@ -2239,7 +2389,7 @@ impl ECStore {
resolve_latest_object_access(bucket, object, info, idx, opts)
}
pub(super) async fn select_data_movement_pool_idx(
pub(crate) async fn select_data_movement_pool_idx(
&self,
bucket: &str,
object: &str,
@@ -2257,6 +2407,16 @@ impl ECStore {
return Err(err);
}
if let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
let expected_data_bytes = opts
.capacity_expected_data_bytes()
.or_else(|| usize::try_from(size).ok())
.unwrap_or_default();
return self
.select_decommission_capacity_target_pool(owner, expected_data_bytes)
.await;
}
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
}
}
@@ -2293,13 +2453,18 @@ impl ECStore {
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_delete_marker_resume_result(
let equivalent = resolve_data_movement_delete_marker_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
)?;
if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
.await?;
}
Ok(equivalent)
}
async fn has_equivalent_data_movement_tiered_object(
@@ -2310,13 +2475,19 @@ impl ECStore {
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_tiered_resume_result(
let equivalent = resolve_data_movement_tiered_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
)?;
if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
let expected_data_bytes = usize::try_from(source.size).unwrap_or_default();
self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
.await?;
}
Ok(equivalent)
}
async fn has_equivalent_data_movement_tier_free_version(
@@ -2331,9 +2502,15 @@ impl ECStore {
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
pool.get_disks_by_key(object)
let equivalent = pool
.get_disks_by_key(object)
.has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
.await
.await?;
if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
.await?;
}
Ok(equivalent)
}
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
@@ -2374,6 +2551,7 @@ impl ECStore {
let logical_object = object;
let object = encode_dir_object(logical_object);
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
if self.single_pool() {
return Self::resolve_decommission_tiered_object_result(
Err(Error::other("single pool deployments cannot decommission tiered objects")),
@@ -2428,9 +2606,15 @@ impl ECStore {
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
}
if opts.data_movement && idx == opts.src_pool_idx {
let resume_target_pool_idx = self
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
.await;
let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
Some(
self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default())
.await?,
)
} else {
self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
.await
};
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
if is_free_version && target_pool_idx == opts.src_pool_idx {
return Err(Error::DiskFull);
@@ -2449,17 +2633,21 @@ impl ECStore {
return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
}
let result = if is_free_version {
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tier_free_version(bucket, &object, &fi, &opts)
.await
} else {
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, &fi, &opts)
.await
};
let result = self
.run_decommission_capacity_admitted_mutation(idx, DecommissionCapacityOwner::from_options(&opts), None, || async {
if is_free_version {
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tier_free_version(bucket, &object, &fi, &opts)
.await
} else {
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, &fi, &opts)
.await
}
})
.await;
if matches!(result, Err(Error::PreconditionFailed)) {
if self
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
@@ -2611,15 +2799,29 @@ impl ECStore {
return Err(Error::other("data movement PUT requires data_movement options"));
}
let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence)
.await;
let result = self.pools[idx]
.put_object_with_old_current_size(bucket, &object, data, &opts)
.await
.map(|(object_info, _)| object_info);
let expected_data_bytes = usize::try_from(data.size()).ok();
let result = self
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
idx,
DecommissionCapacityOwner::from_options(&opts),
expected_data_bytes,
|capacity_lease| async move {
if let Some(capacity_lease) = capacity_lease {
opts.add_namespace_lock_lost_signal(capacity_lease);
}
self.pools[idx]
.put_object_with_old_current_size(bucket, &object, data, &opts)
.await
.map(|(object_info, _)| object_info)
},
)
.await;
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self, bucket).await;
@@ -2640,11 +2842,10 @@ impl ECStore {
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
// Keep PUT atomic-read friendly: SetDisks takes the object write lock only
// around precondition checks and the final rename/commit.
let mut opts = opts;
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
self.pools[idx]
.put_object_with_old_current_size(bucket, &object, data, &opts)
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
.await
}
@@ -2761,8 +2962,20 @@ impl ECStore {
&& let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
&& src_vid == dst_vid
{
return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
let capacity_object = dst_object.clone();
return self
.run_external_decommission_capacity_object_mutation(
pool_idx,
dst_bucket,
&capacity_object,
&capacity_object,
dst_opts.clone(),
|opts| async move {
self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
.await
},
)
.await;
}
@@ -2777,12 +2990,24 @@ impl ECStore {
// metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling
// versioned branch below resolves the same risk by rewriting through
// put_object (issue #4238).
return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
let capacity_object = dst_object.clone();
return self
.run_external_decommission_capacity_object_mutation(
pool_idx,
dst_bucket,
&capacity_object,
&capacity_object,
dst_opts.clone(),
|opts| async move {
self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
.await
},
)
.await;
}
// Transitioned object self-copy: restore from tier into the same pool.
let put_opts = ObjectOptions {
let mut put_opts = ObjectOptions {
user_defined: (*src_info.user_defined).clone(),
versioned: dst_opts.versioned,
version_id: dst_opts.version_id.clone(),
@@ -2796,6 +3021,8 @@ impl ECStore {
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
..Default::default()
};
put_opts.decommission_capacity_admission =
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
return if let Some(reader) = src_info.put_object_reader.as_mut() {
self.pools[pool_idx]
.put_object(dst_bucket, &dst_object, reader, &put_opts)
@@ -2816,7 +3043,7 @@ impl ECStore {
// stays consistent with the new version's metadata. Sharing the source data_dir via
// a metadata-only version copy would corrupt SSE/compressed objects (issue #4238).
if let Some(reader) = src_info.put_object_reader.as_mut() {
let put_opts = ObjectOptions {
let mut put_opts = ObjectOptions {
user_defined: (*src_info.user_defined).clone(),
versioned: dst_opts.versioned,
version_id: dst_opts.version_id.clone(),
@@ -2830,13 +3057,27 @@ impl ECStore {
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
..Default::default()
};
put_opts.decommission_capacity_admission =
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
return self.pools[pool_idx]
.put_object(dst_bucket, &dst_object, reader, &put_opts)
.await;
}
src_info.version_only = true;
return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
let capacity_object = dst_object.clone();
return self
.run_external_decommission_capacity_object_mutation(
pool_idx,
dst_bucket,
&capacity_object,
&capacity_object,
dst_opts.clone(),
|opts| async move {
self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
.await
},
)
.await;
}
}
@@ -2846,8 +3087,9 @@ impl ECStore {
} else {
self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
};
let dst_object_name = dst_object.as_str();
let put_opts = ObjectOptions {
let mut put_opts = ObjectOptions {
user_defined: (*src_info.user_defined).clone(),
versioned: dst_opts.versioned,
version_id: dst_opts.version_id.clone(),
@@ -2861,10 +3103,11 @@ impl ECStore {
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
..Default::default()
};
put_opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
return self.pools[pool_idx]
.put_object(dst_bucket, &dst_object, put_object_reader, &put_opts)
.put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
.await;
}
@@ -2982,6 +3225,7 @@ impl ECStore {
};
let object = object.as_str();
let mut opts = opts;
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
Some(get_expiry_configs(self, bucket).await?)
} else {
@@ -3132,11 +3376,21 @@ impl ECStore {
let selected_target_pool_idx =
match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
Some(pool_idx) => pool_idx,
None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
None => {
if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
self.select_decommission_capacity_target_pool(owner, 0).await?
} else {
self.get_pool_idx_no_lock(bucket, object, 0).await?
}
}
};
let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
.await
if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
Some(self.select_decommission_capacity_target_pool(owner, 0).await?)
} else {
self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
.await
}
} else {
None
};
@@ -3174,6 +3428,10 @@ impl ECStore {
.await?;
if let Some(target) = target {
if is_equivalent_data_movement_delete_marker(&source, &target) {
if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) {
self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
.await?;
}
let mut target = target;
target.name = decode_dir_object(object);
return Ok(target);
@@ -3214,7 +3472,20 @@ impl ECStore {
}
let target_opts = delete_marker_target_opts.unwrap_or(opts);
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await?;
let mut obj = self
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
target_pool_idx,
DecommissionCapacityOwner::from_options(&target_opts),
None,
|capacity_lease| async move {
let mut target_opts = target_opts;
if let Some(capacity_lease) = capacity_lease {
target_opts.add_namespace_lock_lost_signal(capacity_lease);
}
self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await
},
)
.await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
@@ -3225,7 +3496,17 @@ impl ECStore {
Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
let pool = self.pools[target_pool_idx].clone();
let mut obj = self
.run_external_decommission_capacity_object_mutation(
target_pool_idx,
bucket,
object,
object,
opts,
|opts| async move { pool.delete_object(bucket, object, opts).await },
)
.await?;
#[cfg(test)]
pause_versioned_delete_marker_after_commit(bucket, object).await;
obj.name = decode_dir_object(object);
@@ -3288,7 +3569,19 @@ impl ECStore {
continue;
}
match pool.delete_object(bucket, object, opts.clone()).await {
let pool_idx = pool.pool_idx;
let pool = pool.clone();
match self
.run_external_decommission_capacity_object_mutation(
pool_idx,
bucket,
object,
object,
opts.clone(),
|opts| async move { pool.delete_object(bucket, object, opts).await },
)
.await
{
Ok(res) => {
#[cfg(test)]
pause_versioned_delete_marker_after_commit(bucket, object).await;
@@ -3474,6 +3767,19 @@ impl ECStore {
}
}
let _capacity_fence = if latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
let target_pool_indices = (0..self.pools.len()).collect::<Vec<_>>();
match self
.acquire_external_decommission_capacity_fence(&target_pool_indices, "batch_delete")
.await
{
Ok(fence) => Some(fence),
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
}
} else {
None
};
let mut futures = Vec::with_capacity(self.pools.len());
for pool in self.pools.iter() {
if self.is_pool_rebalancing(pool.pool_idx).await {
@@ -3777,20 +4083,23 @@ impl ECStore {
// re-check is tracked separately. Holding the lock here instead
// (#4877) blocked HEAD/get_object_info for the whole copy-back and
// self-deadlocked on the inner commits.
let object_name = object.as_str();
if self.single_pool() {
opts.decommission_capacity_admission = Some(Arc::clone(&self));
return self.pools[0]
.clone()
.restore_transitioned_object(bucket, &object, &opts)
.restore_transitioned_object(bucket, object_name, &opts)
.await;
}
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
.get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock))
.await?;
opts.decommission_capacity_admission = Some(Arc::clone(&self));
self.pools[idx]
.clone()
.restore_transitioned_object(bucket, &object, &opts)
.restore_transitioned_object(bucket, object_name, &opts)
.await
}
@@ -3824,14 +4133,36 @@ impl ECStore {
};
if self.single_pool() {
return self.pools[0].put_object_metadata(bucket, object.as_str(), &opts).await;
let pool = self.pools[0].clone();
let capacity_object = object.clone();
return self
.run_external_decommission_capacity_object_mutation(
0,
bucket,
capacity_object.as_str(),
capacity_object.as_str(),
opts,
|opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
)
.await;
}
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
.await?;
let result = self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await;
let pool = self.pools[idx].clone();
let capacity_object = object.clone();
let result = self
.run_external_decommission_capacity_object_mutation(
idx,
bucket,
capacity_object.as_str(),
capacity_object.as_str(),
opts,
|opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
)
.await;
drop(bucket_lifecycle_guard);
result
}
@@ -3857,16 +4188,34 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
let object_name = object.as_str();
if self.single_pool() {
return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await;
return self
.run_external_decommission_capacity_object_mutation(
0,
bucket,
object_name,
object_name,
opts.clone(),
|opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await },
)
.await;
}
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock))
.get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
.await?;
self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
self.run_external_decommission_capacity_object_mutation(
idx,
bucket,
object_name,
object_name,
opts.clone(),
|opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await },
)
.await
}
#[instrument(skip(self))]
@@ -3892,16 +4241,34 @@ impl ECStore {
#[instrument(skip(self))]
pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
let object_name = object.as_str();
if self.single_pool() {
return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await;
return self
.run_external_decommission_capacity_object_mutation(
0,
bucket,
object_name,
object_name,
opts.clone(),
|opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await },
)
.await;
}
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock))
.get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
.await?;
self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
self.run_external_decommission_capacity_object_mutation(
idx,
bucket,
object_name,
object_name,
opts.clone(),
|opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await },
)
.await
}
pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
crates/ecstore/src/store/rebalance.rs
+11 -1
View File
@@ -925,9 +925,19 @@ impl ECStore {
}
if let Some(idx) = pe.index {
let pool = self.pools[idx].clone();
results.push(RebalanceDeletePoolResult {
pool_idx: idx,
result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
result: self
.run_external_decommission_capacity_object_mutation(
idx,
bucket,
object,
object,
opts.clone(),
|opts| async move { pool.delete_object(bucket, object, opts).await },
)
.await,
});
}
}