gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): fence deletes against decommission commits (#6363)

Browse Source 
* fix(ecstore): fence deletes against decommission commits

* fix(ecstore): preserve decommission target write locks

* fix(ecstore): preserve delete markers during source cleanup

* fix(ecstore): route batch delete markers to active pools

* fix(ecstore): preserve batch delete pool errors

* fix(ecstore): retain source-set lock during cleanup

* test(ecstore): exercise decommission delete fences

* test(ecstore): finish decommission delete fence scenario

* fix(ecstore): reuse fixed fence for reverse decommission

* fix(ecstore): fence decommission commit loss

* fix(ecstore): annotate batch delete fallback

* test(ecstore): fix decommission fence fixtures

* fix(ecstore): unblock decommission delete fences

* fix(ecstore): preserve distributed decommission set locks

* fix(ecstore): match decommission lock backend domain

* test(ecstore): align decommission fence barriers

* fix(ecstore): satisfy delete fence lint checks

* fix(rebalance): preserve access-denied delete errors
This commit is contained in:
Zhengchao An
2026-08-23 00:40:38 +08:00
committed by GitHub
parent 3ddf1a81ac
commit 7c1a76dfd9
13 changed files with 2606 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/data_movement/mod.rs
+161 -42
View File
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{HTTPPreconditions, ObjectOperations as _},
};
use crate::store::ECStore;
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
is_unversioned_data_movement_object(source)
&& is_unversioned_data_movement_object(target)
&& !target.delete_marker
&& source
.mod_time
.zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
pub(crate) struct SourceCleanupBucketFence<'a> {
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
}
fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
struct SourceCleanupDeleteBarrierState {
bucket: String,
object: String,
fence_pending: tokio::sync::Notify,
arrived: tokio::sync::Notify,
is_paused: AtomicBool,
release: tokio::sync::Notify,
}
@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
}
#[cfg(test)]
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
let state = Arc::new(SourceCleanupDeleteBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
fence_pending: tokio::sync::Notify::new(),
arrived: tokio::sync::Notify::new(),
is_paused: AtomicBool::new(false),
release: tokio::sync::Notify::new(),
});
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
*slot = Some(Arc::clone(&state));
assert!(
!barriers
.iter()
.any(|barrier| barrier.bucket == bucket && barrier.object == object),
"source cleanup delete barrier must be unique per object"
);
barriers.push(Arc::clone(&state));
Self { state }
}
@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
.expect("source cleanup should reach the pre-delete barrier");
}
pub(crate) async fn wait_until_fence_pending(&self) {
tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
.await
.expect("source cleanup should attempt the fixed mutation fence");
}
pub(crate) fn is_paused(&self) -> bool {
self.state.is_paused.load(Ordering::Acquire)
}
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
}
#[cfg(test)]
pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.fence_pending.notify_one();
}
}
#[cfg(test)]
impl Drop for SourceCleanupDeleteBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
}
}
#[cfg(test)]
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.is_paused.store(true, Ordering::Release);
barrier.arrived.notify_one();
barrier.release.notified().await;
}
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
op_label: &str,
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
let cleanup_key = encode_dir_object(object);
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?;
let source_guard = if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::source_lock_covered)
{
None
} else {
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?,
)
};
if bucket_fence
.lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
"{op_label}: bucket incarnation fence was lost before source cleanup"
))));
}
if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::is_lock_lost)
{
return Err(SourceCleanupError::Storage(Error::other(format!(
"{op_label}: object mutation fence was lost before source cleanup"
))));
}
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
..Default::default()
};
opts.add_namespace_lock_guard(&_guard);
if let Some(source_guard) = source_guard.as_ref() {
opts.add_namespace_lock_guard(source_guard);
}
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
object_mutation_fence.add_namespace_lock_fence(&mut opts);
}
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
}
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
}
}
pub(crate) async fn migrate_decommission_object(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
let source = rd.object_info.clone();
let _mutation_fence = store
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
.await?;
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
.await?
.ok_or(Error::FileNotFound)?;
if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
return Err(Error::FileNotFound);
}
migrate_object_inner(
store,
pool_idx,
bucket,
rd,
source_bucket_incarnation_id,
op_label,
Some(&_mutation_fence),
)
.await
}
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
}
async fn migrate_object_inner(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<()> {
let object_info = rd.object_info.clone();
let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.await
{
Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
target_pool_idx,
(target_pool_idx, mutation_fence),
&bucket,
&object_info.name,
&res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.await
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
}
#[test]
fn test_precondition_conflict_rejects_newer_delete_marker() {
let source = ObjectInfo {
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
for version_id in [None, Some(Uuid::nil())] {
let source = ObjectInfo {
version_id,
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
let should_resume =
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
.expect("delete marker conflict should be evaluated");
assert!(
resolve_data_movement_overwrite_resume_result(
&Error::PreconditionFailed,
Ok(Some(target.clone())),
&source,
0,
1,
)
.expect("newer null delete marker should be evaluated")
);
assert!(!should_resume);
let mut same_time = target.clone();
same_time.mod_time = source.mod_time;
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
.expect("same-generation null delete marker should be rejected")
);
let mut versioned = target;
versioned.version_id = Some(Uuid::new_v4());
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
.expect("a UUID delete marker must not erase a null source version")
);
}
}
#[test]