gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 02:08:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): own publication mutations through storage drain (#6867)

Browse Source 
* fix(scanner): own publication mutations through storage drain

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(storage): remove unused rename data shim

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-30 10:39:07 +08:00
committed by GitHub
parent 90ab2e24c3
commit ee39e4fccb
15 changed files with 1360 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/ops/object.rs
+95 -7
View File
@@ -66,6 +66,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps;
use crate::bucket::utils::is_meta_bucketname;
use crate::bucket::versioning::VersioningApi;
use crate::disk::DiskAPI;
use crate::object_api::ScannerPublicationCommitScopeGuard;
use crate::set_disk::coding;
use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome;
use crate::set_disk::mem;
@@ -272,6 +273,22 @@ const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.js
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
fn begin_scanner_publication_delete_mutation(scope: Option<&crate::object_api::ScannerPublicationCommitScope>) -> Result<()> {
let Some(scope) = scope else {
return Ok(());
};
if scope.state() == crate::object_api::ScannerPublicationCommitState::Admitted {
scope
.try_begin()
.map_err(|_| Error::other("scanner publication delete scope cannot start"))?;
}
if !scope.can_commit() {
let _ = scope.mark_indeterminate();
return Err(StorageError::OperationCanceled);
}
Ok(())
}
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
return Ok(None);
@@ -2627,6 +2644,10 @@ impl SetDisks {
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
crate::hp_guard!("SetDisks::put_object");
let mut scope_outcome_guard = opts
.scanner_publication_commit_scope
.clone()
.map(ScannerPublicationCommitScopeGuard::new);
let storage_class_config = self.storage_class_config_snapshot();
self.invalidate_get_object_metadata_cache(bucket, object).await;
@@ -3397,8 +3418,16 @@ impl SetDisks {
let commit_tmp_dir = tmp_dir.clone();
let commit_object_lock_guard = object_lock_guard.take();
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let commit_allows_early_ack = commit_object_lock_guard.is_some();
let detach_commit_owner = commit_allows_early_ack || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence;
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
// A scanner publication scope owns the movement permit until the
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
// remote leases.
let commit_allows_early_ack = commit_object_lock_guard.is_some() && commit_scanner_publication_scope.is_none();
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|| commit_allows_early_ack
|| commit_bucket_lifecycle_guard.is_some()
|| quota_mutation_fence;
let commit_write_path_label = write_path.metric_label();
let commit_is_versioned = opts.versioned || opts.version_suspended;
let commit_versioned = opts.versioned;
@@ -3494,7 +3523,7 @@ impl SetDisks {
}
Ok(())
};
let pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
let mut pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
tokio::select! {
biased;
_ = wait_for_put_object_commit_cancellation(cancellation.as_ref(), request_cancellation.as_ref()) => {
@@ -3505,6 +3534,20 @@ impl SetDisks {
} else {
pre_rename.await
};
if pre_rename_result.is_ok()
&& let Some(scope) = commit_scanner_publication_scope.as_ref()
&& let Err(err) = scope.try_begin()
{
let _ = scope.mark_aborted_before_commit();
pre_rename_result = Err(Error::other(format!("scanner publication commit scope cannot start: {err:?}")));
}
if pre_rename_result.is_ok()
&& let Some(scope) = commit_scanner_publication_scope.as_ref()
&& !scope.can_commit()
{
let _ = scope.mark_indeterminate();
pre_rename_result = Err(StorageError::OperationCanceled);
}
if let Err(err) = pre_rename_result {
SetDisks::abort_quota_reservation_after_fence(
quota_reservation,
@@ -3540,9 +3583,17 @@ impl SetDisks {
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
),
)
.with_publication_scope(commit_scanner_publication_scope.clone()),
)
.await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
if rename_result.is_ok() {
let _ = scope.mark_committed();
} else {
let _ = scope.mark_indeterminate();
}
}
#[cfg(any(test, feature = "test-util"))]
if rename_result.is_ok() {
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
@@ -3857,6 +3908,11 @@ impl SetDisks {
let _ = handoff.send(());
}
if detach_commit_owner {
if let Some(scope_outcome_guard) = scope_outcome_guard.as_mut() {
// The spawned commit closure owns the scope clone and is
// now responsible for its terminal outcome.
scope_outcome_guard.disarm();
}
let mut cancellation = PutObjectCommitCancellation::new();
let child_token = cancellation.child_token();
let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await })
@@ -7054,6 +7110,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self, opts))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
let _scope_outcome_guard = opts
.scanner_publication_commit_scope
.clone()
.map(ScannerPublicationCommitScopeGuard::new);
let scanner_publication_commit_scope = opts.scanner_publication_commit_scope.clone();
// Scanner cleanup carries the per-peer lease fence as transient
// request metadata. Consume it before any delete-prefix fanout so it
// cannot be persisted or treated as user metadata.
@@ -7148,6 +7209,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
delete_request.set_skip_tier_free_version();
}
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
@@ -7162,6 +7224,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
}
for version in &versions.free_versions {
@@ -7173,10 +7236,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
delete_request.set_tier_free_version();
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
}
}
}
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
return Ok(ObjectInfo::default());
}
@@ -7184,10 +7251,19 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_prefix_with_scanner_publication_lease(
bucket,
object,
scanner_publication_lease_tokens.as_ref(),
scanner_publication_commit_scope.clone(),
)
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
self.invalidate_all_get_object_metadata_cache();
return Ok(ObjectInfo::default());
}
@@ -7260,10 +7336,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &dfi, false)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
return Ok(ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended));
}
@@ -7337,6 +7417,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
};
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts))
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7348,6 +7429,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
oi.user_tags = Arc::clone(&goi.user_tags);
oi.replication_decision = goi.replication_decision;
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
return Ok(oi);
}
@@ -7373,6 +7457,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &dfi, opts.delete_marker)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7398,6 +7483,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
obj_info.delete_marker = true;
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
Ok(obj_info)
}