gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-04 12:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(tier): sweep transitioned copies from delete handlers (#2287)

Browse Source
This commit is contained in:
weisd
2026-03-25 16:06:36 +08:00
committed by GitHub
parent fb2ced4d27
commit 41dcebda44
12 changed files with 520 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/lifecycle_transition_api_test.rs
+105 -3
View File
@@ -35,6 +35,7 @@ use rustfs_ecstore::{
warm_backend::{WarmBackend, WarmBackendGetOpts},
},
};
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header};
use s3s::{S3Request, dto::*};
use serial_test::serial;
use std::{
@@ -141,12 +142,17 @@ async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
.expect("Failed to create test bucket");
}
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
async fn upload_test_object(
ecstore: &Arc<ECStore>,
bucket: &str,
object: &str,
data: &[u8],
) -> rustfs_ecstore::store_api::ObjectInfo {
let mut reader = PutObjReader::from_vec(data.to_vec());
(**ecstore)
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("Failed to upload test object");
.expect("Failed to upload test object")
}
async fn set_bucket_lifecycle_transition_with_tier(
@@ -282,6 +288,42 @@ async fn wait_for_transition(
}
}
async fn wait_for_remote_absence(backend: &MockWarmBackend, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if !backend.objects.lock().await.contains_key(object) {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if ecstore
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.is_err()
{
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
@@ -353,7 +395,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
.await
.expect("Failed to set destination lifecycle configuration");
upload_test_object(&ecstore, src_bucket.as_str(), src_object, copy_payload).await;
let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, copy_payload).await;
let copy_input = CopyObjectInput::builder()
.copy_source(CopySource::Bucket {
@@ -437,3 +479,63 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() {
assert_eq!(info.transitioned_object.tier, tier_name);
assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "requires isolated global object layer state"]
async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultObjectUsecase::without_context();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
let bucket = format!("test-api-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/object.txt";
let payload = b"delete transitioned object through delete API";
create_test_bucket(&ecstore, bucket.as_str()).await;
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
.await
.expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket.as_str(),
)
.await
.expect("Failed to enqueue transitioned object");
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("object should transition before delete usecase runs");
let remote_object = transitioned.transitioned_object.name.clone();
assert!(backend.objects.lock().await.contains_key(&remote_object));
let mut req = build_request(
DeleteObjectInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.build()
.unwrap(),
Method::DELETE,
);
insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
usecase
.execute_delete_object(req)
.await
.expect("Failed to delete object through usecase");
assert!(
wait_for_object_absence(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
"object should be removed from hot tier after delete usecase"
);
assert!(
wait_for_remote_absence(&backend, &remote_object, TRANSITION_WAIT_TIMEOUT).await,
"transitioned object should be removed from remote tier after delete usecase"
);
}
rustfs/src/app/object_usecase.rs
+57 -3
View File
@@ -142,6 +142,42 @@ impl Drop for DeadlockRequestGuard {
}
}
async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) {
let Some(existing) = existing else {
return;
};
let je = if opts.delete_prefix {
rustfs_ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object)
} else {
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
rustfs_ecstore::bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
version_id,
opts.versioned,
opts.version_suspended,
&existing.transitioned_object,
)
};
let Some(je) = je else {
return;
};
let mut expiry_state = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
.write()
.await;
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
warn!(
bucket,
object,
remote_object = %existing.transitioned_object.name,
remote_version_id = %existing.transitioned_object.version_id,
tier = %existing.transitioned_object.tier,
error = ?err,
"failed to enqueue transitioned object cleanup"
);
}
}
pin_project! {
struct ExtractArchiveEtagReader<R> {
#[pin]
@@ -2794,6 +2830,7 @@ impl DefaultObjectUsecase {
let mut object_to_delete = Vec::new();
let mut object_to_delete_idx = Vec::new();
let mut object_sizes = Vec::new();
let mut existing_object_infos = Vec::new();
for (idx, obj_id) in delete.objects.iter().enumerate() {
let raw_version_id = obj_id.version_id.clone();
let (version_id, version_uuid) = match normalize_delete_objects_version_id(raw_version_id.clone()) {
@@ -2893,6 +2930,7 @@ impl DefaultObjectUsecase {
object_to_delete_idx.push(idx);
object_to_delete.push(object);
existing_object_infos.push(gerr.is_none().then_some(goi));
}
let (mut dobjs, errs) = store
@@ -2944,6 +2982,18 @@ impl DefaultObjectUsecase {
dobjs[i].replication_state = Some(object_to_delete[i].replication_state());
}
delete_results[didx].delete_object = Some(dobjs[i].clone());
enqueue_transitioned_delete_cleanup(
&bucket,
&object_to_delete[i].object_name,
&ObjectOptions {
version_id: object_to_delete[i].version_id.map(|v| v.to_string()),
versioned: version_cfg.prefix_enabled(object_to_delete[i].object_name.as_str()),
version_suspended: version_cfg.suspended(),
..Default::default()
},
existing_object_infos[i].as_ref(),
)
.await;
let size = object_sizes[i];
if size > 0 {
rustfs_ecstore::data_usage::decrement_bucket_usage_memory(&bucket, size as u64).await;
@@ -3121,7 +3171,7 @@ impl DefaultObjectUsecase {
.await
.map_err(ApiError::from)?;
match store.get_object_info(&bucket, &key, &get_opts).await {
let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await {
Ok(obj_info) => {
// Check for bypass governance retention header (permission already verified in access.rs)
let bypass_governance = has_bypass_governance_header(&req.headers);
@@ -3129,17 +3179,19 @@ impl DefaultObjectUsecase {
if let Some(block_reason) = check_object_lock_for_deletion(&bucket, &obj_info, bypass_governance).await {
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message()));
}
Some(obj_info)
}
Err(err) => {
// If object not found, allow deletion to proceed (will return 204 No Content)
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(ApiError::from(err).into());
}
None
}
}
};
let obj_info = {
match store.delete_object(&bucket, &key, opts).await {
match store.delete_object(&bucket, &key, opts.clone()).await {
Ok(obj) => obj,
Err(err) => {
if is_err_bucket_not_found(&err) {
@@ -3157,6 +3209,8 @@ impl DefaultObjectUsecase {
}
};
enqueue_transitioned_delete_cleanup(&bucket, &key, &opts, existing_object_info.as_ref()).await;
// Fast in-memory update for immediate quota consistency
rustfs_ecstore::data_usage::decrement_bucket_usage_memory(&bucket, obj_info.size as u64).await;