mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-07 13:53:12 +00:00
fix(replication): harden live delete admission (#5599)
This commit is contained in:
@@ -34,8 +34,8 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
replication::{
|
||||
ReplicationTargetValidationError, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
ReplicationTargetValidationError, invalid_replication_config_status_field, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
},
|
||||
target::{BucketTargetType, BucketTargets},
|
||||
utils::serialize,
|
||||
@@ -90,9 +90,9 @@ use rustfs_utils::http::{SUFFIX_FORCE_DELETE, get_header};
|
||||
use rustfs_utils::obj::extract_user_defined_metadata;
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, BucketLocationConstraint, CommonPrefix, CreateBucketInput, CreateBucketOutput,
|
||||
DeleteBucketCorsInput, DeleteBucketCorsOutput, DeleteBucketEncryptionInput, DeleteBucketEncryptionOutput, DeleteBucketInput,
|
||||
DeleteBucketLifecycleInput, DeleteBucketLifecycleOutput, DeleteBucketOutput, DeleteBucketPolicyInput,
|
||||
BucketLifecycleConfiguration, BucketLocationConstraint, BucketVersioningStatus, CommonPrefix, CreateBucketInput,
|
||||
CreateBucketOutput, DeleteBucketCorsInput, DeleteBucketCorsOutput, DeleteBucketEncryptionInput, DeleteBucketEncryptionOutput,
|
||||
DeleteBucketInput, DeleteBucketLifecycleInput, DeleteBucketLifecycleOutput, DeleteBucketOutput, DeleteBucketPolicyInput,
|
||||
DeleteBucketPolicyOutput, DeleteBucketReplicationInput, DeleteBucketReplicationOutput, DeleteBucketTaggingInput,
|
||||
DeleteBucketTaggingOutput, DeleteMarkerEntry, DeletePublicAccessBlockInput, DeletePublicAccessBlockOutput, EncodingType,
|
||||
ExpirationStatus, GetBucketCorsInput, GetBucketCorsOutput, GetBucketEncryptionInput, GetBucketEncryptionOutput,
|
||||
@@ -557,6 +557,12 @@ fn validate_replication_config_targets(targets: &BucketTargets, config: &Replica
|
||||
}
|
||||
|
||||
fn validate_replication_config_capabilities(config: &ReplicationConfiguration) -> S3Result<()> {
|
||||
if let Some(field) = invalid_replication_config_status_field(config) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("replication field {field} has an invalid status"),
|
||||
));
|
||||
}
|
||||
if let Some(field) = unsupported_replication_config_field(config) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
@@ -677,6 +683,16 @@ fn versioning_configuration_has_object_lock_incompatible_settings(config: &Versi
|
||||
}
|
||||
|
||||
async fn validate_bucket_versioning_update(bucket: &str, config: &VersioningConfiguration) -> S3Result<()> {
|
||||
if config
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"bucket versioning configuration has an invalid status",
|
||||
));
|
||||
}
|
||||
match metadata_sys::get_object_lock_config(bucket).await {
|
||||
Ok((object_lock_config, _)) => {
|
||||
if object_lock_config.enabled() && versioning_configuration_has_object_lock_incompatible_settings(config) {
|
||||
@@ -3106,6 +3122,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_replication_config_capabilities_rejects_invalid_status_before_write() {
|
||||
let mut rule = replication_rule_for_target("arn:rustfs:replication:us-east-1:target:bucket");
|
||||
rule.status = ReplicationRuleStatus::from_static("Invalid");
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
|
||||
let err = validate_replication_config_capabilities(&config)
|
||||
.expect_err("an invalid string-backed replication status must be rejected before persistence");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert!(err.to_string().contains("Rule.Status has an invalid status"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_bucket_versioning_update_rejects_invalid_status_before_metadata_lookup() {
|
||||
let config = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static("Invalid")),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_bucket_versioning_update("unregistered-test-bucket", &config)
|
||||
.await
|
||||
.expect_err("an invalid string-backed versioning status must be rejected before persistence");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_replication_targets_from_config_targets_only_removes_referenced_replication_targets() {
|
||||
let removed_arn = "arn:rustfs:replication:us-east-1:removed:bucket";
|
||||
|
||||
@@ -40,7 +40,7 @@ use futures::stream;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri, header::IF_NONE_MATCH};
|
||||
use rustfs_config::{ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header};
|
||||
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, insert_header};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use s3s::{S3Request, dto::*};
|
||||
use serial_test::serial;
|
||||
@@ -2498,6 +2498,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
use super::storage_api::test::bucket::replication::DELETE_CONFIG_SNAPSHOT_LOADS;
|
||||
use super::storage_api::test::{ReqInfo, VERSIONING_CONFIG_LOOKUPS};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
@@ -2536,6 +2537,7 @@ async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
});
|
||||
|
||||
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
|
||||
let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
@@ -2551,12 +2553,573 @@ async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
);
|
||||
|
||||
let lookups = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst) - lookups_before;
|
||||
let snapshots = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst) - snapshots_before;
|
||||
assert_eq!(
|
||||
lookups,
|
||||
snapshots,
|
||||
1,
|
||||
"DeleteObjects must resolve bucket versioning exactly once per request; got {lookups} lookups for {} keys",
|
||||
"DeleteObjects must resolve delete admission config exactly once per request; got {snapshots} snapshots for {} keys",
|
||||
keys.len()
|
||||
);
|
||||
assert_eq!(lookups, 0, "DeleteObjects must not fall back to legacy per-key versioning lookups");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_treats_empty_directory_version_as_absent() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delobjs-empty-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "directory/";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, b"").await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: Some(" \t ".to_string()),
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect("empty VersionId should be treated as an absent version");
|
||||
let deleted = response
|
||||
.output
|
||||
.deleted
|
||||
.expect("delete response should contain the directory entry");
|
||||
|
||||
assert_eq!(deleted.len(), 1);
|
||||
assert_eq!(
|
||||
deleted[0].delete_marker, None,
|
||||
"the synthetic null version must be purged, not hidden by a new marker"
|
||||
);
|
||||
assert_eq!(deleted[0].version_id, None, "the synthetic null sentinel must stay off the wire");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_preserves_explicit_null_directory_version() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delobjs-null-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "directory/";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, b"").await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: Some("null".to_string()),
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect("explicit null VersionId should delete the directory's null version");
|
||||
let deleted = response
|
||||
.output
|
||||
.deleted
|
||||
.expect("delete response should contain the directory entry");
|
||||
|
||||
assert_eq!(deleted.len(), 1);
|
||||
assert_eq!(deleted[0].delete_marker, None);
|
||||
assert_eq!(deleted[0].version_id.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
async fn install_malformed_versioning_config(bucket: &str) {
|
||||
use super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before corruption injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
|
||||
metadata.versioning_config = None;
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("malformed versioning metadata should be installed for the request test");
|
||||
}
|
||||
|
||||
async fn install_malformed_replication_config(bucket: &str) {
|
||||
use super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before corruption injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
metadata.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
|
||||
metadata.replication_config = None;
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("malformed replication metadata should be installed for the request test");
|
||||
}
|
||||
|
||||
async fn install_delete_replication_config(
|
||||
bucket: &str,
|
||||
delete_enabled: bool,
|
||||
replica_modifications: bool,
|
||||
required_tag: Option<(&str, &str)>,
|
||||
) {
|
||||
use super::storage_api::test::{bucket::utils::serialize, get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before replication config injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
let target = "arn:aws:s3:::target-bucket";
|
||||
let delete_status = if delete_enabled {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
} else {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
};
|
||||
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
metadata.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
let filter = required_tag.map(|(key, value)| ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some(key.to_string()),
|
||||
value: Some(value.to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
let source_selection_criteria = replica_modifications.then(|| SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication_config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication { status: delete_status }),
|
||||
destination: Destination {
|
||||
bucket: target.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter,
|
||||
id: Some("delete".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
};
|
||||
metadata.replication_config_xml = serialize(&replication_config).expect("replication test config should serialize");
|
||||
metadata.replication_config = Some(replication_config);
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("delete replication metadata should be installed for the request test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_object_versioning_config_failure_leaves_latest_object_intact() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-versioning-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"local delete must not commit before versioning admission";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_versioning_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("versioning config failure must reject DeleteObject");
|
||||
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_object_replication_config_failure_leaves_latest_object_intact() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"local delete must not commit before replication admission";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObject");
|
||||
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn delete_object_uses_one_replication_config_generation() {
|
||||
use super::object_usecase::install_delete_snapshot_test_hook;
|
||||
use super::storage_api::test::ReqInfo;
|
||||
use super::storage_api::test::bucket::replication::take_scheduled_replication_deletes;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-config-generation-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"generation-bound delete".to_vec());
|
||||
let uploaded = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be uploaded");
|
||||
let version_id = uploaded.version_id.expect("versioned upload should return a version ID");
|
||||
let version_id_string = version_id.to_string();
|
||||
|
||||
install_delete_replication_config(&bucket, true, false, None).await;
|
||||
assert!(take_scheduled_replication_deletes().is_empty());
|
||||
|
||||
let loaded = Arc::new(Barrier::new(2));
|
||||
let resume = Arc::new(Barrier::new(2));
|
||||
install_delete_snapshot_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.version_id(Some(version_id_string.clone()))
|
||||
.build()
|
||||
.expect("version delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let update_config = async {
|
||||
loaded.wait().await;
|
||||
install_delete_replication_config(&bucket, false, false, None).await;
|
||||
resume.wait().await;
|
||||
};
|
||||
let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_config);
|
||||
let response = response.expect("delete admitted by the first config generation should succeed");
|
||||
|
||||
assert_eq!(response.output.version_id.as_deref(), Some(version_id_string.as_str()));
|
||||
let scheduled = take_scheduled_replication_deletes();
|
||||
assert_eq!(scheduled.len(), 1, "the first config generation must control post-delete queueing");
|
||||
assert_eq!(scheduled[0].version_id, Some(version_id));
|
||||
let state = scheduled[0]
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.expect("admitted version purge should carry replication state");
|
||||
assert_eq!(
|
||||
state.version_purge_status_internal.as_deref(),
|
||||
Some("arn:aws:s3:::target-bucket=PENDING;")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replica_delete_uses_authorized_source_version_and_its_tags() {
|
||||
use super::object_usecase::install_delete_source_test_hook;
|
||||
use super::storage_api::test::ReqInfo;
|
||||
use super::storage_api::test::bucket::replication::{ReplicationStatusType, take_scheduled_replication_deletes};
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::{SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID};
|
||||
use std::collections::HashMap;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-replica-source-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
install_delete_replication_config(&bucket, true, true, Some(("generation", "updated"))).await;
|
||||
|
||||
let mut old_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=old".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
old_opts.set_replica_status(ReplicationStatusType::Replica);
|
||||
let mut old_reader = PutObjReader::from_vec(b"old replica version".to_vec());
|
||||
let old = ecstore
|
||||
.put_object(&bucket, object, &mut old_reader, &old_opts)
|
||||
.await
|
||||
.expect("old replica version should be uploaded");
|
||||
let old_version_id = old.version_id.expect("old replica version should have a version ID");
|
||||
let old_version_id_string = old_version_id.to_string();
|
||||
|
||||
let mut latest_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=latest".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
latest_opts.set_replica_status(ReplicationStatusType::Replica);
|
||||
let mut latest_reader = PutObjReader::from_vec(b"latest replica version".to_vec());
|
||||
let latest = ecstore
|
||||
.put_object(&bucket, object, &mut latest_reader, &latest_opts)
|
||||
.await
|
||||
.expect("latest replica version should be uploaded");
|
||||
assert_ne!(latest.version_id, Some(old_version_id));
|
||||
assert!(take_scheduled_replication_deletes().is_empty());
|
||||
|
||||
let source_loaded = Arc::new(Barrier::new(2));
|
||||
let resume_delete = Arc::new(Barrier::new(2));
|
||||
install_delete_source_test_hook(bucket.clone(), Arc::clone(&source_loaded), Arc::clone(&resume_delete));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("replica delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.headers
|
||||
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
|
||||
insert_header(&mut req.headers, SUFFIX_SOURCE_VERSION_ID, old_version_id_string.clone());
|
||||
insert_header(&mut req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let update_tags = async {
|
||||
source_loaded.wait().await;
|
||||
ecstore
|
||||
.put_object_tags(
|
||||
&bucket,
|
||||
object,
|
||||
"generation=updated",
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(old_version_id_string.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tag update ordered before the locked delete should succeed");
|
||||
resume_delete.wait().await;
|
||||
};
|
||||
let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_tags);
|
||||
let response = response.expect("authorized replica version delete should succeed");
|
||||
|
||||
assert_eq!(response.output.version_id.as_deref(), Some(old_version_id_string.as_str()));
|
||||
let scheduled = take_scheduled_replication_deletes();
|
||||
assert_eq!(
|
||||
scheduled.len(),
|
||||
1,
|
||||
"the tag committed after the advisory pre-read must control downstream fanout"
|
||||
);
|
||||
assert_eq!(scheduled[0].version_id, Some(old_version_id));
|
||||
assert_eq!(
|
||||
scheduled[0]
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.and_then(|state| state.version_purge_status_internal.as_deref()),
|
||||
Some("arn:aws:s3:::target-bucket=PENDING;")
|
||||
);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, b"latest replica version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn force_delete_replication_config_failure_leaves_prefix_intact() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-force-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let prefix = "prefix/";
|
||||
let object = "prefix/object.txt";
|
||||
let payload = b"force delete must validate replication metadata before deleting descendants";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(prefix.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject force delete");
|
||||
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replica_delete_replication_config_failure_leaves_object_intact() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-replica-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"incoming replica delete must use the admitted request snapshot";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.headers
|
||||
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject incoming replica delete");
|
||||
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_objects_replication_config_failure_does_not_call_storage_delete() {
|
||||
use super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-repls-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"batch delete must fail closed on replication metadata errors";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObjects");
|
||||
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
/// Regression pin for the nonexistent-bucket fast path: GET/HEAD/DeleteObject
|
||||
|
||||
+268
-263
@@ -41,11 +41,10 @@ use super::storage_api::object_usecase::bucket::{
|
||||
predict_lifecycle_expiration,
|
||||
quota::{QuotaCheckResult, QuotaError, QuotaOperation},
|
||||
replication::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicationStatusType, VersionPurgeStatusType, check_replicate_delete,
|
||||
delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
|
||||
must_replicate_object, schedule_object_replication, schedule_replication_delete, set_deleted_object_replication_state,
|
||||
set_object_to_delete_version_purge_status, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, delete_replication_state_from_config,
|
||||
delete_replication_version_id, deleted_object_has_pending_replication_delete, has_active_delete_rule,
|
||||
load_delete_config_snapshot, must_replicate_object, schedule_object_replication, schedule_replication_delete,
|
||||
set_deleted_object_replication_state, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
},
|
||||
tagging::decode_tags,
|
||||
validate_restore_request,
|
||||
@@ -81,9 +80,9 @@ use super::storage_api::object_usecase::object_cache::lookup_get_object_body_cac
|
||||
use super::storage_api::object_usecase::object_cache::{GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len};
|
||||
use super::storage_api::object_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::object_usecase::options::{
|
||||
bucket_versioning_config, copy_dst_opts, copy_src_opts, del_opts, del_opts_with_versioning, extract_metadata,
|
||||
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts,
|
||||
namespace_reserved_user_metadata, normalize_content_encoding_for_storage, put_opts, validate_archive_content_encoding,
|
||||
copy_dst_opts, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime_with_object_name,
|
||||
filter_object_metadata, get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata,
|
||||
normalize_content_encoding_for_storage, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
use super::storage_api::object_usecase::request_context::{self, spawn_traced};
|
||||
use super::storage_api::object_usecase::s3_api::multipart::parse_list_parts_params;
|
||||
@@ -100,10 +99,10 @@ use super::storage_api::object_usecase::storage_class as storageclass;
|
||||
use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
use super::storage_api::object_usecase::{ECStore, OldCurrentSize};
|
||||
use super::storage_api::object_usecase::{
|
||||
RFC1123, check_preconditions, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
|
||||
parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata,
|
||||
validate_bucket_exists, validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
RFC1123, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists,
|
||||
validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write,
|
||||
validate_ssec_for_read, wrap_response_with_cors,
|
||||
};
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_expiry_state_handle, current_notify_interface_for_context,
|
||||
@@ -115,14 +114,14 @@ use crate::error::ApiError;
|
||||
use crate::server::convert_ecstore_object_info;
|
||||
use crate::table_catalog;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use futures::{Stream, StreamExt, TryStreamExt};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use metrics::{counter, histogram};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::{RestoreStatusOps, parse_restore_obj_status};
|
||||
use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, parse_restore_obj_status};
|
||||
use rustfs_io_core::{BytesPool, PooledBuffer};
|
||||
use rustfs_io_metrics;
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
@@ -2519,6 +2518,76 @@ fn normalize_delete_objects_version_id(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type DeleteSnapshotTestHook = (String, Arc<tokio::sync::Barrier>, Arc<tokio::sync::Barrier>);
|
||||
|
||||
#[cfg(test)]
|
||||
static DELETE_SNAPSHOT_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>> = OnceLock::new();
|
||||
#[cfg(test)]
|
||||
static DELETE_SOURCE_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_snapshot_test_hook(
|
||||
bucket: String,
|
||||
loaded: Arc<tokio::sync::Barrier>,
|
||||
resume: Arc<tokio::sync::Barrier>,
|
||||
) {
|
||||
*DELETE_SNAPSHOT_TEST_HOOK
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete snapshot test hook lock should not be poisoned") = Some((bucket, loaded, resume));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn wait_for_delete_snapshot_test_hook(bucket: &str) {
|
||||
let hook = {
|
||||
let mut slot = DELETE_SNAPSHOT_TEST_HOOK
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete snapshot test hook lock should not be poisoned");
|
||||
if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
|
||||
slot.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some((_bucket, loaded, resume)) = hook {
|
||||
loaded.wait().await;
|
||||
resume.wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_source_test_hook(
|
||||
bucket: String,
|
||||
loaded: Arc<tokio::sync::Barrier>,
|
||||
resume: Arc<tokio::sync::Barrier>,
|
||||
) {
|
||||
*DELETE_SOURCE_TEST_HOOK
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete source test hook lock should not be poisoned") = Some((bucket, loaded, resume));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn wait_for_delete_source_test_hook(bucket: &str) {
|
||||
let hook = {
|
||||
let mut slot = DELETE_SOURCE_TEST_HOOK
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete source test hook lock should not be poisoned");
|
||||
if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
|
||||
slot.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some((_bucket, loaded, resume)) = hook {
|
||||
loaded.wait().await;
|
||||
resume.wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String> {
|
||||
if !event.action.delete() {
|
||||
return None;
|
||||
@@ -2534,8 +2603,8 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
|
||||
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
|
||||
}
|
||||
|
||||
async fn enrich_delete_replication_state_if_needed(
|
||||
bucket: &str,
|
||||
fn enrich_delete_replication_state_if_needed(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
delete_object: &mut StorageDeletedObject,
|
||||
obj_info: &ObjectInfo,
|
||||
) {
|
||||
@@ -2549,7 +2618,7 @@ async fn enrich_delete_replication_state_if_needed(
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok((config, _)) = metadata_sys::get_replication_config(bucket).await else {
|
||||
let Some(config) = snapshot.replication_config() else {
|
||||
return;
|
||||
};
|
||||
let version_id = if delete_object.delete_marker {
|
||||
@@ -2560,7 +2629,7 @@ async fn enrich_delete_replication_state_if_needed(
|
||||
delete_object.version_id
|
||||
};
|
||||
if let Some(local_state) = delete_replication_state_from_config(
|
||||
&config,
|
||||
config,
|
||||
obj_info,
|
||||
version_id,
|
||||
obj_info.replication_status == ReplicationStatusType::Replica,
|
||||
@@ -2569,16 +2638,16 @@ async fn enrich_delete_replication_state_if_needed(
|
||||
}
|
||||
}
|
||||
|
||||
async fn should_schedule_replica_delete_replication(
|
||||
bucket: &str,
|
||||
fn should_schedule_replica_delete_replication(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
replication_source: &ObjectInfo,
|
||||
version_id: Option<Uuid>,
|
||||
) -> bool {
|
||||
let Ok((config, _)) = metadata_sys::get_replication_config(bucket).await else {
|
||||
let Some(config) = snapshot.replication_config() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
delete_replication_state_from_config(&config, replication_source, version_id, true).is_some()
|
||||
delete_replication_state_from_config(config, replication_source, version_id, true).is_some()
|
||||
}
|
||||
|
||||
fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
|
||||
@@ -2635,23 +2704,6 @@ where
|
||||
.map_err(|err| ApiError::from(copy_namespace_lock_error(bucket, &object, "write", err)).into())
|
||||
}
|
||||
|
||||
fn delete_replication_state_source<'a>(
|
||||
opts: &ObjectOptions,
|
||||
existing_object_info: Option<&'a ObjectInfo>,
|
||||
deleted_object_info: &'a ObjectInfo,
|
||||
) -> &'a ObjectInfo {
|
||||
if should_use_existing_delete_replication_source(
|
||||
opts.replication_request,
|
||||
deleted_object_info.delete_marker,
|
||||
existing_object_info.is_some(),
|
||||
) && let Some(existing) = existing_object_info
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
deleted_object_info
|
||||
}
|
||||
|
||||
const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract";
|
||||
#[cfg(test)]
|
||||
const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix";
|
||||
@@ -3030,8 +3082,8 @@ const DELETE_OBJECTS_PRE_STAT_CONCURRENCY: usize = 16;
|
||||
/// - the app-layer object-lock admission check never runs for deletes that
|
||||
/// create a delete marker, and non-lock buckets cannot hold retention or
|
||||
/// legal-hold metadata (`bucket_lock_enabled == false`);
|
||||
/// - the replication delete decision is only consulted when the bucket has
|
||||
/// active replication rules for the batch (`replicate_deletes == false`);
|
||||
/// - replication reads its authoritative source metadata later while the
|
||||
/// SetDisks write lock is held, so it does not consume this advisory stat;
|
||||
/// - usage accounting for delete-marker creation goes through
|
||||
/// `record_bucket_delete_marker_memory` and never reads the object size
|
||||
/// (`accounting_creates_delete_marker` is computed from the same versioning
|
||||
@@ -3044,11 +3096,10 @@ const DELETE_OBJECTS_PRE_STAT_CONCURRENCY: usize = 16;
|
||||
/// byte-for-byte the pre-#929 one (see PR #4297).
|
||||
fn can_skip_delete_objects_pre_stat(
|
||||
bucket_lock_enabled: bool,
|
||||
replicate_deletes: bool,
|
||||
opts: &ObjectOptions,
|
||||
accounting_creates_delete_marker: bool,
|
||||
) -> bool {
|
||||
!bucket_lock_enabled && !replicate_deletes && delete_creates_delete_marker(opts) && accounting_creates_delete_marker
|
||||
!bucket_lock_enabled && delete_creates_delete_marker(opts) && accounting_creates_delete_marker
|
||||
}
|
||||
|
||||
fn complete_delete_noop(
|
||||
@@ -3070,6 +3121,16 @@ fn complete_delete_noop(
|
||||
(result, helper)
|
||||
}
|
||||
|
||||
fn delete_response_version_id(version_id: Option<Uuid>, synthetic_version_id: bool) -> Option<String> {
|
||||
if synthetic_version_id {
|
||||
None
|
||||
} else if version_id == Some(Uuid::nil()) {
|
||||
Some(NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
version_id.map(|version_id| version_id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
|
||||
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
|
||||
.map(|value| normalize_snowball_prefix(&value))
|
||||
@@ -6553,24 +6614,10 @@ impl DefaultObjectUsecase {
|
||||
));
|
||||
}
|
||||
|
||||
let replicate_deletes = has_replication_rules(
|
||||
&bucket,
|
||||
&delete
|
||||
.objects
|
||||
.iter()
|
||||
.map(|v| ObjectToDelete {
|
||||
object_name: v.key.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<ObjectToDelete>>(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let version_cfg = bucket_versioning_config(&bucket).await;
|
||||
let bypass_governance = has_bypass_governance_header(&req.headers);
|
||||
let bucket_lock_enabled = bucket_object_locking_enabled(&bucket).await;
|
||||
|
||||
@@ -6578,22 +6625,18 @@ impl DefaultObjectUsecase {
|
||||
struct DeleteResult {
|
||||
delete_object: Option<StorageDeletedObject>,
|
||||
error: Option<s3s::dto::Error>,
|
||||
synthetic_version_id: bool,
|
||||
}
|
||||
|
||||
let mut delete_results = vec![DeleteResult::default(); delete.objects.len()];
|
||||
|
||||
struct PreparedDelete {
|
||||
struct AuthorizedDelete {
|
||||
idx: usize,
|
||||
object: ObjectToDelete,
|
||||
opts: ObjectOptions,
|
||||
version_id: Option<String>,
|
||||
skip_stat: bool,
|
||||
}
|
||||
|
||||
// Phase 1 (serial): request-scoped validation and authorization. These
|
||||
// steps mutate the request info between authorization calls, so they
|
||||
// stay sequential; they perform no per-object disk I/O.
|
||||
let mut prepared_deletes: Vec<PreparedDelete> = Vec::with_capacity(delete.objects.len());
|
||||
let mut authorized_deletes = Vec::with_capacity(delete.objects.len());
|
||||
for (idx, obj_id) in delete.objects.iter().enumerate() {
|
||||
let raw_version_id = obj_id.version_id.clone();
|
||||
let (version_id, version_uuid) = match normalize_delete_objects_version_id(raw_version_id.clone()) {
|
||||
@@ -6650,22 +6693,62 @@ impl DefaultObjectUsecase {
|
||||
continue;
|
||||
}
|
||||
|
||||
let synthetic_version_id = version_id.is_none() && is_dir_object(&obj_id.key);
|
||||
let object = ObjectToDelete {
|
||||
object_name: obj_id.key.clone(),
|
||||
version_id: version_uuid,
|
||||
synthetic_version_id,
|
||||
..Default::default()
|
||||
};
|
||||
delete_results[idx].synthetic_version_id = synthetic_version_id;
|
||||
|
||||
authorized_deletes.push(AuthorizedDelete { idx, object, version_id });
|
||||
}
|
||||
|
||||
if authorized_deletes.is_empty() {
|
||||
let output = DeleteObjectsOutput {
|
||||
deleted: Some(Vec::new()),
|
||||
errors: Some(delete_results.into_iter().filter_map(|result| result.error).collect()),
|
||||
..Default::default()
|
||||
};
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
|
||||
let delete_config_snapshot = Arc::new(
|
||||
load_delete_config_snapshot(store.as_ref(), &bucket)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
let version_cfg = delete_config_snapshot.versioning_config();
|
||||
let replicate_deletes = authorized_deletes
|
||||
.iter()
|
||||
.any(|authorized| has_active_delete_rule(&delete_config_snapshot, &authorized.object.object_name));
|
||||
|
||||
struct PreparedDelete {
|
||||
idx: usize,
|
||||
object: ObjectToDelete,
|
||||
opts: ObjectOptions,
|
||||
version_id: Option<String>,
|
||||
skip_stat: bool,
|
||||
}
|
||||
|
||||
// Phase 1 (serial): derive storage options from the request-scoped
|
||||
// configuration after every candidate has passed authorization.
|
||||
let mut prepared_deletes: Vec<PreparedDelete> = Vec::with_capacity(authorized_deletes.len());
|
||||
for authorized in authorized_deletes {
|
||||
let AuthorizedDelete { idx, object, version_id } = authorized;
|
||||
|
||||
let metadata = extract_metadata(&req.headers);
|
||||
// Reuse the request-level versioning config fetched above instead of
|
||||
// re-resolving it per key (up to 1000 identical lookups per request).
|
||||
let opts: ObjectOptions = del_opts_with_versioning(
|
||||
&bucket,
|
||||
&object.object_name,
|
||||
object.version_id.map(|f| f.to_string()),
|
||||
&req.headers,
|
||||
metadata,
|
||||
&version_cfg,
|
||||
version_cfg,
|
||||
false,
|
||||
)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
@@ -6673,11 +6756,8 @@ impl DefaultObjectUsecase {
|
||||
// decides delete-marker vs object-delete from this exact snapshot,
|
||||
// so evaluate it here with the same inputs to keep the stat-skip
|
||||
// decision and the accounting path provably consistent.
|
||||
let accounting_creates_delete_marker = object.version_id.is_none()
|
||||
&& version_cfg.prefix_enabled(object.object_name.as_str())
|
||||
&& !version_cfg.suspended();
|
||||
let skip_stat =
|
||||
can_skip_delete_objects_pre_stat(bucket_lock_enabled, replicate_deletes, &opts, accounting_creates_delete_marker);
|
||||
let accounting_creates_delete_marker = object.version_id.is_none() && opts.versioned && !opts.version_suspended;
|
||||
let skip_stat = can_skip_delete_objects_pre_stat(bucket_lock_enabled, &opts, accounting_creates_delete_marker);
|
||||
|
||||
prepared_deletes.push(PreparedDelete {
|
||||
idx,
|
||||
@@ -6691,6 +6771,8 @@ impl DefaultObjectUsecase {
|
||||
struct AdmittedDelete {
|
||||
idx: usize,
|
||||
object: ObjectToDelete,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
size: i64,
|
||||
existing: Option<ObjectInfo>,
|
||||
blocked: Option<s3s::dto::Error>,
|
||||
@@ -6705,7 +6787,7 @@ impl DefaultObjectUsecase {
|
||||
// the same early, advisory rejection as before.
|
||||
let store_ref = &store;
|
||||
let bucket_ref = bucket.as_str();
|
||||
let admitted_deletes: Vec<Result<AdmittedDelete, ApiError>> =
|
||||
let admitted_deletes: Vec<AdmittedDelete> =
|
||||
futures::stream::iter(prepared_deletes.into_iter().map(|prepared| async move {
|
||||
let PreparedDelete {
|
||||
idx,
|
||||
@@ -6714,18 +6796,21 @@ impl DefaultObjectUsecase {
|
||||
version_id,
|
||||
skip_stat,
|
||||
} = prepared;
|
||||
|
||||
let (goi, gerr) = if skip_stat {
|
||||
(ObjectInfo::default(), None)
|
||||
let synthetic_version_id = object.version_id.is_none() && is_dir_object(&object.object_name);
|
||||
let (goi, source_missing) = if skip_stat {
|
||||
(ObjectInfo::default(), false)
|
||||
} else {
|
||||
match store_ref.get_object_info(bucket_ref, &object.object_name, &opts).await {
|
||||
Ok(res) => (res, None),
|
||||
Err(e) => (ObjectInfo::default(), Some(e.to_string())),
|
||||
Ok(res) => (res, false),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
(ObjectInfo::default(), true)
|
||||
}
|
||||
Err(err) => return Err(ApiError::from(err)),
|
||||
}
|
||||
};
|
||||
|
||||
if !skip_stat
|
||||
&& gerr.is_none()
|
||||
&& !source_missing
|
||||
&& !delete_creates_delete_marker(&opts)
|
||||
&& let Some(block_reason) = check_object_lock_for_deletion(bucket_ref, &goi, bypass_governance).await
|
||||
{
|
||||
@@ -6733,6 +6818,8 @@ impl DefaultObjectUsecase {
|
||||
return Ok(AdmittedDelete {
|
||||
idx,
|
||||
object,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
size: 0,
|
||||
existing: None,
|
||||
blocked: Some(s3s::dto::Error {
|
||||
@@ -6746,48 +6833,24 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let size = goi.size;
|
||||
|
||||
if replicate_deletes {
|
||||
let dsc = check_replicate_delete(
|
||||
bucket_ref,
|
||||
&ObjectToDelete {
|
||||
object_name: object.object_name.clone(),
|
||||
version_id: object.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&goi,
|
||||
&opts,
|
||||
gerr.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
if dsc.replicate_any() {
|
||||
if object.version_id.is_some() {
|
||||
set_object_to_delete_version_purge_status(&mut object, VersionPurgeStatusType::Pending);
|
||||
object.version_purge_statuses = dsc.pending_status();
|
||||
} else {
|
||||
object.delete_marker_replication_status = dsc.pending_status();
|
||||
}
|
||||
object.replicate_decision_str = Some(dsc.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
if is_dir_object(&object.object_name) && object.version_id.is_none() {
|
||||
if synthetic_version_id {
|
||||
object.version_id = Some(Uuid::nil());
|
||||
}
|
||||
|
||||
let existing = (!skip_stat && gerr.is_none()).then_some(goi);
|
||||
Ok(AdmittedDelete {
|
||||
let existing = (!skip_stat && !source_missing).then_some(goi);
|
||||
Ok::<_, ApiError>(AdmittedDelete {
|
||||
idx,
|
||||
object,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
size,
|
||||
existing,
|
||||
blocked: None,
|
||||
})
|
||||
}))
|
||||
.buffered(DELETE_OBJECTS_PRE_STAT_CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
let admitted_deletes: Vec<AdmittedDelete> = admitted_deletes.into_iter().collect::<Result<_, ApiError>>()?;
|
||||
.try_collect()
|
||||
.await?;
|
||||
|
||||
// Phase 3 (serial): apply outcomes in the original request order so
|
||||
// per-key success/failure reporting is unchanged.
|
||||
@@ -6795,6 +6858,7 @@ impl DefaultObjectUsecase {
|
||||
let mut object_to_delete_idx = Vec::new();
|
||||
let mut object_sizes = Vec::new();
|
||||
let mut existing_object_infos = Vec::new();
|
||||
let mut object_versioning = Vec::new();
|
||||
for admitted in admitted_deletes {
|
||||
if let Some(err) = admitted.blocked {
|
||||
delete_results[admitted.idx].error = Some(err);
|
||||
@@ -6803,10 +6867,10 @@ impl DefaultObjectUsecase {
|
||||
|
||||
object_sizes.push(admitted.size);
|
||||
object_to_delete_idx.push(admitted.idx);
|
||||
object_versioning.push((admitted.versioned, admitted.version_suspended));
|
||||
object_to_delete.push(admitted.object);
|
||||
existing_object_infos.push(admitted.existing);
|
||||
}
|
||||
|
||||
let cache_adapter = self.object_data_cache();
|
||||
let cache_keys_before_delete = object_to_delete
|
||||
.iter()
|
||||
@@ -6814,13 +6878,14 @@ impl DefaultObjectUsecase {
|
||||
.collect::<Vec<_>>();
|
||||
invalidate_object_data_cache_objects_before_mutation(&cache_adapter, &bucket, cache_keys_before_delete.iter()).await;
|
||||
|
||||
let (mut dobjs, errs) = store
|
||||
let (dobjs, errs) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
object_to_delete.clone(),
|
||||
ObjectOptions {
|
||||
versioned: version_cfg.enabled(),
|
||||
version_suspended: version_cfg.suspended(),
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
object_lock_delete: Some(StorageObjectLockDeleteOptions { bypass_governance }),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -6849,18 +6914,16 @@ impl DefaultObjectUsecase {
|
||||
.clone()
|
||||
.is_some_and(|v| is_err_object_not_found(&v) || is_err_version_not_found(&v))
|
||||
{
|
||||
if replicate_deletes {
|
||||
dobjs[i].replication_state = Some(object_to_delete[i].replication_state());
|
||||
}
|
||||
delete_results[didx].delete_object = Some(dobjs[i].clone());
|
||||
let (versioned, version_suspended) = object_versioning[i];
|
||||
if let Err(err) = enqueue_transitioned_delete_cleanup(
|
||||
store.clone(),
|
||||
&bucket,
|
||||
&object_to_delete[i].object_name,
|
||||
&ObjectOptions {
|
||||
version_id: object_to_delete[i].version_id.map(|v| v.to_string()),
|
||||
versioned: version_cfg.prefix_enabled(object_to_delete[i].object_name.as_str()),
|
||||
version_suspended: version_cfg.suspended(),
|
||||
versioned,
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_infos[i].as_ref(),
|
||||
@@ -6874,9 +6937,7 @@ impl DefaultObjectUsecase {
|
||||
"failed to persist transitioned object cleanup journal"
|
||||
);
|
||||
}
|
||||
let creates_delete_marker = object_to_delete[i].version_id.is_none()
|
||||
&& version_cfg.prefix_enabled(object_to_delete[i].object_name.as_str())
|
||||
&& !version_cfg.suspended();
|
||||
let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
|
||||
if creates_delete_marker {
|
||||
record_bucket_delete_marker_memory(&bucket).await;
|
||||
} else {
|
||||
@@ -6896,25 +6957,25 @@ impl DefaultObjectUsecase {
|
||||
code: Some(err.to_string()),
|
||||
key: Some(object_to_delete[i].object_name.clone()),
|
||||
message: Some(err.to_string()),
|
||||
version_id: object_to_delete[i].version_id.map(|v| v.to_string()),
|
||||
version_id: delete_response_version_id(
|
||||
object_to_delete[i].version_id,
|
||||
delete_results[didx].synthetic_version_id,
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = delete_results
|
||||
.iter()
|
||||
.filter_map(|v| v.delete_object.clone())
|
||||
.map(|v| DeletedObject {
|
||||
delete_marker: { if v.delete_marker { Some(true) } else { None } },
|
||||
delete_marker_version_id: v.delete_marker_version_id.map(|v| v.to_string()),
|
||||
key: Some(v.object_name.clone()),
|
||||
version_id: if is_dir_object(v.object_name.as_str()) && v.version_id == Some(Uuid::nil()) {
|
||||
None
|
||||
} else if v.version_id == Some(Uuid::nil()) {
|
||||
Some("null".to_string())
|
||||
} else {
|
||||
v.version_id.map(|v| v.to_string())
|
||||
},
|
||||
.filter_map(|result| result.delete_object.as_ref().map(|object| (result, object)))
|
||||
.map(|(result, object)| DeletedObject {
|
||||
delete_marker: { if object.delete_marker { Some(true) } else { None } },
|
||||
delete_marker_version_id: delete_response_version_id(
|
||||
object.delete_marker_version_id,
|
||||
result.synthetic_version_id,
|
||||
),
|
||||
key: Some(object.object_name.clone()),
|
||||
version_id: delete_response_version_id(object.version_id, result.synthetic_version_id),
|
||||
})
|
||||
.collect();
|
||||
let deleted_cache_keys = delete_results
|
||||
@@ -6933,18 +6994,13 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for dobjs in &delete_results {
|
||||
if let Some(dobj) = &dobjs.delete_object
|
||||
for result in &delete_results {
|
||||
if let Some(dobj) = &result.delete_object
|
||||
&& replicate_deletes
|
||||
&& deleted_object_has_pending_replication_delete(dobj)
|
||||
{
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
|
||||
let mut dobj = dobj.clone();
|
||||
if is_dir_object(dobj.object_name.as_str()) && dobj.version_id.is_none() {
|
||||
dobj.version_id = Some(Uuid::nil());
|
||||
}
|
||||
|
||||
schedule_replication_delete(dobj, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
schedule_replication_delete(dobj.clone(), bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6969,7 +7025,7 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.version_id(dobj.version_id.map(|v| v.to_string()).unwrap_or_default())
|
||||
.version_id(delete_response_version_id(dobj.version_id, res.synthetic_version_id).unwrap_or_default())
|
||||
.req_params(req_params.clone())
|
||||
.resp_elements(resp_elements.clone())
|
||||
.host(get_request_host(&req_headers))
|
||||
@@ -7030,10 +7086,20 @@ impl DefaultObjectUsecase {
|
||||
let metadata = extract_metadata(&req.headers);
|
||||
// Clone version_id before it's moved
|
||||
let version_id_clone = version_id.clone();
|
||||
let synthetic_version_id = version_id_clone.is_none() && is_dir_object(&key);
|
||||
|
||||
let mut opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let delete_config_snapshot = Arc::new(
|
||||
load_delete_config_snapshot(store.as_ref(), &bucket)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
#[cfg(test)]
|
||||
wait_for_delete_snapshot_test_hook(&bucket).await;
|
||||
let version_cfg = delete_config_snapshot.versioning_config();
|
||||
let mut opts: ObjectOptions =
|
||||
del_opts_with_versioning(&bucket, &key, version_id, &req.headers, metadata, version_cfg, replica)
|
||||
.map_err(ApiError::from)?;
|
||||
opts.delete_replication_config_snapshot = Some(Arc::clone(&delete_config_snapshot));
|
||||
opts.object_lock_delete = Some(StorageObjectLockDeleteOptions {
|
||||
bypass_governance: has_bypass_governance_header(&req.headers),
|
||||
});
|
||||
@@ -7066,24 +7132,13 @@ impl DefaultObjectUsecase {
|
||||
validate_undo_delete_version(expected_current_version_id.as_deref(), opts.version_id.as_deref())?;
|
||||
opts.expected_current_version_id = expected_current_version_id.clone();
|
||||
|
||||
let replicate_force_delete = force_delete
|
||||
&& !replica
|
||||
&& has_replication_rules(
|
||||
&bucket,
|
||||
&[ObjectToDelete {
|
||||
object_name: key.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key);
|
||||
|
||||
// Check Object Lock retention before deletion
|
||||
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
|
||||
// 1. Add a lightweight get_object_lock_info() that only fetches retention metadata
|
||||
// 2. Or use combined get-and-delete in storage layer with retention check callback
|
||||
let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let get_opts = opts.clone();
|
||||
let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await {
|
||||
Ok(obj_info) => {
|
||||
// Check for bypass governance retention header (permission already verified in access.rs)
|
||||
@@ -7104,28 +7159,8 @@ impl DefaultObjectUsecase {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let delete_replication_state = if !replica && !force_delete && (opts.versioned || opts.version_suspended) {
|
||||
let fallback_source;
|
||||
let source = if let Some(source) = existing_object_info.as_ref() {
|
||||
source
|
||||
} else {
|
||||
fallback_source = ObjectInfo {
|
||||
name: key.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
&fallback_source
|
||||
};
|
||||
match metadata_sys::get_replication_config(&bucket).await {
|
||||
Ok((config, _)) => {
|
||||
delete_replication_state_from_config(&config, source, version_id_clone.as_ref().and(source.version_id), false)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
wait_for_delete_source_test_hook(&bucket).await;
|
||||
|
||||
let cache_adapter = self.object_data_cache();
|
||||
// A force (delete_prefix) delete removes every object under `key` as a
|
||||
@@ -7211,18 +7246,26 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let deleted_replication_info = existing_object_info
|
||||
.as_ref()
|
||||
.filter(|_| should_use_existing_delete_replication_info(&opts));
|
||||
.filter(|_| should_use_existing_delete_replication_info(&opts, opts.version_id.is_some()));
|
||||
let _delete_tail_guard = DeleteTailActivityGuard::new(DeleteTailStage::Tail);
|
||||
let deleted_object_source = deleted_replication_info.unwrap_or(&obj_info);
|
||||
let replication_state_source =
|
||||
delete_replication_state_source(&opts, existing_object_info.as_ref(), deleted_object_source);
|
||||
let replication_state_source = &obj_info;
|
||||
let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker);
|
||||
|
||||
let delete_replication_version_id = delete_replication_version_id(deleted_object_source, deleted_delete_marker_version);
|
||||
let schedule_delete_replication = if opts.replication_request && replica {
|
||||
should_schedule_replica_delete_replication(&bucket, replication_state_source, delete_replication_version_id).await
|
||||
should_schedule_replica_delete_replication(
|
||||
&delete_config_snapshot,
|
||||
replication_state_source,
|
||||
delete_replication_version_id,
|
||||
)
|
||||
} else {
|
||||
delete_replication_state.is_some()
|
||||
should_schedule_delete_replication(
|
||||
&opts,
|
||||
replication_state_source,
|
||||
deleted_delete_marker_version,
|
||||
opts.version_id.is_some(),
|
||||
)
|
||||
};
|
||||
|
||||
if schedule_delete_replication {
|
||||
@@ -7244,30 +7287,25 @@ impl DefaultObjectUsecase {
|
||||
replication_state: None,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(state) = delete_replication_state.as_ref() {
|
||||
set_deleted_object_replication_state(&mut deleted_object, state);
|
||||
} else {
|
||||
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
|
||||
enrich_delete_replication_state_if_needed(&bucket, &mut deleted_object, replication_state_source).await;
|
||||
}
|
||||
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
|
||||
enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
|
||||
schedule_replication_delete(deleted_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
}
|
||||
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let version_id = obj_info.version_id;
|
||||
let response_version_id = delete_response_version_id(version_id, synthetic_version_id);
|
||||
|
||||
let output = DeleteObjectOutput {
|
||||
delete_marker: Some(delete_marker),
|
||||
version_id: version_id.map(|v| v.to_string()),
|
||||
version_id: response_version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event_name = delete_event_name_for_marker(delete_marker);
|
||||
|
||||
helper = helper.event_name(event_name);
|
||||
helper = helper
|
||||
.object(obj_info)
|
||||
.version_id(version_id.map(|v| v.to_string()).unwrap_or_default());
|
||||
helper = helper.object(obj_info).version_id(response_version_id.unwrap_or_default());
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
// Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead)
|
||||
@@ -8403,6 +8441,16 @@ mod tests {
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
#[test]
|
||||
fn delete_response_version_id_preserves_null_and_synthetic_semantics() {
|
||||
let version_id = Uuid::new_v4();
|
||||
|
||||
assert_eq!(delete_response_version_id(Some(version_id), false), Some(version_id.to_string()));
|
||||
assert_eq!(delete_response_version_id(Some(Uuid::nil()), false), Some("null".to_string()));
|
||||
assert_eq!(delete_response_version_id(Some(Uuid::nil()), true), None);
|
||||
assert_eq!(delete_response_version_id(None, false), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn io_queue_congestion_warn_throttle_emits_once_per_interval() {
|
||||
let throttle = IoQueueCongestionWarnThrottle::new();
|
||||
@@ -13331,6 +13379,11 @@ mod tests {
|
||||
|
||||
assert_eq!(wire_version_id.as_deref(), Some("null"));
|
||||
assert_eq!(internal_version_id, Some(Uuid::nil()));
|
||||
|
||||
let (wire_version_id, internal_version_id) =
|
||||
normalize_delete_objects_version_id(Some(" \t ".to_string())).expect("empty version marker should normalize");
|
||||
assert_eq!(wire_version_id, None);
|
||||
assert_eq!(internal_version_id, None);
|
||||
}
|
||||
|
||||
// backlog#929 (HP-8): the pre-delete stat may only be skipped when every
|
||||
@@ -13347,17 +13400,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn delete_objects_pre_stat_skippable_for_delete_marker_on_plain_bucket() {
|
||||
assert!(can_skip_delete_objects_pre_stat(false, false, &delete_marker_creating_opts(), true));
|
||||
assert!(can_skip_delete_objects_pre_stat(false, &delete_marker_creating_opts(), true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_objects_pre_stat_kept_for_object_lock_buckets() {
|
||||
assert!(!can_skip_delete_objects_pre_stat(true, false, &delete_marker_creating_opts(), true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_objects_pre_stat_kept_when_replication_rules_match() {
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, true, &delete_marker_creating_opts(), true));
|
||||
assert!(!can_skip_delete_objects_pre_stat(true, &delete_marker_creating_opts(), true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -13368,7 +13416,7 @@ mod tests {
|
||||
version_suspended: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, false, &opts, true));
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, &opts, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -13381,7 +13429,7 @@ mod tests {
|
||||
version_suspended: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, false, &opts, false));
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, &opts, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -13392,7 +13440,7 @@ mod tests {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, false, &opts, false));
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, &opts, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -13400,7 +13448,7 @@ mod tests {
|
||||
// If the accounting-side versioning snapshot does not also classify the
|
||||
// delete as a delete-marker creation, the stat must stay so usage
|
||||
// accounting keeps its size input.
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, false, &delete_marker_creating_opts(), false));
|
||||
assert!(!can_skip_delete_objects_pre_stat(false, &delete_marker_creating_opts(), false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -13659,17 +13707,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_state_from_config_tracks_delete_marker_version_purges() {
|
||||
fn delete_replication_state_from_config_requires_delete_switch_for_marker_version_purges() {
|
||||
let arn = "arn:aws:s3:::target-bucket".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: arn.clone(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
bucket: arn.clone(),
|
||||
..Default::default()
|
||||
@@ -13694,8 +13740,16 @@ mod tests {
|
||||
};
|
||||
|
||||
let version_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
delete_replication_state_from_config(&config, &obj_info, version_id, false).is_none(),
|
||||
"delete-marker version purge must not use DeleteMarkerReplication"
|
||||
);
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let state = delete_replication_state_from_config(&config, &obj_info, version_id, false)
|
||||
.expect("delete-marker version purge should honor delete-marker replication rules");
|
||||
.expect("delete-marker version purge should honor DeleteReplication");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
|
||||
@@ -13703,55 +13757,6 @@ mod tests {
|
||||
assert!(state.purge_targets.contains_key(&arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_state_source_prefers_existing_replica_for_replication_delete_marker_creation() {
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let existing = ObjectInfo {
|
||||
name: "test/object.txt".to_string(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
let deleted = ObjectInfo {
|
||||
name: "test/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let source = delete_replication_state_source(&opts, Some(&existing), &deleted);
|
||||
|
||||
assert_eq!(source.replication_status, ReplicationStatusType::Completed);
|
||||
assert!(
|
||||
!source.delete_marker,
|
||||
"downstream fanout should inherit replica identity from the pre-delete object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_state_source_keeps_deleted_marker_for_non_replication_requests() {
|
||||
let opts = ObjectOptions::default();
|
||||
let existing = ObjectInfo {
|
||||
name: "test/object.txt".to_string(),
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
};
|
||||
let deleted = ObjectInfo {
|
||||
name: "test/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let source = delete_replication_state_source(&opts, Some(&existing), &deleted);
|
||||
|
||||
assert!(
|
||||
source.delete_marker,
|
||||
"source-originated deletes should keep using the new delete marker state"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replica_delete_enrichment_must_not_reuse_upstream_targets() {
|
||||
let upstream_state = ReplicationState {
|
||||
@@ -13819,7 +13824,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(
|
||||
!should_use_existing_delete_replication_info(&opts),
|
||||
!should_use_existing_delete_replication_info(&opts, true),
|
||||
"replicated delete-marker creation carries a source version id header but must not be treated as a version purge"
|
||||
);
|
||||
}
|
||||
@@ -13832,7 +13837,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(
|
||||
should_use_existing_delete_replication_info(&opts),
|
||||
should_use_existing_delete_replication_info(&opts, true),
|
||||
"true version-delete requests should keep using the pre-delete object info"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -236,7 +236,6 @@ pub(crate) mod bucket {
|
||||
|
||||
pub(crate) trait VersioningConfigExt {
|
||||
fn enabled(&self) -> bool;
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool;
|
||||
fn suspended(&self) -> bool;
|
||||
}
|
||||
|
||||
@@ -247,12 +246,6 @@ pub(crate) mod bucket {
|
||||
)
|
||||
}
|
||||
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool {
|
||||
<s3s::dto::VersioningConfiguration as crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi>::prefix_enabled(
|
||||
self, prefix,
|
||||
)
|
||||
}
|
||||
|
||||
fn suspended(&self) -> bool {
|
||||
<s3s::dto::VersioningConfiguration as crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi>::suspended(self)
|
||||
}
|
||||
@@ -623,6 +616,8 @@ pub(crate) mod bucket {
|
||||
use crate::storage::storage_api::ecstore_bucket::replication as replication_contracts;
|
||||
|
||||
type ReplicationObjectBridge = crate::storage::storage_api::ecstore_bucket::replication::ReplicationObjectBridge;
|
||||
pub(crate) type DeleteReplicationConfigSnapshot =
|
||||
crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
pub(crate) type ReplicateDecision = replication_contracts::ReplicateDecision;
|
||||
#[cfg(test)]
|
||||
pub(crate) type ReplicationState = replication_contracts::ReplicationState;
|
||||
@@ -642,15 +637,32 @@ pub(crate) mod bucket {
|
||||
/// fails the test.
|
||||
#[cfg(test)]
|
||||
pub(crate) static MUST_REPLICATE_OBJECT_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
#[cfg(test)]
|
||||
pub(crate) static DELETE_CONFIG_SNAPSHOT_LOADS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
#[cfg(test)]
|
||||
static SCHEDULED_REPLICATION_DELETES: std::sync::Mutex<Vec<crate::storage::storage_api::StorageDeletedObject>> =
|
||||
std::sync::Mutex::new(Vec::new());
|
||||
|
||||
pub(crate) async fn check_replicate_delete(
|
||||
#[cfg(test)]
|
||||
pub(crate) fn take_scheduled_replication_deletes() -> Vec<crate::storage::storage_api::StorageDeletedObject> {
|
||||
std::mem::take(
|
||||
&mut *SCHEDULED_REPLICATION_DELETES
|
||||
.lock()
|
||||
.expect("scheduled replication delete lock should not be poisoned"),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_delete_config_snapshot(
|
||||
store: &crate::storage::storage_api::ECStore,
|
||||
bucket: &str,
|
||||
dobj: &super::super::storage_contracts::ObjectToDelete,
|
||||
oi: &crate::storage::storage_api::StorageObjectInfo,
|
||||
del_opts: &crate::storage::storage_api::StorageObjectOptions,
|
||||
gerr: Option<String>,
|
||||
) -> Result<ReplicateDecision, crate::storage::storage_api::StorageError> {
|
||||
ReplicationObjectBridge::check_delete_strict(bucket, dobj, oi, del_opts, gerr).await
|
||||
) -> Result<DeleteReplicationConfigSnapshot, crate::storage::storage_api::StorageError> {
|
||||
#[cfg(test)]
|
||||
DELETE_CONFIG_SNAPSHOT_LOADS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
ReplicationObjectBridge::delete_request_config(store, bucket).await
|
||||
}
|
||||
|
||||
pub(crate) fn has_active_delete_rule(snapshot: &DeleteReplicationConfigSnapshot, object: &str) -> bool {
|
||||
ReplicationObjectBridge::has_active_delete_rule(snapshot, object)
|
||||
}
|
||||
|
||||
pub(crate) fn delete_replication_version_id(
|
||||
@@ -697,6 +709,11 @@ pub(crate) mod bucket {
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
SCHEDULED_REPLICATION_DELETES
|
||||
.lock()
|
||||
.expect("scheduled replication delete lock should not be poisoned")
|
||||
.push(delete_object.clone());
|
||||
ReplicationObjectBridge::schedule_storage_delete(delete_object, bucket, event_type).await;
|
||||
}
|
||||
|
||||
@@ -707,13 +724,6 @@ pub(crate) mod bucket {
|
||||
delete_object.replication_state = Some(replication_contracts::replication_state_to_filemeta(state));
|
||||
}
|
||||
|
||||
pub(crate) fn set_object_to_delete_version_purge_status(
|
||||
object: &mut crate::storage::storage_api::StorageObjectToDelete,
|
||||
status: VersionPurgeStatusType,
|
||||
) {
|
||||
object.version_purge_status = Some(replication_contracts::version_purge_status_to_filemeta(status));
|
||||
}
|
||||
|
||||
pub(crate) fn deleted_object_has_pending_replication_delete(
|
||||
deleted_object: &crate::storage::storage_api::StorageDeletedObject,
|
||||
) -> bool {
|
||||
@@ -751,22 +761,27 @@ pub(crate) mod bucket {
|
||||
replication_contracts::should_remove_replication_target(target_arn, is_replication_service, target_arns)
|
||||
}
|
||||
|
||||
pub(crate) fn should_use_existing_delete_replication_info(
|
||||
pub(crate) fn should_schedule_delete_replication(
|
||||
opts: &crate::storage::storage_api::StorageObjectOptions,
|
||||
replication_source: &crate::storage::storage_api::StorageObjectInfo,
|
||||
deleted_delete_marker_version: bool,
|
||||
version_id_requested: bool,
|
||||
) -> bool {
|
||||
replication_contracts::should_use_existing_delete_replication_info(opts.version_id.is_some(), opts.delete_marker)
|
||||
replication_contracts::should_schedule_delete_replication(replication_contracts::ReplicationDeleteScheduleInput {
|
||||
replication_request: opts.replication_request,
|
||||
version_id_requested,
|
||||
source_delete_marker: replication_source.delete_marker,
|
||||
source_replication_status: &replication_source.replication_status,
|
||||
source_version_purge_status: &replication_source.version_purge_status,
|
||||
deleted_delete_marker_version,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn should_use_existing_delete_replication_source(
|
||||
replication_request: bool,
|
||||
deleted_delete_marker: bool,
|
||||
has_existing_info: bool,
|
||||
pub(crate) fn should_use_existing_delete_replication_info(
|
||||
opts: &crate::storage::storage_api::StorageObjectOptions,
|
||||
version_id_requested: bool,
|
||||
) -> bool {
|
||||
replication_contracts::should_use_existing_delete_replication_source(
|
||||
replication_request,
|
||||
deleted_delete_marker,
|
||||
has_existing_info,
|
||||
)
|
||||
replication_contracts::should_use_existing_delete_replication_info(version_id_requested, opts.delete_marker)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_replication_config_target_arns<'a>(
|
||||
@@ -779,6 +794,12 @@ pub(crate) mod bucket {
|
||||
pub(crate) fn unsupported_replication_config_field(config: &s3s::dto::ReplicationConfiguration) -> Option<&'static str> {
|
||||
replication_contracts::unsupported_replication_config_field(config)
|
||||
}
|
||||
|
||||
pub(crate) fn invalid_replication_config_status_field(
|
||||
config: &s3s::dto::ReplicationConfiguration,
|
||||
) -> Option<&'static str> {
|
||||
replication_contracts::invalid_replication_config_status_field(config)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod tagging {
|
||||
@@ -863,10 +884,10 @@ pub(crate) mod options {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::options_consumer::VERSIONING_CONFIG_LOOKUPS;
|
||||
pub(crate) use crate::storage::storage_api::options_consumer::{
|
||||
bucket_versioning_config, copy_dst_opts, copy_src_opts, del_opts, del_opts_with_versioning, extract_metadata,
|
||||
extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata,
|
||||
get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata,
|
||||
normalize_content_encoding_for_storage, parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
copy_dst_opts, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime,
|
||||
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts,
|
||||
get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, normalize_content_encoding_for_storage,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -994,7 +1015,7 @@ pub(crate) mod object_usecase {
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo,
|
||||
StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions,
|
||||
has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists,
|
||||
validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write,
|
||||
validate_ssec_for_read, wrap_response_with_cors,
|
||||
@@ -1078,6 +1099,7 @@ pub(crate) mod test {
|
||||
pub(crate) use super::access::ReqInfo;
|
||||
pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS;
|
||||
pub(crate) use super::{bucket, data_usage, ecfs, object_utils, runtime};
|
||||
pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
|
||||
};
|
||||
|
||||
@@ -12,19 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::StorageReplicationConfigExt as _;
|
||||
use super::{
|
||||
StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, get_bucket_replication_config,
|
||||
};
|
||||
use super::{StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config};
|
||||
use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled};
|
||||
use crate::error::ApiError;
|
||||
use crate::server::cors;
|
||||
use crate::storage::ecfs::ListObjectUnorderedQuery;
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER;
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::contract::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
object::ObjectToDelete,
|
||||
};
|
||||
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use metrics::counter;
|
||||
@@ -731,22 +725,6 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelete]) -> bool {
|
||||
let (cfg, _created) = match get_bucket_replication_config(bucket).await {
|
||||
Ok(replication_config) => replication_config,
|
||||
Err(_err) => {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
for object in objects {
|
||||
if cfg.has_active_rules(&object.object_name, true) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Bucket validation cache to avoid repeated stat_volume() calls on every GET.
|
||||
///
|
||||
/// **Adaptive strategy** (selected once at startup via env var):
|
||||
|
||||
+111
-18
@@ -109,7 +109,7 @@ pub async fn del_opts(
|
||||
metadata: HashMap<String, String>,
|
||||
) -> Result<ObjectOptions> {
|
||||
let versioning_cfg = bucket_versioning_config(bucket).await;
|
||||
del_opts_with_versioning(bucket, object, vid, headers, metadata, &versioning_cfg)
|
||||
del_opts_with_versioning(bucket, object, vid, headers, metadata, &versioning_cfg, false)
|
||||
}
|
||||
|
||||
/// Like [`del_opts`], but derives versioning state from an already-fetched
|
||||
@@ -122,15 +122,16 @@ pub fn del_opts_with_versioning(
|
||||
headers: &HeaderMap<HeaderValue>,
|
||||
metadata: HashMap<String, String>,
|
||||
versioning_cfg: &VersioningConfiguration,
|
||||
replication_request_authorized: bool,
|
||||
) -> Result<ObjectOptions> {
|
||||
let versioned = versioning_cfg.prefix_enabled(object);
|
||||
let version_suspended = versioning_cfg.suspended();
|
||||
let (versioned, version_suspended) = versioning_cfg.delete_state(object);
|
||||
|
||||
let vid = if vid.is_none() {
|
||||
let vid = if vid.is_none() && replication_request_authorized {
|
||||
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
|
||||
} else {
|
||||
vid
|
||||
};
|
||||
let synthetic_version_id = is_dir_object(object) && vid.is_none();
|
||||
|
||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||
|
||||
@@ -152,7 +153,12 @@ pub fn del_opts_with_versioning(
|
||||
None
|
||||
};
|
||||
|
||||
let mut opts = put_opts_from_headers(headers, metadata).map_err(|err| {
|
||||
let mut opts = if replication_request_authorized {
|
||||
put_opts_from_headers(headers, metadata)
|
||||
} else {
|
||||
get_default_opts(headers, metadata, false)
|
||||
}
|
||||
.map_err(|err| {
|
||||
error!("del_opts: invalid argument: {} error: {}", object, err);
|
||||
StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())
|
||||
})?;
|
||||
@@ -161,19 +167,15 @@ pub fn del_opts_with_versioning(
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
opts.version_id = {
|
||||
if is_dir_object(object) && vid.is_none() {
|
||||
Some(Uuid::nil().to_string())
|
||||
} else {
|
||||
vid
|
||||
}
|
||||
};
|
||||
opts.version_id = synthetic_version_id.then(|| Uuid::nil().to_string()).or(vid);
|
||||
opts.synthetic_version_id = synthetic_version_id;
|
||||
opts.version_suspended = version_suspended;
|
||||
opts.versioned = versioned;
|
||||
|
||||
opts.delete_marker = get_header(headers, SUFFIX_SOURCE_DELETEMARKER)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
opts.delete_marker = replication_request_authorized
|
||||
&& get_header(headers, SUFFIX_SOURCE_DELETEMARKER)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
fill_conditional_writes_opts_from_header(headers, &mut opts)?;
|
||||
|
||||
@@ -924,7 +926,7 @@ mod tests {
|
||||
use super::super::StorageError;
|
||||
use super::{
|
||||
ENV_REJECT_ARCHIVE_CONTENT_ENCODING, ReplicationStatusType, SUPPORTED_HEADERS, copy_dst_opts, copy_src_opts, del_opts,
|
||||
detect_content_type_from_object_name, extract_metadata, extract_metadata_from_mime,
|
||||
del_opts_with_versioning, detect_content_type_from_object_name, extract_metadata, extract_metadata_from_mime,
|
||||
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts,
|
||||
get_default_opts, get_opts, namespace_reserved_user_metadata, parse_copy_source_range, put_opts, put_opts_from_headers,
|
||||
validate_archive_content_encoding,
|
||||
@@ -932,10 +934,11 @@ mod tests {
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
insert_header,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||
};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -979,6 +982,17 @@ mod tests {
|
||||
assert_eq!(opts.version_id, Some(Uuid::nil().to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_del_opts_preserves_explicit_null_directory_version() {
|
||||
let headers = create_test_headers();
|
||||
|
||||
let opts = del_opts("test-bucket", "test-dir/", Some("null".to_string()), &headers, HashMap::new())
|
||||
.await
|
||||
.expect("explicit null directory version should be accepted");
|
||||
|
||||
assert_eq!(opts.version_id, Some(Uuid::nil().to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_del_opts_with_valid_version_id() {
|
||||
let headers = create_test_headers();
|
||||
@@ -998,6 +1012,85 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_del_opts_only_trusts_replication_headers_after_authorization() {
|
||||
let source_version_id = Uuid::new_v4().to_string();
|
||||
let source_mtime = "2024-05-20T10:30:00+08:00";
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, source_version_id.clone());
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, source_mtime);
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
|
||||
headers.insert(
|
||||
AMZ_BUCKET_REPLICATION_STATUS,
|
||||
HeaderValue::from_static(ReplicationStatusType::Replica.as_str()),
|
||||
);
|
||||
|
||||
let untrusted = del_opts_with_versioning(
|
||||
"test-bucket",
|
||||
"test-object",
|
||||
None,
|
||||
&headers,
|
||||
HashMap::new(),
|
||||
&VersioningConfiguration::default(),
|
||||
false,
|
||||
)
|
||||
.expect("ordinary delete options should ignore internal replication headers");
|
||||
|
||||
assert_eq!(untrusted.version_id, None);
|
||||
assert!(!untrusted.replication_request);
|
||||
assert!(untrusted.mod_time.is_none());
|
||||
assert!(!untrusted.delete_marker);
|
||||
assert!(untrusted.delete_replication.is_none());
|
||||
|
||||
let trusted = del_opts_with_versioning(
|
||||
"test-bucket",
|
||||
"test-object",
|
||||
None,
|
||||
&headers,
|
||||
HashMap::new(),
|
||||
&VersioningConfiguration::default(),
|
||||
true,
|
||||
)
|
||||
.expect("authorized replication delete options should accept internal headers");
|
||||
|
||||
assert_eq!(trusted.version_id.as_deref(), Some(source_version_id.as_str()));
|
||||
assert!(trusted.replication_request);
|
||||
assert!(trusted.mod_time.is_some());
|
||||
assert!(trusted.delete_marker);
|
||||
assert_eq!(trusted.delete_marker_replication_status(), ReplicationStatusType::Replica);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_del_opts_treats_excluded_prefix_as_unversioned() {
|
||||
let versioning = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let excluded = del_opts_with_versioning(
|
||||
"test-bucket",
|
||||
"archive/object",
|
||||
None,
|
||||
&HeaderMap::new(),
|
||||
HashMap::new(),
|
||||
&versioning,
|
||||
false,
|
||||
)
|
||||
.expect("excluded-prefix delete options should be derived");
|
||||
assert!(!excluded.versioned);
|
||||
assert!(!excluded.version_suspended);
|
||||
|
||||
let included =
|
||||
del_opts_with_versioning("test-bucket", "live/object", None, &HeaderMap::new(), HashMap::new(), &versioning, false)
|
||||
.expect("included-prefix delete options should be derived");
|
||||
assert!(included.versioned);
|
||||
assert!(!included.version_suspended);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_del_opts_with_invalid_version_id() {
|
||||
let headers = create_test_headers();
|
||||
|
||||
@@ -89,8 +89,8 @@ pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
|
||||
pub(crate) type StoragePutObjReader = super::PutObjReader;
|
||||
pub(crate) use super::ecfs_extend::{
|
||||
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, get_buffer_size_opt_in,
|
||||
get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
|
||||
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
get_validated_store, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter, validate_object_key, wrap_response_with_cors,
|
||||
};
|
||||
@@ -151,10 +151,6 @@ pub(crate) mod ecfs_extend_consumer {
|
||||
pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::contract::object::ObjectToDelete;
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use super::super::super::contract::multipart::MAX_MULTIPART_PART_NUMBER;
|
||||
}
|
||||
@@ -183,10 +179,10 @@ pub(crate) mod options_consumer {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::super::options::VERSIONING_CONFIG_LOOKUPS;
|
||||
pub(crate) use super::super::options::{
|
||||
bucket_versioning_config, copy_dst_opts, copy_src_opts, del_opts, del_opts_with_versioning, extract_metadata,
|
||||
extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata,
|
||||
get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata,
|
||||
normalize_content_encoding_for_storage, parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
copy_dst_opts, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime,
|
||||
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts,
|
||||
get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, normalize_content_encoding_for_storage,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
|
||||
pub(crate) mod contract {
|
||||
|
||||
Reference in New Issue
Block a user