fix(replication): fence stale metadata status writeback (#7083)

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-09-03 16:05:58 +08:00
committed by GitHub
parent 8023cf3e26
commit 86ebcb325c
29 changed files with 3219 additions and 229 deletions
+125 -17
View File
@@ -99,11 +99,11 @@ use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
use rustfs_utils::http::insert_header;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, insert_header};
use rustfs_utils::http::{
SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header,
get_source_scheme,
SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header, get_source_scheme,
headers::{AMZ_CHECKSUM_TYPE, AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
insert_str,
};
@@ -733,6 +733,43 @@ impl DefaultMultipartUsecase {
None => None,
};
// Replication admission is decided at the object-creation commit, not
// at multipart-session creation. Configuration and matching targets
// can change while parts are uploaded, so persist the exact decision's
// generation and PENDING set atomically with the completed object and
// reuse the same decision for scheduling below.
let completion_replication_decision = must_replicate_object(
&bucket,
&key,
&multipart_info.user_defined,
"".to_string(),
opts.delete_marker_replication_status(),
opts.clone(),
)
.await;
let mut completion_replication_metadata = HashMap::new();
if completion_replication_decision.replicate_any() {
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_GENERATION,
Uuid::new_v4().to_string(),
);
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_TIMESTAMP,
jiff::Zoned::now().to_string(),
);
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_STATUS,
completion_replication_decision.pending_status().unwrap_or_default(),
);
}
// `Some(empty)` deliberately means that completion re-evaluated the
// session as not admitted; storage removes stale Create-MPU admission
// metadata in the same final-object commit.
opts.eval_metadata = Some(completion_replication_metadata);
let complete_commit = spawn_traced_join({
let store = Arc::clone(&store);
let bucket = bucket.clone();
@@ -760,20 +797,9 @@ impl DefaultMultipartUsecase {
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
let mt2 = obj_info.user_defined.clone();
let dsc = must_replicate_object(
&bucket,
&key,
&mt2,
"".to_string(),
opts.delete_marker_replication_status(),
opts.clone(),
)
.await;
if dsc.replicate_any() {
if completion_replication_decision.replicate_any() {
warn!("need multipart replication");
schedule_object_replication(obj_info.clone(), store, dsc).await;
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
}
rustfs_scanner::record_dirty_usage_bucket(&bucket);
@@ -1037,6 +1063,7 @@ impl DefaultMultipartUsecase {
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
.await;
if dsc.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
@@ -2499,6 +2526,87 @@ mod tests {
}
}
#[test]
#[serial_test::serial]
fn authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state() {
crate::app::gating_test_env::run_large_stack_test(
"authorized-replica-only-multipart-complete",
authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner,
);
}
async fn authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("replica-only-complete", 16_384).await;
let object = "object";
let usecase = DefaultMultipartUsecase::from_global();
let create_input = CreateMultipartUploadInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.build()
.expect("create multipart input should build");
let mut create_request = build_request(create_input, Method::POST);
create_request
.headers
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
create_request.extensions.insert(crate::storage::access::ReqInfo {
replication_request_authorized: true,
..Default::default()
});
let upload_id = usecase
.execute_create_multipart_upload(create_request)
.await
.expect("authorized REPLICA-only create should succeed")
.output
.upload_id
.expect("create response should contain upload id");
let mut reader = PutObjReader::from_vec(b"authorized replica-only multipart body".to_vec());
let part = store
.put_object_part(&bucket, object, &upload_id, 1, &mut reader, &ObjectOptions::default())
.await
.expect("replica part should be staged");
let complete_input = CompleteMultipartUploadInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.upload_id(upload_id)
.multipart_upload(Some(CompletedMultipartUpload {
parts: Some(vec![CompletedPart {
part_number: Some(1),
e_tag: part.etag.map(|etag| to_s3s_etag(&etag)),
..Default::default()
}]),
}))
.build()
.expect("complete multipart input should build");
let mut complete_request = build_request(complete_input, Method::POST);
complete_request
.headers
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
complete_request.extensions.insert(crate::storage::access::ReqInfo {
replication_request_authorized: true,
..Default::default()
});
usecase
.execute_complete_multipart_upload(complete_request)
.await
.expect("authorized REPLICA-only completion should succeed without the source-request marker");
let persisted = store
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("completed replica should be readable from storage");
assert_eq!(
persisted
.user_defined
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS))
.map(|(_, value)| value.as_str()),
Some("REPLICA"),
"the committed xl.meta must retain REPLICA to prevent scanner cascades"
);
}
#[tokio::test]
#[serial_test::serial]
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
+33 -5
View File
@@ -721,11 +721,7 @@ impl DefaultObjectUsecase {
// exempt: the authorized replication request owns these keys (see
// copy_dst_opts_with_replication_authorization above).
if !dst_opts.replication_request {
user_defined.retain(|k, _| !k.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS));
remove_str(&mut user_defined, SUFFIX_REPLICATION_STATUS);
remove_str(&mut user_defined, SUFFIX_REPLICATION_TIMESTAMP);
remove_str(&mut user_defined, SUFFIX_REPLICA_STATUS);
remove_str(&mut user_defined, SUFFIX_REPLICA_TIMESTAMP);
remove_source_replication_bookkeeping(&mut user_defined);
}
// Compute the replication decision exactly once per copy. The same
@@ -746,6 +742,7 @@ impl DefaultObjectUsecase {
)
.await;
if dsc.replicate_any() {
insert_str(&mut user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut user_defined, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
}
@@ -914,6 +911,37 @@ mod tests {
use std::sync::Arc;
use tokio::io::AsyncReadExt;
#[test]
fn local_copy_does_not_inherit_source_replication_generation() {
let mut metadata = HashMap::from([
(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string()),
("x-amz-meta-owner".to_string(), "source".to_string()),
]);
for suffix in [
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP,
] {
insert_str(&mut metadata, suffix, format!("source-{suffix}"));
}
remove_source_replication_bookkeeping(&mut metadata);
assert_eq!(metadata.get("x-amz-meta-owner").map(String::as_str), Some("source"));
assert!(!metadata.contains_key(AMZ_BUCKET_REPLICATION_STATUS));
for suffix in [
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP,
] {
assert!(!rustfs_utils::http::contains_key_str(&metadata, suffix));
}
}
// A malformed bucket-default algorithm reaches this resolution only through
// corrupt or hand-edited bucket metadata (PutBucketEncryption validates the
// value), so the invariant is pinned here rather than end-to-end: the copy
+1
View File
@@ -2535,6 +2535,7 @@ impl DefaultObjectUsecase {
)
.await;
if replication.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
+402 -62
View File
@@ -44,7 +44,8 @@ use http::HeaderName;
/// `Content-Language`, `Expires`), `user_metadata` carries `x-amz-meta-*`
/// entries with the prefix stripped, `tags` is the `x-amz-tagging` query
/// string and `internal_metadata` holds `x-rustfs-internal-*` /
/// `x-minio-internal-*` keys written verbatim.
/// `x-minio-internal-*` keys. Source replication bookkeeping is stripped;
/// other internal provenance is written verbatim.
pub(crate) struct InternalPutContext {
pub(crate) bucket: String,
pub(crate) key: String,
@@ -242,7 +243,7 @@ impl DefaultObjectUsecase {
content_headers,
user_metadata,
tags,
internal_metadata,
mut internal_metadata,
emit_events,
principal_id,
} = ctx;
@@ -253,6 +254,7 @@ impl DefaultObjectUsecase {
let headers = internal_put_headers(&content_headers)?;
validate_internal_write_target(&key, &bucket, &headers).await?;
remove_source_replication_bookkeeping(&mut internal_metadata);
let write = PutObjectWriteRequest {
bucket,
@@ -349,6 +351,7 @@ impl DefaultObjectUsecase {
);
}
metadata.extend(ctx.internal_metadata.clone());
remove_source_replication_bookkeeping(&mut metadata);
let mt2 = metadata.clone();
let mut opts = put_opts_with_replication_authorization(&ctx.bucket, &ctx.key, None, &headers, metadata, false)
@@ -365,6 +368,7 @@ impl DefaultObjectUsecase {
)
.await;
if dsc.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
@@ -533,6 +537,11 @@ impl DefaultObjectUsecase {
let capacity_scope_token = Uuid::new_v4();
opts.capacity_scope_token = Some(capacity_scope_token);
let multipart_info = store
.get_multipart_info(&bucket, &key, upload_id, &opts)
.await
.map_err(ApiError::from)?;
let current_opts =
internal_object_info_lookup_opts(get_opts(&bucket, &key, None, None, &headers).await.map_err(ApiError::from)?);
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket)
@@ -576,6 +585,44 @@ impl DefaultObjectUsecase {
None => None,
};
// Internal multipart writes use the same object-creation admission
// contract as the S3 completion path. Re-evaluate against the staged
// metadata immediately before commit, persist the exact generation and
// PENDING target set atomically with the object, and reuse that same
// immutable decision for scheduling below.
let mut completion_source_metadata = multipart_info.user_defined.clone();
remove_source_replication_bookkeeping(&mut completion_source_metadata);
let completion_replication_decision = must_replicate_object(
&bucket,
&key,
&completion_source_metadata,
"".to_string(),
opts.delete_marker_replication_status(),
opts.clone(),
)
.await;
let mut completion_replication_metadata = HashMap::new();
if completion_replication_decision.replicate_any() {
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_GENERATION,
Uuid::new_v4().to_string(),
);
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_TIMESTAMP,
jiff::Zoned::now().to_string(),
);
insert_str(
&mut completion_replication_metadata,
SUFFIX_REPLICATION_STATUS,
completion_replication_decision.pending_status().unwrap_or_default(),
);
}
// `Some(empty)` clears a stale create-time admission when replication
// was disabled or no rule matches at completion.
opts.eval_metadata = Some(completion_replication_metadata);
let event = ctx.emit_events.then(|| {
InternalPutObjectEvent::new(
current_notify_interface_for_context(self.context.as_deref()),
@@ -615,18 +662,8 @@ impl DefaultObjectUsecase {
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
let mt2 = obj_info.user_defined.clone();
let dsc = must_replicate_object(
&bucket,
&key,
&mt2,
"".to_string(),
opts.delete_marker_replication_status(),
opts.clone(),
)
.await;
if dsc.replicate_any() {
schedule_object_replication(obj_info.clone(), store, dsc).await;
if completion_replication_decision.replicate_any() {
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
}
rustfs_scanner::record_dirty_usage_bucket(&bucket);
@@ -663,10 +700,16 @@ impl DefaultObjectUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::storage_api::s3::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, Tag, VersioningConfiguration,
};
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
use rustfs_utils::http::{
MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_ODM_PULLED_AT, SUFFIX_ODM_SOURCE, SUFFIX_ODM_SOURCE_ETAG,
SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_str,
SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_consistent_str, get_str,
has_internal_suffix,
};
const TEST_PRINCIPAL: &str = "rustfs-internal-put-test";
@@ -727,6 +770,70 @@ mod tests {
(store, bucket)
}
async fn install_internal_replication_config(bucket: &str, target: Option<&str>) {
install_internal_replication_config_with_tag(bucket, target, None).await;
}
async fn install_internal_replication_config_with_tag(
bucket: &str,
target: Option<&str>,
required_tag: Option<(&str, &str)>,
) {
use crate::app::storage_api::test::bucket::utils::serialize;
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
let metadata = {
let sys = sys.read().await;
sys.get(bucket)
.await
.expect("bucket metadata should be cached before replication config injection")
};
let mut metadata = (*metadata).clone();
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
metadata.versioning_config = Some(VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
});
if let Some(target) = target {
let filter = required_tag.map(|(key, value)| ReplicationRuleFilter {
tag: Some(Tag {
key: Some(key.to_string()),
value: Some(value.to_string()),
}),
..Default::default()
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
}),
delete_replication: None,
destination: Destination {
bucket: target.to_string(),
..Default::default()
},
existing_object_replication: None,
filter,
id: Some("internal-multipart".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
};
metadata.replication_config_xml = serialize(&config).expect("replication test config should serialize");
metadata.replication_config = Some(config);
} else {
metadata.replication_config_xml.clear();
metadata.replication_config = None;
}
set_bucket_metadata(bucket.to_string(), metadata)
.await
.expect("replication test metadata should be installed");
}
#[tokio::test]
#[serial_test::serial]
async fn internal_put_object_writes_through_the_shared_put_path() {
@@ -770,6 +877,120 @@ mod tests {
assert_eq!(get_str(metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:source-bucket"));
}
#[tokio::test]
#[serial_test::serial]
async fn internal_put_drops_foreign_admission_before_later_matching_metadata() {
let (store, bucket) = internal_put_test_bucket("internal-put-foreign-admission").await;
let target = "arn:aws:s3:::internal-tag-filter-target";
install_internal_replication_config_with_tag(&bucket, Some(target), Some(("replicate", "yes"))).await;
let body = b"initially unadmitted internal object".to_vec();
let mut ctx = internal_context(&bucket, "foreign-admission.txt", &body);
ctx.tags = Some("replicate=no".to_string());
ctx.internal_metadata
.insert("X-Minio-Internal-Replication-Status".to_string(), format!("{target}=COMPLETED;"));
ctx.internal_metadata
.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string());
DefaultObjectUsecase::from_global()
.internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
.await
.expect("internal put with foreign bookkeeping should succeed after sanitization");
let stored = store
.get_object_info(&bucket, "foreign-admission.txt", &ObjectOptions::default())
.await
.expect("sanitized internal object should be readable");
assert!(!contains_key_str(&stored.user_defined, SUFFIX_REPLICATION_STATUS));
assert!(
stored
.user_defined
.keys()
.all(|key| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)),
"an unadmitted object must not retain a caller-supplied surfaced status"
);
let later_matching = crate::app::storage_api::object_usecase::bucket::replication::must_replicate_metadata(
&bucket,
"foreign-admission.txt",
&stored.user_defined,
"replicate=yes".to_string(),
stored.replication_status.clone(),
ObjectOptions::default(),
)
.await;
assert!(
!later_matching.replicate_any(),
"a caller-supplied source status must not forge historical admission for a later matching tag"
);
}
#[tokio::test]
#[serial_test::serial]
async fn internal_put_replaces_mixed_case_foreign_generation_with_local_canonical_aliases() {
let (store, bucket) = internal_put_test_bucket("internal-put-foreign-gen").await;
let target = "arn:aws:s3:::internal-generation-target";
install_internal_replication_config(&bucket, Some(target)).await;
let body = b"replication-admitted internal object".to_vec();
let mut ctx = internal_context(&bucket, "foreign-generation.txt", &body);
let foreign_generation = Uuid::from_u128(9001).to_string();
for (key, value) in [
("X-Minio-Internal-Replication-Generation", foreign_generation.as_str()),
("X-Minio-Internal-Replication-Timestamp", "foreign-replication-time"),
("X-Minio-Internal-Replication-Status", "arn:foreign=COMPLETED;"),
("X-Minio-Internal-Replica-Status", "REPLICA"),
("X-Minio-Internal-Replica-Timestamp", "foreign-replica-time"),
] {
ctx.internal_metadata.insert(key.to_string(), value.to_string());
}
ctx.internal_metadata
.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
DefaultObjectUsecase::from_global()
.internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
.await
.expect("replication-admitted internal put should sanitize foreign bookkeeping");
let stored = store
.get_object_info(&bucket, "foreign-generation.txt", &ObjectOptions::default())
.await
.expect("replication-admitted internal object should be readable");
let local_generation =
get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_GENERATION).expect("local generation aliases must agree");
Uuid::parse_str(local_generation).expect("local generation must be a UUID");
assert_ne!(local_generation, foreign_generation);
assert!(get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_TIMESTAMP).is_some());
assert!(
get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_STATUS).is_some_and(|status| status.contains(target))
);
for suffix in [
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICATION_STATUS,
] {
assert_eq!(
stored
.user_defined
.keys()
.filter(|key| has_internal_suffix(key, suffix))
.count(),
2,
"{suffix} must be persisted as exactly one canonical dual-key pair"
);
}
for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] {
assert!(!contains_key_str(&stored.user_defined, suffix), "foreign {suffix} must be removed");
}
assert!(
stored
.user_defined
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS))
.all(|(_, value)| value != "REPLICA"),
"a local source admission may surface its own status but must not preserve foreign REPLICA state"
);
}
#[tokio::test]
#[serial_test::serial]
async fn internal_put_object_preserves_the_caller_etag() {
@@ -840,7 +1061,7 @@ mod tests {
async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing_inner() {
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
let (store, bucket) = internal_put_test_bucket("internal-mpu").await;
let (store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu")).await;
let usecase = DefaultObjectUsecase::from_global();
let first_part = vec![0x41u8; FIRST_PART_SIZE];
let last_part = b"tail of the multipart object".to_vec();
@@ -849,37 +1070,33 @@ mod tests {
ctx.expected_md5_hex = None;
ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-2".to_string());
let upload_id = usecase
.internal_create_multipart_upload(&ctx)
let upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx))
.await
.expect("internal multipart create must succeed");
let part_one = usecase
.internal_upload_part(
&ctx,
&upload_id,
1,
first_part.len() as u64,
Some(md5_hex(&first_part)),
body_stream(vec![Bytes::from(first_part.clone())]),
)
.await
.expect("first internal part must stage");
let part_two = usecase
.internal_upload_part(
&ctx,
&upload_id,
2,
last_part.len() as u64,
Some(md5_hex(&last_part)),
body_stream(vec![Bytes::from(last_part.clone())]),
)
.await
.expect("last internal part must stage");
let part_one = Box::pin(usecase.internal_upload_part(
&ctx,
&upload_id,
1,
first_part.len() as u64,
Some(md5_hex(&first_part)),
body_stream(vec![Bytes::from(first_part.clone())]),
))
.await
.expect("first internal part must stage");
let part_two = Box::pin(usecase.internal_upload_part(
&ctx,
&upload_id,
2,
last_part.len() as u64,
Some(md5_hex(&last_part)),
body_stream(vec![Bytes::from(last_part.clone())]),
))
.await
.expect("last internal part must stage");
assert_eq!(part_one.part_num, 1);
assert_eq!(part_two.part_num, 2);
let obj_info = usecase
.internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two])
let obj_info = Box::pin(usecase.internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two]))
.await
.expect("internal multipart complete must succeed");
assert_eq!(obj_info.size, (first_part.len() + last_part.len()) as i64);
@@ -893,27 +1110,23 @@ mod tests {
assert_eq!(stored.user_defined.get("origin").map(String::as_str), Some("unit-test"));
assert!(contains_key_str(&stored.user_defined, SUFFIX_ODM_SOURCE));
let aborted_upload_id = usecase
.internal_create_multipart_upload(&ctx)
let aborted_upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx))
.await
.expect("second internal multipart create must succeed");
usecase
.internal_upload_part(
&ctx,
&aborted_upload_id,
1,
last_part.len() as u64,
None,
body_stream(vec![Bytes::from(last_part.clone())]),
)
.await
.expect("part of the aborted upload must stage");
usecase
.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id)
Box::pin(usecase.internal_upload_part(
&ctx,
&aborted_upload_id,
1,
last_part.len() as u64,
None,
body_stream(vec![Bytes::from(last_part.clone())]),
))
.await
.expect("part of the aborted upload must stage");
Box::pin(usecase.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id))
.await
.expect("internal abort must succeed");
let uploads = store
.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100)
let uploads = Box::pin(store.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100))
.await
.expect("list multipart uploads after abort");
assert!(
@@ -923,10 +1136,137 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn internal_multipart_completion_recomputes_replication_admission_atomically() {
let (store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu-replication")).await;
let usecase = DefaultObjectUsecase::from_global();
let target_a = "arn:aws:s3:::internal-target-a";
let target_b = "arn:aws:s3:::internal-target-b";
Box::pin(install_internal_replication_config(&bucket, Some(target_a))).await;
let payload = b"internal multipart replication body".to_vec();
let mut ctx = internal_context(&bucket, "multipart/replicated.bin", &[]);
ctx.size = None;
ctx.expected_md5_hex = None;
let foreign_generation = Uuid::from_u128(9002).to_string();
for (key, value) in [
("X-Minio-Internal-Replication-Generation", foreign_generation.as_str()),
("X-Minio-Internal-Replication-Timestamp", "foreign-multipart-time"),
("X-Minio-Internal-Replication-Status", "arn:foreign=COMPLETED;"),
("X-Minio-Internal-Replica-Status", "REPLICA"),
("X-Minio-Internal-Replica-Timestamp", "foreign-replica-time"),
] {
ctx.internal_metadata.insert(key.to_string(), value.to_string());
}
ctx.internal_metadata
.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
let upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx))
.await
.expect("internal multipart create must succeed");
let staged = Box::pin(store.get_multipart_info(&bucket, &ctx.key, &upload_id, &ObjectOptions::default()))
.await
.expect("staged internal multipart metadata must be readable");
let staged_generation = get_str(&staged.user_defined, SUFFIX_REPLICATION_GENERATION)
.expect("replication-admitted internal create must persist a generation");
Uuid::parse_str(&staged_generation).expect("staged replication generation must be a UUID");
assert_ne!(staged_generation, foreign_generation);
let staged_status = get_str(&staged.user_defined, SUFFIX_REPLICATION_STATUS)
.expect("replication-admitted internal create must persist PENDING");
assert!(staged_status.contains(target_a));
for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] {
assert!(!contains_key_str(&staged.user_defined, suffix), "staged foreign {suffix} must be removed");
}
assert!(
staged
.user_defined
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS))
.all(|(_, value)| value != "REPLICA")
);
let part = Box::pin(usecase.internal_upload_part(
&ctx,
&upload_id,
1,
payload.len() as u64,
Some(md5_hex(&payload)),
body_stream(vec![Bytes::from(payload)]),
))
.await
.expect("internal multipart part must stage");
Box::pin(install_internal_replication_config(&bucket, Some(target_b))).await;
let completed = Box::pin(usecase.internal_complete_multipart_upload(&ctx, &upload_id, vec![part]))
.await
.expect("internal multipart completion must succeed");
let completion_generation = get_str(&completed.user_defined, SUFFIX_REPLICATION_GENERATION)
.expect("completion must persist a new replication generation");
Uuid::parse_str(&completion_generation).expect("completion replication generation must be a UUID");
assert_ne!(
completion_generation, staged_generation,
"completion must replace the create-time generation"
);
let completion_status = completed
.replication_status_internal
.as_deref()
.expect("completion must persist its PENDING target set");
assert!(completion_status.contains(target_b));
assert!(!completion_status.contains(target_a));
for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] {
assert!(
!contains_key_str(&completed.user_defined, suffix),
"completed foreign {suffix} must remain removed"
);
}
assert!(
completed
.user_defined
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS))
.all(|(_, value)| value != "REPLICA")
);
let disabled_payload = b"internal multipart replication disabled".to_vec();
let mut disabled_ctx = internal_context(&bucket, "multipart/disabled.bin", &[]);
disabled_ctx.size = None;
disabled_ctx.expected_md5_hex = None;
Box::pin(install_internal_replication_config(&bucket, Some(target_a))).await;
let disabled_upload_id = Box::pin(usecase.internal_create_multipart_upload(&disabled_ctx))
.await
.expect("replication-admitted internal multipart create must succeed");
let disabled_part = Box::pin(usecase.internal_upload_part(
&disabled_ctx,
&disabled_upload_id,
1,
disabled_payload.len() as u64,
Some(md5_hex(&disabled_payload)),
body_stream(vec![Bytes::from(disabled_payload)]),
))
.await
.expect("disabled-case internal multipart part must stage");
Box::pin(install_internal_replication_config(&bucket, None)).await;
let disabled =
Box::pin(usecase.internal_complete_multipart_upload(&disabled_ctx, &disabled_upload_id, vec![disabled_part]))
.await
.expect("internal multipart completion with replication disabled must succeed");
assert!(disabled.replication_status_internal.is_none());
for suffix in [
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICATION_STATUS,
] {
assert!(
!contains_key_str(&disabled.user_defined, suffix),
"disabled completion must clear stale {suffix}"
);
}
}
#[tokio::test]
#[serial_test::serial]
async fn internal_complete_multipart_upload_rejects_unordered_parts() {
let (_store, bucket) = internal_put_test_bucket("internal-mpu-order").await;
let (_store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu-order")).await;
let ctx = internal_context(&bucket, "unordered.bin", &[]);
let parts = vec![
CompletePart {
@@ -938,8 +1278,8 @@ mod tests {
..Default::default()
},
];
let err = DefaultObjectUsecase::from_global()
.internal_complete_multipart_upload(&ctx, "upload", parts)
let usecase = DefaultObjectUsecase::from_global();
let err = Box::pin(usecase.internal_complete_multipart_upload(&ctx, "upload", parts))
.await
.expect_err("unordered parts must be rejected before touching the store");
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
+18 -2
View File
@@ -146,8 +146,8 @@ use rustfs_utils::http::insert_header;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_RESTORE_OPERATION_ID,
SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, get_header,
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_RESTORE_OPERATION_ID, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, get_header,
headers::{
AMZ_CONTENT_SHA256, AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS,
AMZ_MINIO_SNOWBALL_PREFIX, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE,
@@ -186,6 +186,22 @@ fn object_s3_error_default(code: S3ErrorCode) -> S3Error {
S3Error::new(code)
}
/// Remove replication history owned by another source object before a local
/// object-creation decision is made. Compatibility keys are case-insensitive,
/// so use the shared removers rather than deleting only canonical spellings.
fn remove_source_replication_bookkeeping(user_defined: &mut HashMap<String, String>) {
user_defined.retain(|key, _| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS));
for suffix in [
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP,
] {
remove_str(user_defined, suffix);
}
}
mod copy;
mod delete;
mod extract;
@@ -280,14 +280,14 @@ mod tests {
use crate::app::storage_api::object_usecase::on_demand_migration::{PullFailureReason, SourceSse};
use crate::app::storage_api::s3::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleStatus, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
ServerSideEncryptionRule, VersioningConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
};
use crate::app::storage_api::test::bucket::utils::serialize;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
use http::Method;
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, get_str};
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, contains_key_str, get_str};
use sha2::{Digest as Sha256Digest, Sha256};
use std::time::SystemTime;
use tokio::io::AsyncReadExt;
@@ -614,9 +614,16 @@ mod tests {
.expect("install bucket default SSE");
}
#[tokio::test]
#[test]
#[serial_test::serial]
async fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag() {
fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag() {
crate::app::gating_test_env::run_large_stack_test(
"odm-write-back-under-bucket-default-sse",
write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag_inner,
);
}
async fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag_inner() {
let local_sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
temp_env::async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(local_sse_master_key))], async {
let (store, bucket) = write_back_test_bucket("odm-wb-sse", false).await;
@@ -677,6 +684,10 @@ mod tests {
}
async fn install_replication_rule(bucket: &str) {
install_replication_rule_with_tag(bucket, None).await;
}
async fn install_replication_rule_with_tag(bucket: &str, required_tag: Option<(&str, &str)>) {
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system");
let metadata = {
let sys = sys.read().await;
@@ -700,7 +711,13 @@ mod tests {
..Default::default()
},
existing_object_replication: None,
filter: None,
filter: required_tag.map(|(key, value)| ReplicationRuleFilter {
tag: Some(Tag {
key: Some(key.to_string()),
value: Some(value.to_string()),
}),
..Default::default()
}),
id: Some("odm".to_string()),
prefix: Some(String::new()),
priority: Some(1),
@@ -750,6 +767,75 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn write_back_namespaces_source_user_metadata_that_looks_like_replication_bookkeeping() {
let (store, bucket) = write_back_test_bucket("odm-wb-user-metadata", true).await;
let target = "arn:aws:s3:::target-bucket";
install_replication_rule_with_tag(&bucket, Some(("replicate", "yes"))).await;
let body = b"initially unadmitted ODM object".to_vec();
let mut head = source_head(&body);
let forged_status_key = "X-Minio-Internal-Replication-Status";
let forged_generation_key = "X-RuStFs-InTeRnAl-RePlIcAtIoN-GeNeRaTiOn";
let forged_replica_key = "X-Minio-Internal-Replica-Status";
let forged_status = format!("{target}=COMPLETED;");
let forged_generation = Uuid::from_u128(9003).to_string();
head.user_metadata
.insert(forged_status_key.to_string(), forged_status.clone());
head.user_metadata
.insert(forged_generation_key.to_string(), forged_generation.clone());
head.user_metadata
.insert(forged_replica_key.to_string(), ReplicationStatusType::Replica.as_str().to_string());
let mut write_request = request(&bucket, "unadmitted.txt", head);
write_request.tags = Some(HashMap::from([("replicate".to_string(), "no".to_string())]));
OnDemandMigrationWriteBack::new()
.put_object(&write_request, body_stream(&body))
.await
.expect("ODM write-back with reserved-looking user metadata must commit safely");
let stored = stored_object(&store, &bucket, "unadmitted.txt").await;
for suffix in [
SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICA_STATUS,
] {
assert!(
!contains_key_str(&stored.user_defined, suffix),
"source user metadata must not become trusted {suffix} bookkeeping"
);
}
for (key, value) in [
(forged_status_key, forged_status.as_str()),
(forged_generation_key, forged_generation.as_str()),
(forged_replica_key, ReplicationStatusType::Replica.as_str()),
] {
let namespaced = format!("x-amz-meta-{key}");
assert!(
stored
.user_defined
.iter()
.any(|(stored_key, stored_value)| stored_key.eq_ignore_ascii_case(&namespaced) && stored_value == value),
"reserved-looking source metadata must survive only in the user namespace: {namespaced}"
);
}
let later_matching = crate::app::storage_api::object_usecase::bucket::replication::must_replicate_metadata(
&bucket,
"unadmitted.txt",
&stored.user_defined,
"replicate=yes".to_string(),
stored.replication_status.clone(),
ObjectOptions::default(),
)
.await;
assert!(
!later_matching.replicate_any(),
"source user metadata must not forge historical target admission after a later matching tag"
);
}
#[test]
fn content_headers_follow_the_allowlist() {
let head = SourceHead {
+1
View File
@@ -1854,6 +1854,7 @@ impl DefaultObjectUsecase {
rustfs_io_metrics::record_put_object_stage_duration_from("app_replication_decision", replication_decision_stage_start);
if dsc.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
+2 -2
View File
@@ -30,8 +30,8 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleStatus, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
ServerSideEncryptionRule, VersioningConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
};
}
+7 -1
View File
@@ -45,7 +45,9 @@ use rustfs_targets::EventName;
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str};
use rustfs_utils::http::{
SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str,
};
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
use std::collections::HashMap;
use std::fmt::Debug;
@@ -679,6 +681,7 @@ impl S3 for FS {
.await;
if dsc.replicate_any() {
let mut eval_metadata = HashMap::new();
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
insert_str(
@@ -1602,6 +1605,7 @@ impl S3 for FS {
let mut eval_metadata = parse_object_lock_legal_hold(legal_hold)?;
if dsc.replicate_any() {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
}
@@ -1820,6 +1824,7 @@ impl S3 for FS {
let mut eval_metadata = parse_object_lock_retention(retention)?;
if dsc.replicate_any() {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
}
@@ -1908,6 +1913,7 @@ impl S3 for FS {
.await;
if dsc.replicate_any() {
let mut eval_metadata = HashMap::new();
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
insert_str(
+4
View File
@@ -1876,6 +1876,10 @@ mod tests {
let authorized = get_complete_multipart_upload_opts_with_replication_authorization(&headers, true)
.expect("authorized replica status header should parse");
assert!(
!authorized.replication_request,
"REPLICA-only compatibility requests intentionally omit the source-request marker"
);
assert_eq!(authorized.delete_marker_replication_status(), ReplicationStatusType::Replica);
// For multipart the on-disk stamp actually comes from the SAME header
// at initiate time (create-multipart builds its options through