mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-07 22:03:14 +00:00
fix(replication): schedule replication for CopyObject and snowball extracted objects (#5753)
* test(replication): expect CopyObject and snowball extract to schedule replication
Red-phase TDD tests for P0-6: CopyObject never consults the bucket
replication config (no pending stamp, no schedule, and the destination
inherits the source's stale replication status metadata wholesale), and
snowball auto-extract members are never scheduled either.
- usecase white-box: observe MUST_REPLICATE_OBJECT_CALLS for
execute_copy_object (currently 0, must be 1) and
execute_put_object_extract (currently 0, must be 2 for a two-member
archive), plus stale replication-status metadata cleanup assertions
(MinIO filterReplicationStatusMetadata parity).
- e2e: CopyObject destination and snowball-extracted members must appear
on the remote replication target and reach COMPLETED on the source.
Red evidence (before fix):
copy_object_computes_replication_decision_and_strips_stale_status
assertion failed: left: 0, right: 1
put_object_extract_computes_replication_decision_per_entry
assertion failed: left: 0, right: 2
* fix(replication): schedule replication for CopyObject and snowball extracted objects
CopyObject and snowball auto-extract never consulted the bucket
replication config: no PENDING stamp, no post-commit schedule, and no
scanner-heal backstop (heal only re-drives Pending/Failed objects, and
these objects carried no status at all). Worse, the copy path cloned the
source metadata wholesale, so a destination object inherited the
source's replication bookkeeping and could present a fake
COMPLETED/REPLICA state.
Mirroring the PUT path (single immutable decision drives both the
pending metadata and the post-commit schedule, rustfs/backlog#1320):
- execute_copy_object: strip the source's replication status metadata
(internal replication/replica status + timestamps under both
compatibility prefixes, plus x-amz-replication-status) for
non-inbound requests — MinIO filterReplicationStatusMetadata parity;
the cleanup runs before the decision so an inherited REPLICA status
cannot suppress it. Then compute must_replicate_object once, stamp
PENDING when it replicates, and schedule after the copy commits and
the self-copy lock guard is released. Inbound replica writes keep
their authorized metadata and are declined inside
must_replicate_object, so replicas are never re-scheduled outbound.
- execute_put_object_extract: same stamp + schedule per extracted
member object (MinIO PutObjectExtract parity).
- execute_put_object dispatch: an authorized inbound replication PUT is
stored verbatim instead of being re-dispatched into the extract path.
Extracted members keep x-amz-meta-snowball-auto-extract in their user
metadata and the replication client replays stored metadata as
headers, so the target used to try to untar each member's own bytes,
permanently failing replication for non-archive members (surfaced by
the new snowball e2e test).
Green evidence:
- copy_object_computes_replication_decision_and_strips_stale_status,
put_object_extract_computes_replication_decision_per_entry (red: 0
decisions; green: 1 and 2), plus the existing PUT/object-lock
decision-count tests stay green.
- e2e test_copy_object_replicates_to_target and
test_snowball_extract_replicates_members_to_target pass against two
live instances.
This commit is contained in:
@@ -2436,6 +2436,107 @@ async fn build_replication_pair(
|
||||
Ok((source_env, target_env, source_bucket.to_string()))
|
||||
}
|
||||
|
||||
/// P0-6: CopyObject creates a new object on the destination key, so it must be
|
||||
/// scheduled for bucket replication exactly like PutObject (MinIO
|
||||
/// CopyObjectHandler parity). Before the fix the copy path never consulted the
|
||||
/// replication config: the destination object stayed local forever (its status
|
||||
/// metadata was inherited wholesale from the source, so the scanner heal pass
|
||||
/// skipped it too — no PENDING/FAILED marker meant nothing to re-drive).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_replicates_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let target_bucket = "replication-check-dst";
|
||||
|
||||
let src_key = "copy-repl-source.txt";
|
||||
let dst_key = "copy-repl-destination.txt";
|
||||
let payload = b"copy object replication payload".to_vec();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(src_key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(wait_for_object_on_target(&target_client, target_bucket, src_key).await?, payload);
|
||||
// Wait for the source object's terminal COMPLETED status so the copy below
|
||||
// starts from metadata that carries a stale terminal replication state; the
|
||||
// copy must not inherit it (MinIO filterReplicationStatusMetadata parity)
|
||||
// and must drive its own PENDING -> COMPLETED cycle.
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, src_key, "COMPLETED", false).await?;
|
||||
|
||||
source_client
|
||||
.copy_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(dst_key)
|
||||
.copy_source(format!("{source_bucket}/{src_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
wait_for_object_on_target(&target_client, target_bucket, dst_key).await?,
|
||||
payload,
|
||||
"CopyObject destination must replicate to the remote target"
|
||||
);
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, dst_key, "COMPLETED", false).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P0-6 companion: snowball auto-extract writes each archive member as an
|
||||
/// independent object; every member must replicate to the remote target like a
|
||||
/// regular PUT (MinIO PutObjectExtract parity).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_snowball_extract_replicates_members_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let target_bucket = "replication-check-dst";
|
||||
|
||||
let members: [(&str, &[u8]); 2] = [
|
||||
("snowball/member-one.txt", b"first member payload"),
|
||||
("snowball/member-two.txt", b"second member payload"),
|
||||
];
|
||||
|
||||
let mut builder = tokio_tar::Builder::new(std::io::Cursor::new(Vec::new()));
|
||||
for (path, data) in members {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, path, std::io::Cursor::new(data)).await?;
|
||||
}
|
||||
let archive = builder.into_inner().await?.into_inner();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key("members.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
for (key, data) in members {
|
||||
assert_eq!(
|
||||
wait_for_object_on_target(&target_client, target_bucket, key).await?,
|
||||
data,
|
||||
"snowball-extracted member {key} must replicate to the remote target"
|
||||
);
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", false).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
|
||||
Reference in New Issue
Block a user