From 86ebcb325c23af76310b5b56d574b73ab98bfb3d Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 3 Sep 2026 16:05:58 +0800 Subject: [PATCH] fix(replication): fence stale metadata status writeback (#7083) Co-authored-by: houseme --- .../src/replication_extension_test.rs | 433 ++++++- crates/ecstore/src/bucket/replication/mod.rs | 2 +- .../replication_filemeta_boundary.rs | 2 +- .../replication/replication_object_config.rs | 51 +- .../bucket/replication/replication_pool.rs | 21 +- .../replication/replication_resyncer.rs | 403 +++++- .../replication_storage_boundary.rs | 6 +- .../replication_target_boundary.rs | 12 + crates/ecstore/src/object_api/mod.rs | 5 +- crates/ecstore/src/object_api/types.rs | 195 +++ crates/ecstore/src/set_disk/ops/multipart.rs | 248 ++++ crates/ecstore/src/set_disk/ops/object.rs | 1077 ++++++++++++++++- crates/filemeta/src/filemeta.rs | 19 + crates/filemeta/src/filemeta/version.rs | 7 +- crates/replication/src/filemeta.rs | 18 + crates/replication/src/http.rs | 54 +- crates/replication/src/lib.rs | 8 +- crates/replication/src/operation.rs | 105 +- crates/utils/src/http/metadata_compat.rs | 2 + rustfs/src/app/multipart_usecase.rs | 142 ++- rustfs/src/app/object/copy.rs | 38 +- rustfs/src/app/object/extract.rs | 1 + rustfs/src/app/object/internal_put.rs | 464 ++++++- rustfs/src/app/object/mod.rs | 20 +- .../src/app/object/on_demand_migration_put.rs | 98 +- rustfs/src/app/object/put.rs | 1 + rustfs/src/app/storage_api.rs | 4 +- rustfs/src/storage/ecfs.rs | 8 +- rustfs/src/storage/options.rs | 4 + 29 files changed, 3219 insertions(+), 229 deletions(-) diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 73f43a05a..f5d308c7e 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -69,7 +69,7 @@ use std::net::IpAddr; use std::path::Path; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use time::{Duration as TimeDuration, OffsetDateTime}; use tokio::fs; use tokio::net::TcpListener; @@ -2008,6 +2008,87 @@ fn proxy_error_response(error: impl std::fmt::Display) -> Response, + backend_committed: watch::Sender, + release: watch::Receiver, +} + +impl ReplicationResponseHoldRuntime { + fn try_claim(&self) -> bool { + self.armed + .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +struct ReplicationResponseHold { + armed: Arc, + backend_committed_signal: watch::Sender, + backend_committed: watch::Receiver, + release: watch::Sender, +} + +impl ReplicationResponseHold { + fn arm(&mut self) -> Result<(), Box> { + if self.armed.load(Ordering::Acquire) { + return Err("replication response hold was already armed".into()); + } + self.backend_committed_signal + .send(false) + .map_err(|_| "replication response hold closed before rearming")?; + self.release + .send(false) + .map_err(|_| "replication response hold closed before rearming")?; + self.armed + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| "replication response hold was already armed")?; + Ok(()) + } + + async fn wait_for_backend_commit(&mut self) -> Result<(), Box> { + let wait = async { + while !*self.backend_committed.borrow() { + self.backend_committed + .changed() + .await + .map_err(|_| "replication response hold closed before the backend committed")?; + } + Ok::<(), Box>(()) + }; + timeout(Duration::from_secs(60), wait) + .await + .map_err(|_| "timed out waiting for the replication backend to commit")? + } + + fn release(&self) -> Result<(), Box> { + self.release + .send(true) + .map_err(|_| "replication response hold closed before release")?; + Ok(()) + } +} + +fn replication_response_hold() -> (ReplicationResponseHoldRuntime, ReplicationResponseHold) { + let armed = Arc::new(AtomicBool::new(false)); + let (backend_committed, backend_committed_rx) = watch::channel(false); + let (release, release_rx) = watch::channel(false); + ( + ReplicationResponseHoldRuntime { + armed: armed.clone(), + backend_committed: backend_committed.clone(), + release: release_rx, + }, + ReplicationResponseHold { + armed, + backend_committed_signal: backend_committed, + backend_committed: backend_committed_rx, + release, + }, + ) +} + async fn forward_replication_proxy_request( request: Request, backend_url: &str, @@ -2015,6 +2096,7 @@ async fn forward_replication_proxy_request( request_count: &AtomicU64, mut replication_enabled: watch::Receiver, mut held_tagging: watch::Receiver>, + mut response_hold: ReplicationResponseHoldRuntime, ) -> Response> { let (parts, body) = request.into_parts(); let is_replication = parts @@ -2040,6 +2122,7 @@ async fn forward_replication_proxy_request( } } } + let hold_response = is_replication && parts.method == http::Method::PUT && response_hold.try_claim(); let Some(path_and_query) = parts.uri.path_and_query() else { return proxy_error_response("request URI omitted path"); @@ -2062,6 +2145,16 @@ async fn forward_replication_proxy_request( Ok(body) => body, Err(error) => return proxy_error_response(error), }; + if hold_response && status.is_success() { + if response_hold.backend_committed.send(true).is_err() { + return proxy_error_response("replication response hold closed after the backend committed"); + } + while !*response_hold.release.borrow() { + if response_hold.release.changed().await.is_err() { + return proxy_error_response("replication response hold closed before release"); + } + } + } let mut proxied = Response::builder().status(status); for (name, value) in &headers { proxied = proxied.header(name, value); @@ -2073,7 +2166,7 @@ async fn start_replication_counting_proxy( backend_url: &str, tasks: &mut JoinSet<()>, ) -> Result<(String, Arc, watch::Sender), Box> { - let (proxy_url, request_count, replication_enabled, _held_tagging) = + let (proxy_url, request_count, replication_enabled, _held_tagging, _response_hold) = start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?; Ok((proxy_url, request_count, replication_enabled)) } @@ -2085,7 +2178,16 @@ async fn start_replication_counting_proxy( async fn start_replication_counting_proxy_with_tag_hold( backend_url: &str, tasks: &mut JoinSet<()>, -) -> Result<(String, Arc, watch::Sender, watch::Sender>), Box> { +) -> Result< + ( + String, + Arc, + watch::Sender, + watch::Sender>, + ReplicationResponseHold, + ), + Box, +> { let listener = TcpListener::bind("127.0.0.1:0").await?; let proxy_url = format!("http://{}", listener.local_addr()?); let backend_url = backend_url.to_string(); @@ -2093,6 +2195,7 @@ async fn start_replication_counting_proxy_with_tag_hold( let task_request_count = request_count.clone(); let (replication_enabled, task_replication_enabled) = watch::channel(true); let (held_tagging, task_held_tagging) = watch::channel(None); + let (response_hold_runtime, response_hold) = replication_response_hold(); tasks.spawn(async move { let client = local_http_client(); let mut connections = JoinSet::new(); @@ -2105,6 +2208,7 @@ async fn start_replication_counting_proxy_with_tag_hold( let request_count = task_request_count.clone(); let replication_enabled = task_replication_enabled.clone(); let held_tagging = task_held_tagging.clone(); + let response_hold = response_hold_runtime.clone(); connections.spawn(async move { let service = service_fn(move |request| { let backend_url = backend_url.clone(); @@ -2112,6 +2216,7 @@ async fn start_replication_counting_proxy_with_tag_hold( let request_count = request_count.clone(); let replication_enabled = replication_enabled.clone(); let held_tagging = held_tagging.clone(); + let response_hold = response_hold.clone(); async move { Ok::<_, Infallible>( forward_replication_proxy_request( @@ -2121,6 +2226,7 @@ async fn start_replication_counting_proxy_with_tag_hold( &request_count, replication_enabled, held_tagging, + response_hold, ) .await, ) @@ -2133,7 +2239,7 @@ async fn start_replication_counting_proxy_with_tag_hold( } } }); - Ok((proxy_url, request_count, replication_enabled, held_tagging)) + Ok((proxy_url, request_count, replication_enabled, held_tagging, response_hold)) } async fn site_replication_remove( @@ -3824,6 +3930,12 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR let mut source_env = RustFSTestEnvironment::new().await?; let mut source_env_vars = replication_fast_env(); source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + // This matrix verifies request-time rule/admission behavior. Keep the + // background existing-object scanner outside the observation window: with + // ExistingObjectReplication enabled it may legitimately discover an + // object after its tags change, which is a separate data-replication path + // that PR #5696 intentionally did not alter. + source_env_vars.push(("RUSTFS_SCANNER_START_DELAY_SECS", "300")); source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?; let mut target_env_a = RustFSTestEnvironment::new().await?; @@ -4052,6 +4164,13 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR .await?; assert_replication_key_absent(&target_client_b, target_bucket_b, "tagged/no-match.txt", Duration::from_secs(3)).await?; + // A metadata edit must not retroactively admit data that failed the tag + // filter at PUT time. This is the PR #5696 safety boundary: the target ARN + // has no persisted data-admission state for this version, so adding the + // matching tag later remains metadata-only and fails closed. + put_single_tag_current(&source_client, source_bucket, "tagged/no-match.txt", "route", "tagged").await?; + assert_replication_key_absent(&target_client_b, target_bucket_b, "tagged/no-match.txt", Duration::from_secs(3)).await?; + source_client .put_object() .bucket(source_bucket) @@ -7155,6 +7274,27 @@ async fn put_single_tag( Ok(()) } +async fn put_single_tag_current( + client: &Client, + bucket: &str, + key: &str, + tag_key: &str, + tag_value: &str, +) -> Result<(), Box> { + client + .put_object_tagging() + .bucket(bucket) + .key(key) + .tagging( + aws_sdk_s3::types::Tagging::builder() + .tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?) + .build()?, + ) + .send() + .await?; + Ok(()) +} + async fn get_single_tag( client: &Client, bucket: &str, @@ -7202,6 +7342,28 @@ async fn wait_for_single_tag( } } +/// Poll one site until `tag_key` is absent from the selected version. +async fn wait_for_tag_absent( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + tag_key: &str, + site: &str, +) -> Result<(), Box> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + loop { + let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?; + if observed.is_none() { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!("{site}: {bucket}/{key}?versionId={version_id} tag {tag_key} remained {observed:?}").into()); + } + sleep(Duration::from_millis(200)).await; + } +} + /// Tag key the dual-node LWW scenario edits on both sites. const LWW_TAG_KEY: &str = "owner"; @@ -7249,6 +7411,265 @@ async fn wait_for_proxy_replication_requests( } } +/// rustfs/backlog#2099: metadata admission must not drop a tag edit made after +/// the target has committed the initial object but before the source persists +/// that replication as COMPLETED. +#[tokio::test] +async fn test_site_replication_tagging_during_initial_pending_window_converges() -> TestResult { + init_logging(); + + // `RustFSTestEnvironment::start_rustfs_server_with_env` resolves (and on a + // cold checkout builds) this binary synchronously. Keep that setup outside + // the scenario timeout so 180 seconds measures the runtime race rather + // than compilation latency. + let _rustfs_binary = rustfs_binary_path(); + + match timeout(Duration::from_secs(180), async { + const PAYLOAD: &str = "tagging during pending replication"; + const TAG_KEY: &str = "window"; + const TAG_VALUE: &str = "pending"; + const DELETE_PAYLOAD: &str = "tag deletion during pending replication"; + const DELETE_TAG_KEY: &str = "remove"; + const DELETE_TAG_VALUE: &str = "while-pending"; + + let mut site_env = replication_fast_env(); + site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + + let mut site_a_env = RustFSTestEnvironment::new().await?; + site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let mut site_b_env = RustFSTestEnvironment::new().await?; + site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let site_a_client = site_a_env.create_s3_client(); + let site_b_client = site_b_env.create_s3_client(); + let mut proxy_tasks = JoinSet::new(); + let ( + site_b_proxy, + _site_b_replication_requests, + _site_b_replication_enabled, + _site_b_held_tagging, + mut site_b_response_hold, + ) = start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?; + + let add_status = site_replication_add( + &site_a_env, + &[ + PeerSite { + name: "pending-site-a".to_string(), + endpoint: site_a_env.url.clone(), + access_key: site_a_env.access_key.clone(), + secret_key: site_a_env.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "pending-site-b".to_string(), + endpoint: site_b_env.url.clone(), + access_key: site_b_env.access_key.clone(), + secret_key: site_b_env.secret_key.clone(), + ..Default::default() + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {add_status:?}"); + + let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?; + wait_for_site_replication_enabled(&site_b_env, 2).await?; + let mut site_b_peer = site_info + .sites + .iter() + .find(|peer| peer.endpoint == site_b_env.url.as_str()) + .ok_or("site B peer missing from replication info")? + .clone(); + site_b_peer.endpoint = site_b_proxy.clone(); + site_b_peer.sync_state = SyncStatus::Enable; + let edit = site_replication_edit(&site_a_env, "", &site_b_peer).await?; + assert!(edit.success, "unexpected site B endpoint edit: {edit:?}"); + for env in [&site_a_env, &site_b_env] { + wait_for_site_replication_info(env, |info| info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)).await?; + } + + let bucket = "site-repl-tag-pending"; + let key = "pending-window.txt"; + site_a_client.create_bucket().bucket(bucket).send().await?; + wait_for_bucket_on_target(&site_b_client, bucket).await?; + + site_b_response_hold.arm()?; + let put_task = { + let client = site_a_client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + tokio::spawn(async move { + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(PAYLOAD.as_bytes())) + .send() + .await + }) + }; + + // The proxy reads B's complete successful response before parking it, + // so B's GET proves the object is committed while A's worker is still + // unable to persist COMPLETED. + site_b_response_hold.wait_for_backend_commit().await?; + wait_for_replicated_object(&site_b_client, bucket, key, PAYLOAD).await?; + let source_head = site_a_client.head_object().bucket(bucket).key(key).send().await?; + let version_id = source_head + .version_id() + .ok_or("source HEAD omitted the pending version ID")? + .to_string(); + assert_eq!( + source_head.replication_status().map(|status| status.as_str()), + Some("PENDING"), + "source must still report PENDING while the initial replication response is held" + ); + + let tag_task = { + let client = site_a_client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + // The original real-machine failure used the current-version S3 + // API (no versionId). The delete phase below deliberately keeps an + // explicit versionId so both request shapes stay covered. + tokio::spawn(async move { put_single_tag_current(&client, &bucket, &key, TAG_KEY, TAG_VALUE).await }) + }; + wait_for_single_tag(&site_a_client, bucket, key, &version_id, TAG_KEY, TAG_VALUE, "site A").await?; + assert_eq!( + head_replication_status(&site_a_client, bucket, key, &version_id) + .await? + .as_deref(), + Some("PENDING"), + "tag update must be authored before the initial replication reaches COMPLETED" + ); + + site_b_response_hold.release()?; + let put_output = timeout(Duration::from_secs(60), put_task) + .await + .map_err(|_| "source PutObject remained blocked after releasing the replication response")???; + timeout(Duration::from_secs(60), tag_task) + .await + .map_err(|_| "PutObjectTagging remained blocked after releasing the replication response")???; + assert_eq!(put_output.version_id(), Some(version_id.as_str())); + + wait_for_single_tag(&site_b_client, bucket, key, &version_id, TAG_KEY, TAG_VALUE, "site B").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?; + + let source_state = list_replication_state(&site_a_client, bucket).await?; + let target_state = list_replication_state(&site_b_client, bucket).await?; + assert_eq!(source_state, target_state, "tag replication must not fork the object version"); + assert_eq!(source_state.len(), 1, "tag replication must leave exactly one object version"); + assert_eq!(source_state[0].key, key); + assert_eq!(source_state[0].version_id, version_id); + + // Re-arm the same response barrier for an initially tagged object. + // Deleting its tag while A is still PENDING proves the same admission + // rule covers DeleteObjectTagging rather than only the original PUT + // symptom. + let delete_key = "pending-delete-window.txt"; + site_b_response_hold.arm()?; + let delete_put_task = { + let client = site_a_client.clone(); + let bucket = bucket.to_string(); + let key = delete_key.to_string(); + tokio::spawn(async move { + client + .put_object() + .bucket(bucket) + .key(key) + .tagging(format!("{DELETE_TAG_KEY}={DELETE_TAG_VALUE}")) + .body(ByteStream::from_static(DELETE_PAYLOAD.as_bytes())) + .send() + .await + }) + }; + + site_b_response_hold.wait_for_backend_commit().await?; + wait_for_replicated_object(&site_b_client, bucket, delete_key, DELETE_PAYLOAD).await?; + let delete_source_head = site_a_client.head_object().bucket(bucket).key(delete_key).send().await?; + let delete_version_id = delete_source_head + .version_id() + .ok_or("source HEAD omitted the pending tagged version ID")? + .to_string(); + assert_eq!( + delete_source_head.replication_status().map(|status| status.as_str()), + Some("PENDING"), + "source tagged object must remain PENDING while its initial response is held" + ); + wait_for_single_tag( + &site_b_client, + bucket, + delete_key, + &delete_version_id, + DELETE_TAG_KEY, + DELETE_TAG_VALUE, + "site B", + ) + .await?; + + let delete_tag_task = { + let client = site_a_client.clone(); + let bucket = bucket.to_string(); + let key = delete_key.to_string(); + let version_id = delete_version_id.clone(); + tokio::spawn(async move { + client + .delete_object_tagging() + .bucket(bucket) + .key(key) + .version_id(version_id) + .send() + .await + }) + }; + wait_for_tag_absent(&site_a_client, bucket, delete_key, &delete_version_id, DELETE_TAG_KEY, "site A").await?; + assert_eq!( + head_replication_status(&site_a_client, bucket, delete_key, &delete_version_id) + .await? + .as_deref(), + Some("PENDING"), + "tag deletion must be authored before the initial replication reaches COMPLETED" + ); + + site_b_response_hold.release()?; + let delete_put_output = timeout(Duration::from_secs(60), delete_put_task) + .await + .map_err(|_| "source tagged PutObject remained blocked after releasing the replication response")???; + timeout(Duration::from_secs(60), delete_tag_task) + .await + .map_err(|_| "DeleteObjectTagging remained blocked after releasing the replication response")???; + assert_eq!(delete_put_output.version_id(), Some(delete_version_id.as_str())); + + wait_for_tag_absent(&site_b_client, bucket, delete_key, &delete_version_id, DELETE_TAG_KEY, "site B").await?; + wait_for_version_replication_status(&site_a_client, bucket, delete_key, &delete_version_id, &["COMPLETED"], "site A") + .await?; + wait_for_version_replication_status(&site_b_client, bucket, delete_key, &delete_version_id, &["REPLICA"], "site B") + .await?; + + let source_state = list_replication_state(&site_a_client, bucket).await?; + let target_state = list_replication_state(&site_b_client, bucket).await?; + assert_eq!(source_state, target_state, "tag deletion must not fork the object version"); + assert_eq!(source_state.len(), 2, "pending-window scenarios must leave exactly two object versions"); + assert!( + source_state + .iter() + .any(|entry| entry.key == delete_key && entry.version_id == delete_version_id), + "tag deletion must preserve the original version identity" + ); + + proxy_tasks.abort_all(); + Ok(()) + }) + .await + { + Ok(result) => result, + Err(_) => Err("pending-window site-replication tagging test timed out".into()), + } +} + /// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated /// metadata categories, exercised end to end over the real dual-node /// active-active site-replication control plane — sender, worker, status @@ -7285,9 +7706,9 @@ async fn test_site_replication_tagging_lww_converges_active_active_real_dual_nod site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?; let mut proxy_tasks = JoinSet::new(); - let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) = + let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging, _site_a_response_hold) = start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?; - let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) = + let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging, _site_b_response_hold) = start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?; let site_a_client = site_a_env.create_s3_client(); diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index a60dd018d..86271d419 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -53,12 +53,12 @@ pub use replication_config_boundary::{ replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, }; -pub(crate) use replication_filemeta_boundary::version_purge_statuses_map; pub use replication_filemeta_boundary::{ MrfOpKind, MrfReplicateEntry, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, version_purge_status_to_filemeta, }; +pub(crate) use replication_filemeta_boundary::{ReplicationGenerationSnapshot, version_purge_statuses_map}; pub(crate) use replication_filemeta_boundary::{ replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta, }; diff --git a/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs b/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs index 917ac65a8..34713cd01 100644 --- a/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs @@ -15,7 +15,7 @@ pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry}; pub(crate) use rustfs_replication::{ REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, - ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision, + ReplicationGenerationSnapshot, ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map, }; pub use rustfs_replication::{ diff --git a/crates/ecstore/src/bucket/replication/replication_object_config.rs b/crates/ecstore/src/bucket/replication/replication_object_config.rs index e059d39e8..655826a75 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_config.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_config.rs @@ -575,7 +575,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic let mut sopts = opts.clone(); sopts.target_arn = arn.clone(); - let replicate = cfg.replicate(&sopts) && mopts.metadata_target_is_eligible(&arn); + let replicate = metadata_target_should_replicate(&cfg, &sopts, &mopts, &arn); let synchronous = if let Some(cli) = cli { cli.replicate_sync } else { false }; dsc.set(ReplicateTargetDecision::new(arn, replicate, synchronous)); @@ -584,6 +584,15 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic dsc } +fn metadata_target_should_replicate( + cfg: &ReplicationConfiguration, + opts: &ObjectOpts, + mopts: &MustReplicateOptions, + arn: &str, +) -> bool { + cfg.replicate(opts) && mopts.metadata_target_is_eligible(arn) +} + #[cfg(test)] mod tests { use s3s::dto::{ @@ -613,6 +622,46 @@ mod tests { } } + #[test] + fn metadata_replication_requires_both_current_rule_match_and_historical_admission() { + let arn = "arn:rustfs:replication:us-east-1:target:bucket"; + let mut rule = replication_rule(); + rule.destination.bucket = arn.to_string(); + rule.filter = Some(ReplicationRuleFilter { + prefix: Some("admitted/".to_string()), + ..Default::default() + }); + let cfg = ReplicationConfiguration { + role: String::new(), + rules: vec![rule], + }; + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_STATUS, format!("{arn}=PENDING;")); + let admitted = MustReplicateOptions::new(&metadata, String::new(), ReplicationType::Metadata, false); + let matching = ObjectOpts { + name: "admitted/object".to_string(), + target_arn: arn.to_string(), + ..Default::default() + }; + assert!(metadata_target_should_replicate(&cfg, &matching, &admitted, arn)); + + let rule_mismatch = ObjectOpts { + name: "outside/object".to_string(), + target_arn: arn.to_string(), + ..Default::default() + }; + assert!( + !metadata_target_should_replicate(&cfg, &rule_mismatch, &admitted, arn), + "historical admission must not bypass the current replication rule" + ); + + let never_admitted = MustReplicateOptions::new(&HashMap::new(), String::new(), ReplicationType::Metadata, false); + assert!( + !metadata_target_should_replicate(&cfg, &matching, &never_admitted, arn), + "a current rule match must not create historical admission" + ); + } + #[test] fn replication_config_empty_and_replicate_follow_config() { let empty = ReplicationConfig::default(); diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 9b227c7c4..b3bf828e7 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -52,7 +52,6 @@ use super::runtime_boundary as runtime_sources; use futures_util::stream::{self, StreamExt}; use metrics::{counter, histogram}; use rustfs_utils::hash::HashAlgorithm; -use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str}; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; @@ -939,7 +938,11 @@ async fn replay_mrf_object_entry( Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await) } else { let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type()); - if replicate_object_with_outcome(roi, storage.clone()).await.1 { + if replicate_object_with_outcome(roi, storage.clone()) + .await + .1 + .consumes_mrf_entry() + { Some(ReplicationQueueAdmission::Queued) } else { Some(ReplicationQueueAdmission::Missed) @@ -978,7 +981,11 @@ async fn replay_mrf_metadata_entry( Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await) } else { let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata); - if replicate_object_with_outcome(roi, storage.clone()).await.1 { + if replicate_object_with_outcome(roi, storage.clone()) + .await + .1 + .consumes_mrf_entry() + { Some(ReplicationQueueAdmission::Queued) } else { Some(ReplicationQueueAdmission::Missed) @@ -2978,8 +2985,11 @@ fn replicate_object_info_from_object_info( ) -> ReplicateObjectInfo { let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default()); let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default()); - let tm = get_str(&oi.user_defined, SUFFIX_REPLICATION_TIMESTAMP) - .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); + let replication_generation = oi.replication_generation_snapshot(); + let tm = replication_generation + .timestamp + .as_deref() + .map(|value| OffsetDateTime::parse(value, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); let mut rstate = oi.replication_state(); rstate.replicate_decision_str = dsc.to_string(); let asz = oi.get_actual_size_or_physical(); @@ -3006,6 +3016,7 @@ fn replicate_object_info_from_object_info( target_statuses: tgt_statuses, target_purge_statuses: purge_statuses, replication_timestamp: tm, + replication_generation, user_tags: (*oi.user_tags).clone(), checksum, retry_count: 0, diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index e69154399..cf78dee70 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -17,6 +17,8 @@ use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt use super::replication_config_store::ReplicationConfigStore; use super::replication_error_boundary::{Error, Result, is_err_object_not_found, is_err_version_not_found}; use super::replication_event_sink::{EventArgs, send_event, send_local_event}; +#[cfg(test)] +use super::replication_filemeta_boundary::ReplicationGenerationSnapshot; use super::replication_filemeta_boundary::{ REPLICATE_EXISTING, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state, @@ -47,12 +49,13 @@ use super::replication_resync_boundary::{ }; #[cfg(test)] use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX}; -#[cfg(test)] -use super::replication_storage_boundary::ReplicationDeletedObject; use super::replication_storage_boundary::{ AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions, - ObjectToDelete, ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions, + ObjectToDelete, ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage, + StatObjectOptions, StorageObjectInfoOrErr, WalkOptions, }; +#[cfg(test)] +use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject}; use super::replication_target_boundary::{ ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, @@ -1625,6 +1628,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC let mut replication_state = oi.replication_state(); replication_state.replicate_decision_str = dsc.to_string(); let actual_size = oi.get_actual_size_or_physical(); + let replication_generation = oi.replication_generation_snapshot(); Ok(ReplicateObjectInfo { name: oi.name.clone(), @@ -1647,6 +1651,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC target_statuses, target_purge_statuses, replication_timestamp: None, + replication_generation, ssec: replication_object_is_ssec_encrypted(&user_defined), user_tags: (*oi.user_tags).clone(), checksum: oi.checksum.clone(), @@ -2931,10 +2936,125 @@ pub async fn replicate_object(roi: ReplicateObjectInfo, s replicate_object_with_outcome(roi, storage).await.0 } +enum ReplicationStatePersistOutcome { + Updated, + Superseded, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReplicationAttemptDisposition { + Persisted, + Superseded, + Retry, +} + +impl ReplicationAttemptDisposition { + pub(crate) fn consumes_mrf_entry(self) -> bool { + matches!(self, Self::Persisted | Self::Superseded) + } +} + +fn replication_attempt_preflight(roi: &ReplicateObjectInfo) -> Option<(ReplicationState, ReplicationAttemptDisposition)> { + roi.replication_generation + .invalid + .then(|| (roi.replication_state.clone().unwrap_or_default(), ReplicationAttemptDisposition::Retry)) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct ReplicationTerminalPublication { + emit_terminal_failure: bool, + emit_event: bool, + update_transition_stats: bool, + update_same_state_failure_stats: bool, +} + +fn replication_terminal_publication( + suppress_terminal_publication: bool, + state_update_needed: bool, + attempt_status: ReplicationStatusType, + previous_internal_matches_attempt: bool, +) -> ReplicationTerminalPublication { + if suppress_terminal_publication { + return ReplicationTerminalPublication::default(); + } + ReplicationTerminalPublication { + emit_terminal_failure: true, + emit_event: true, + update_transition_stats: state_update_needed, + update_same_state_failure_stats: attempt_status != ReplicationStatusType::Completed && previous_internal_matches_attempt, + } +} + +fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationStatusWritebackMode { + if state_update_needed { + ReplicationStatusWritebackMode::Update + } else { + ReplicationStatusWritebackMode::ValidateOnly + } +} + +/// Publish a worker's status with a storage-enforced compare-and-set token. +/// `put_object_metadata` checks the token while it owns the object write lock, +/// avoiding both a read/write race and any need to hold a hot object lock +/// across network I/O. +fn replication_status_writeback_options( + roi: &ReplicateObjectInfo, + replication_lock_guard: &rustfs_lock::NamespaceLockGuard, + new_replication_internal: Option<&String>, + mode: ReplicationStatusWritebackMode, +) -> ObjectOptions { + let mut eval_metadata = HashMap::new(); + if let Some(status) = new_replication_internal { + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, status.clone()); + } + let mut write_opts = ObjectOptions { + version_id: roi.version_id.map(|version_id| version_id.to_string()), + eval_metadata: Some(eval_metadata), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: roi.replication_generation.clone(), + mode, + })), + ..Default::default() + }; + // The remote transfer runs under a renewable replication namespace lock. + // Carry that guard's loss signal into the storage commit so a worker whose + // lease expired while it was doing remote I/O cannot publish over a newer + // worker for the same generation. + write_opts.add_namespace_lock_guard(replication_lock_guard); + write_opts +} + +async fn persist_replication_state_if_current( + roi: &ReplicateObjectInfo, + storage: &Arc, + replication_lock_guard: &rustfs_lock::NamespaceLockGuard, + new_replication_internal: Option<&String>, + mode: ReplicationStatusWritebackMode, + object_info: &mut ObjectInfo, +) -> Result { + let write_opts = replication_status_writeback_options(roi, replication_lock_guard, new_replication_internal, mode); + match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await { + Ok(updated) => { + *object_info = updated; + Ok(ReplicationStatePersistOutcome::Updated) + } + Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded), + Err(error) => Err(error), + } +} + pub(crate) async fn replicate_object_with_outcome( roi: ReplicateObjectInfo, storage: Arc, -) -> (ReplicationState, bool) { +) -> (ReplicationState, ReplicationAttemptDisposition) { + // Conflicting compatibility aliases, empty opaque timestamps, and invalid + // mutation UUIDs are corruption, not evidence that another generation + // superseded this task. Fail before any target I/O and keep the MRF entry + // retryable instead of repeatedly transmitting and then acknowledging it. + if let Some(outcome) = replication_attempt_preflight(&roi) { + return outcome; + } + let bucket = roi.bucket.clone(); let object = roi.name.clone(); @@ -2963,10 +3083,10 @@ pub(crate) async fn replicate_object_with_outcome( user_agent: "Internal: [Replication]".to_string(), ..Default::default() }); - return (roi.replication_state.unwrap_or_default(), false); + return (roi.replication_state.unwrap_or_default(), ReplicationAttemptDisposition::Retry); } }; - let _obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { + let obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { Ok(g) => g, Err(e) => { debug!( @@ -2986,7 +3106,7 @@ pub(crate) async fn replicate_object_with_outcome( user_agent: "Internal: [Replication]".to_string(), ..Default::default() }); - return (roi.replication_state.unwrap_or_default(), false); + return (roi.replication_state.unwrap_or_default(), ReplicationAttemptDisposition::Retry); } }; @@ -3057,72 +3177,98 @@ pub(crate) async fn replicate_object_with_outcome( } let version_id = roi.version_id.map(|v| v.to_string()); - note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos); let previous_state = roi.replication_state.clone().unwrap_or_default(); - let merged_state = get_replication_state(&rinfos, &previous_state, version_id); - let replication_status = merged_state.composite_replication_status(); + let mut merged_state = get_replication_state(&rinfos, &previous_state, version_id.clone()); + let mut replication_status = merged_state.composite_replication_status(); let new_replication_internal = merged_state.replication_status_internal.clone(); let mut object_info = roi.to_object_info(); - let mut state_persisted = true; + let mut disposition = ReplicationAttemptDisposition::Persisted; + let mut suppress_terminal_publication = false; + let state_update_needed = roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced(); + let writeback_mode = replication_status_writeback_mode(state_update_needed); - if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() { - let mut eval_metadata = HashMap::new(); - if let Some(ref s) = new_replication_internal { - insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, s.clone()); - } - let popts = ObjectOptions { - version_id: roi.version_id.map(|v| v.to_string()), - eval_metadata: Some(eval_metadata), - ..Default::default() - }; - - match storage.put_object_metadata(&bucket, &object, &popts).await { - Ok(u) => object_info = u, - Err(e) => { - state_persisted = false; - // Persisting the resynced replication status failed. Don't swallow - // it silently — the object's on-disk status now disagrees with the - // resync result and needs operator visibility (backlog#799 B23). - warn!( - event = EVENT_RESYNC_TARGET_OPERATION_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - object = %object, - error = %e, - "Failed to persist resynced replication status metadata" - ); - } - } - - if let Some(stats) = runtime_sources::replication_stats() { - for tgt in &rinfos.targets { - if tgt.replication_status != tgt.prev_replication_status { - stats - .update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone()) - .await; + match persist_replication_state_if_current( + &roi, + &storage, + &obj_lock_guard, + new_replication_internal.as_ref(), + writeback_mode, + &mut object_info, + ) + .await + { + Ok(ReplicationStatePersistOutcome::Updated) => {} + Ok(ReplicationStatePersistOutcome::Superseded) => { + // A tag/retention/legal-hold mutation committed while this + // worker was in flight. Its PENDING state is authoritative and + // must remain discoverable by the queue/MRF/scanner after a + // crash or missed admission. Return that newer state to sync + // callers and leave its worker to publish the terminal status. + suppress_terminal_publication = true; + disposition = ReplicationAttemptDisposition::Superseded; + let read_opts = ObjectOptions { + version_id: roi.version_id.map(|version_id| version_id.to_string()), + ..Default::default() + }; + match storage.get_object_info(&bucket, &object, &read_opts).await { + Ok(current) => { + object_info = current; + merged_state = object_info.replication_state(); + replication_status = merged_state.composite_replication_status(); + } + Err(error) => { + // The CAS result is authoritative: a best-effort refetch + // failure must not turn a superseded task back into a + // retry that can publish the stale generation later. + debug!( + event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + error = %error, + reason = "source_snapshot_refetch_failed_after_superseded", + "Could not refresh source state after skipping stale replication status update" + ); } } + debug!( + event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + reason = "source_replication_snapshot_superseded", + "Skipped stale replication status update" + ); + } + Err(e) => { + disposition = ReplicationAttemptDisposition::Retry; + suppress_terminal_publication = true; + // Persisting the resynced replication status failed. Don't swallow + // it silently — the object's on-disk status now disagrees with the + // resync result and needs operator visibility (backlog#799 B23). + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + error = %e, + "Failed to persist resynced replication status metadata" + ); } } - let event_name = if replication_status == ReplicationStatusType::Completed { - EventName::ObjectReplicationComplete.to_string() - } else { - EventName::ObjectReplicationFailed.to_string() - }; + let publication = replication_terminal_publication( + suppress_terminal_publication, + state_update_needed, + rinfos.replication_status(), + roi.replication_status_internal == rinfos.replication_status_internal(), + ); - send_local_event(EventArgs { - event_name, - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - - if rinfos.replication_status() != ReplicationStatusType::Completed - && roi.replication_status_internal == rinfos.replication_status_internal() + if publication.update_transition_stats && let Some(stats) = runtime_sources::replication_stats() { for tgt in &rinfos.targets { @@ -3134,7 +3280,39 @@ pub(crate) async fn replicate_object_with_outcome( } } - (merged_state, state_persisted) + if publication.emit_terminal_failure { + note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos); + } + + let event_name = if replication_status == ReplicationStatusType::Completed { + EventName::ObjectReplicationComplete.to_string() + } else { + EventName::ObjectReplicationFailed.to_string() + }; + + if publication.emit_event { + send_local_event(EventArgs { + event_name, + bucket_name: bucket.clone(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + } + + if publication.update_same_state_failure_stats + && let Some(stats) = runtime_sources::replication_stats() + { + for tgt in &rinfos.targets { + if tgt.replication_status != tgt.prev_replication_status { + stats + .update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone()) + .await; + } + } + } + + (merged_state, disposition) } /// Emit the operator-visible record of a replication attempt that ended FAILED. @@ -4567,6 +4745,105 @@ async fn replicate_multipart_parts_and_complete( mod tests { use super::super::replication_filemeta_boundary::ReplicateTargetDecision; + #[test] + fn same_state_terminal_retry_uses_validate_only() { + assert_eq!(replication_status_writeback_mode(false), ReplicationStatusWritebackMode::ValidateOnly); + assert_eq!(replication_status_writeback_mode(true), ReplicationStatusWritebackMode::Update); + } + + #[test] + fn superseded_attempt_has_no_terminal_publication_side_effects() { + assert!(ReplicationAttemptDisposition::Persisted.consumes_mrf_entry()); + assert!(ReplicationAttemptDisposition::Superseded.consumes_mrf_entry()); + assert!(!ReplicationAttemptDisposition::Retry.consumes_mrf_entry()); + assert_eq!( + replication_terminal_publication(true, false, ReplicationStatusType::Failed, true), + ReplicationTerminalPublication::default() + ); + assert_eq!( + replication_terminal_publication(true, true, ReplicationStatusType::Completed, false), + ReplicationTerminalPublication::default() + ); + + let current = replication_terminal_publication(false, false, ReplicationStatusType::Failed, true); + assert!(current.emit_terminal_failure); + assert!(current.emit_event); + assert!(!current.update_transition_stats); + assert!(current.update_same_state_failure_stats); + + assert_eq!( + replication_terminal_publication(true, true, ReplicationStatusType::Failed, false), + ReplicationTerminalPublication::default(), + "a retryable status-persistence failure must not publish a terminal result" + ); + } + + #[test] + fn invalid_generation_retries_before_remote_replication() { + let mut preserved_state = ReplicationState::default(); + preserved_state + .targets + .insert("arn:target".to_string(), ReplicationStatusType::Pending); + let invalid = ReplicateObjectInfo { + replication_generation: ReplicationGenerationSnapshot { + invalid: true, + ..Default::default() + }, + replication_state: Some(preserved_state.clone()), + ..Default::default() + }; + + assert_eq!( + replication_attempt_preflight(&invalid), + Some((preserved_state, ReplicationAttemptDisposition::Retry)) + ); + assert!(replication_attempt_preflight(&ReplicateObjectInfo::default()).is_none()); + } + + #[tokio::test] + async fn terminal_writeback_carries_replication_lock_loss_fence() { + let lock = rustfs_lock::NamespaceLock::new( + "replication-status-writeback-fence".to_string(), + Arc::new(rustfs_lock::LocalClient::new()), + ); + let guard = lock + .get_write_lock( + rustfs_lock::ObjectKey::new("bucket", "/[replicate]/object"), + "worker-a", + std::time::Duration::from_secs(2), + ) + .await + .expect("replication lock should be acquired"); + let signal = guard + .lock_lost_signal() + .expect("distributed guard must expose its loss signal"); + let forced_lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let _test_fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&forced_lost)); + + // Build the writeback after remote work has acquired the guard, then + // lose the lease before storage reaches its commit fence. + let opts = replication_status_writeback_options( + &ReplicateObjectInfo::default(), + &guard, + None, + ReplicationStatusWritebackMode::ValidateOnly, + ); + forced_lost.store(true, std::sync::atomic::Ordering::Release); + + assert!( + opts.namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost), + "terminal CAS must observe a replication lease lost after remote I/O" + ); + assert!(!ReplicationAttemptDisposition::Retry.consumes_mrf_entry()); + assert_eq!( + replication_terminal_publication(true, true, ReplicationStatusType::Failed, false), + ReplicationTerminalPublication::default(), + "a fenced writeback retry must not publish terminal events or statistics" + ); + } + #[test] fn unavailable_object_target_is_persisted_as_failed() { let arn = "arn:object-target"; diff --git a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs index 1fa113f9c..1e39819bf 100644 --- a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs @@ -18,7 +18,11 @@ use tokio_util::sync::CancellationToken; use super::replication_error_boundary::Error; use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta}; pub(crate) type ReplicationObjectStore = crate::store::ECStore; -pub(crate) use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; +pub(crate) use crate::object_api::{ + GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, +}; +#[cfg(test)] +pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence}; pub(crate) use crate::storage_api_contracts::list::{ ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, }; diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 44b319495..784122bbb 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -849,6 +849,11 @@ mod tests { metadata.insert(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "DAREv2-HMAC-SHA256".to_string()); metadata.insert(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), "sealed".to_string()); metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), "true".to_string()); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + Uuid::from_u128(1).to_string(), + ); let object_info = ObjectInfo { user_defined: Arc::new(metadata), @@ -894,6 +899,13 @@ mod tests { assert!(!options.user_metadata.contains_key(AMZ_SERVER_SIDE_ENCRYPTION)); assert!(!options.user_metadata.contains_key(SSEC_ALGORITHM_HEADER)); assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER)); + assert!( + options + .user_metadata + .keys() + .all(|key| !key.contains(rustfs_utils::http::SUFFIX_REPLICATION_GENERATION)), + "source-local mutation generation must never cross the replication wire" + ); assert!( !options .user_metadata diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 4c46ed802..f783bd5e6 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -18,8 +18,9 @@ pub mod object_api_utils; use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::replication::{ - DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, - replication_status_from_filemeta, replication_statuses_map, version_purge_status_from_filemeta, version_purge_statuses_map, + DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationGenerationSnapshot, ReplicationState, ReplicationStatusType, + VersionPurgeStatusType, replication_status_from_filemeta, replication_statuses_map, version_purge_status_from_filemeta, + version_purge_statuses_map, }; use crate::bucket::versioning::VersioningApi as _; use crate::config::storageclass; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index f668575b9..62a45e075 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -19,6 +19,7 @@ use crate::storage_api_contracts::{ HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, }, }; +use sha2::{Digest, Sha256}; use std::io; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard}; @@ -980,6 +981,13 @@ pub struct ObjectOptions { pub lifecycle_audit_event: LcAuditEvent, pub eval_metadata: Option>, + /// Internal compare-and-set condition for replication workers publishing + /// terminal status after remote I/O. Storage validates it while holding + /// the object write lock so an older worker cannot overwrite a newer + /// mutation's PENDING state. Keep the condition boxed because + /// `ObjectOptions` is passed by value through deep storage futures. + #[doc(hidden)] + pub replication_status_writeback: Option>, pub object_lock_retention: Option, pub object_lock_delete: Option, /// Authoritative bucket Object Lock snapshot installed inside `ECStore` @@ -1000,6 +1008,21 @@ pub struct ObjectOptions { pub decommission_capacity_admission: Option>, } +#[derive(Clone, Debug, Default)] +#[doc(hidden)] +pub struct ReplicationStatusWritebackCondition { + pub(crate) expected_generation: ReplicationGenerationSnapshot, + pub(crate) mode: ReplicationStatusWritebackMode, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[doc(hidden)] +pub enum ReplicationStatusWritebackMode { + #[default] + Update, + ValidateOnly, +} + impl ObjectOptions { pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option) -> Self { Self { @@ -1086,6 +1109,7 @@ impl std::fmt::Debug for ObjectOptions { || !self.lifecycle_audit_event.event.storage_class.is_empty()), ) .field("eval_metadata_count", &self.eval_metadata.as_ref().map(HashMap::len)) + .field("replication_status_writeback", &self.replication_status_writeback.is_some()) .field("object_lock_retention", &self.object_lock_retention.is_some()) .field("object_lock_delete", &self.object_lock_delete) .field("object_lock_config_snapshot", &self.object_lock_config_snapshot.is_some()) @@ -1281,6 +1305,32 @@ impl ObjectOptions { } } +fn replication_snapshot_internal_value( + metadata: &HashMap, + suffix: &str, +) -> std::result::Result, ()> { + match rustfs_utils::http::get_consistent_str(metadata, suffix) { + Some(value) => Ok(Some(value.to_string())), + None if rustfs_utils::http::contains_key_str(metadata, suffix) => Err(()), + None => Ok(None), + } +} + +fn update_replication_fingerprint_bytes(hasher: &mut Sha256, value: &[u8]) { + let len = u64::try_from(value.len()).unwrap_or(u64::MAX); + hasher.update(len.to_le_bytes()); + hasher.update(value); +} + +fn update_replication_fingerprint_optional_str(hasher: &mut Sha256, value: Option<&str>) { + if let Some(value) = value { + hasher.update([1]); + update_replication_fingerprint_bytes(hasher, value.as_bytes()); + } else { + hasher.update([0]); + } +} + #[derive(Debug, Default)] pub struct ObjectInfo { pub bucket: String, @@ -1369,6 +1419,144 @@ impl Clone for ObjectInfo { } impl ObjectInfo { + /// Capture the source mutation snapshot used by replication workers when + /// publishing terminal status. The semantic fingerprint is recomputed at + /// the storage CAS boundary, so an older writer that preserves an unknown + /// UUID and collides on the timestamp still cannot hide a payload change. + pub(crate) fn replication_generation_snapshot(&self) -> ReplicationGenerationSnapshot { + let timestamp = replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP); + let mutation_id = + replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_GENERATION); + let tagging_timestamp = + replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_TAGGING_TIMESTAMP); + let retention_timestamp = + replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP); + let legalhold_timestamp = + replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP); + + let invalid = timestamp.is_err() + || mutation_id.is_err() + || tagging_timestamp.is_err() + || retention_timestamp.is_err() + || legalhold_timestamp.is_err(); + let timestamp = timestamp.unwrap_or_default(); + let mutation_id = mutation_id.unwrap_or_default(); + let tagging_timestamp = tagging_timestamp.unwrap_or_default(); + let retention_timestamp = retention_timestamp.unwrap_or_default(); + let legalhold_timestamp = legalhold_timestamp.unwrap_or_default(); + let opaque_timestamp_is_invalid = [ + timestamp.as_deref(), + tagging_timestamp.as_deref(), + retention_timestamp.as_deref(), + legalhold_timestamp.as_deref(), + ] + .into_iter() + .flatten() + .any(str::is_empty); + let mutation_id_is_invalid = mutation_id.as_deref().is_some_and(|value| { + Uuid::parse_str(value) + .ok() + .filter(|generation| !generation.is_nil()) + .is_none() + }); + let invalid = + invalid || opaque_timestamp_is_invalid || mutation_id_is_invalid || (mutation_id.is_some() && timestamp.is_none()); + + let payload_fingerprint = (!invalid).then(|| { + self.replication_payload_fingerprint( + tagging_timestamp.as_deref(), + retention_timestamp.as_deref(), + legalhold_timestamp.as_deref(), + ) + }); + + ReplicationGenerationSnapshot { + timestamp, + mutation_id, + payload_fingerprint, + invalid, + } + } + + fn replication_payload_fingerprint( + &self, + tagging_timestamp: Option<&str>, + retention_timestamp: Option<&str>, + legalhold_timestamp: Option<&str>, + ) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-replication-payload-v1"); + + // A suspended bucket's current null version is represented as + // `Some(Uuid::nil())` at the request/queue boundary and as `None` + // when the same xl.meta is reread without versioning flags. They are + // one persisted identity, so do not let that representation detail + // make a worker permanently supersede its own terminal write-back. + if let Some(version_id) = self.version_id.filter(|version_id| !version_id.is_nil()) { + hasher.update([1]); + hasher.update(version_id.as_bytes()); + } else { + hasher.update([0]); + } + if let Some(data_dir) = self.data_dir { + hasher.update([1]); + hasher.update(data_dir.as_bytes()); + } else { + hasher.update([0]); + } + if let Some(mod_time) = self.mod_time { + hasher.update([1]); + hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes()); + } else { + hasher.update([0]); + } + + update_replication_fingerprint_optional_str(&mut hasher, self.content_type.as_deref()); + update_replication_fingerprint_optional_str(&mut hasher, self.content_encoding.as_deref()); + update_replication_fingerprint_optional_str(&mut hasher, self.storage_class.as_deref()); + if let Some(expires) = self.expires { + hasher.update([1]); + hasher.update(expires.unix_timestamp_nanos().to_le_bytes()); + } else { + hasher.update([0]); + } + update_replication_fingerprint_bytes(&mut hasher, self.user_tags.as_bytes()); + update_replication_fingerprint_optional_str(&mut hasher, tagging_timestamp); + update_replication_fingerprint_optional_str(&mut hasher, retention_timestamp); + update_replication_fingerprint_optional_str(&mut hasher, legalhold_timestamp); + + let mut target_arns = self + .replication_status_internal + .as_deref() + .map(replication_statuses_map) + .unwrap_or_default() + .into_keys() + .collect::>(); + target_arns.sort_unstable(); + hasher.update(u64::try_from(target_arns.len()).unwrap_or(u64::MAX).to_le_bytes()); + for arn in target_arns { + update_replication_fingerprint_bytes(&mut hasher, arn.as_bytes()); + } + update_replication_fingerprint_bytes(&mut hasher, self.replication_decision.as_bytes()); + + let mut user_metadata = self + .user_defined + .iter() + .filter(|(key, _)| { + !rustfs_utils::http::is_internal_key(key) + && !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS) + }) + .collect::>(); + user_metadata.sort_unstable_by(|left, right| left.0.cmp(right.0).then_with(|| left.1.cmp(right.1))); + hasher.update(u64::try_from(user_metadata.len()).unwrap_or(u64::MAX).to_le_bytes()); + for (key, value) in user_metadata { + update_replication_fingerprint_bytes(&mut hasher, key.as_bytes()); + update_replication_fingerprint_bytes(&mut hasher, value.as_bytes()); + } + + hasher.finalize().into() + } + pub fn is_compressed(&self) -> bool { rustfs_utils::http::contains_key_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION) } @@ -2001,6 +2189,13 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker: mod tests { use super::*; + #[test] + fn replication_status_writeback_condition_remains_indirected() { + fn assert_indirected(_: &Option>) {} + + assert_indirected(&ObjectOptions::default().replication_status_writeback); + } + #[test] fn object_lock_config_snapshot_is_bound_to_store_bucket_and_incarnation() { let store_id = Uuid::new_v4(); diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index c0dc10c0a..f31a7836a 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -55,6 +55,7 @@ use crate::api::config::storageclass; #[cfg(test)] use crate::bucket::metadata_sys::ObjectLockConfigState; use crate::bucket::quota::reservation; +use crate::bucket::replication::ReplicationStatusType; use crate::crash_inject::{self, CrashPoint}; use crate::disk::DiskAPI; #[cfg(test)] @@ -2366,6 +2367,43 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; + // The request layer computes replication admission immediately before + // completion. Apply that metadata only after the staged upload and its + // bucket incarnation are validated, while the object/upload commit + // locks are still held, so generation, PENDING targets, and the final + // object become visible as one commit. + if let Some(eval_metadata) = &opts.eval_metadata { + for suffix in [ + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + ] { + rustfs_utils::http::remove_str(&mut fi.metadata, suffix); + } + // A source-side completion creates a new local object identity; + // replica bookkeeping carried by an internal/legacy staged upload + // belongs to the source object and must not survive. Authorized + // inbound replication owns these fields and remains exempt. Older + // and heterogeneous replication clients can send an authorized + // REPLICA status on Complete without repeating the source-request + // marker, so either authenticated parser result is sufficient. + let authorized_inbound_replica = + opts.replication_request || opts.delete_marker_replication_status() == ReplicationStatusType::Replica; + if !authorized_inbound_replica { + fi.metadata + .retain(|key, _| !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)); + for suffix in [ + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + ] { + rustfs_utils::http::remove_str(&mut fi.metadata, suffix); + } + } + for (key, value) in eval_metadata { + fi.metadata.insert(key.clone(), value.clone()); + } + fi.replication_state_internal = rustfs_filemeta::get_internal_replication_state(&fi.metadata); + } let has_layout_candidate = range_seek_rollout_enabled && fi .data_dir @@ -2910,6 +2948,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { meta.mod_time = fi.mod_time; meta.parts.clone_from(&fi.parts); meta.metadata = fi.metadata.clone(); + meta.replication_state_internal = fi.replication_state_internal.clone(); meta.versioned = opts.versioned || opts.version_suspended; meta.version_id = fi.version_id; meta.checksum = fi.checksum.clone(); @@ -8248,6 +8287,215 @@ mod tests { assert_eq!(current.version_id, Some(version_id)); } + #[tokio::test] + #[serial] + async fn complete_multipart_upload_commits_replication_admission_metadata_atomically() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-replication-admission-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_replication_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut create_replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + Uuid::from_u128(1).to_string(), + ); + rustfs_utils::http::insert_str( + &mut create_replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + "create-time".to_string(), + ); + rustfs_utils::http::insert_str( + &mut create_replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn:create-target=PENDING;".to_string(), + ); + rustfs_utils::http::insert_str( + &mut create_replication_metadata, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + "REPLICA".to_string(), + ); + rustfs_utils::http::insert_str( + &mut create_replication_metadata, + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + "foreign-replica-time".to_string(), + ); + create_replication_metadata + .insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string()); + let (upload_id, parts) = stage_upload_with_create_opts( + &set_disks, + bucket, + object, + b"multipart replication body", + &ObjectOptions { + user_defined: create_replication_metadata.clone(), + ..Default::default() + }, + ) + .await; + let generation = Uuid::from_u128(42).to_string(); + let timestamp = "opaque-completion-time"; + let status = "arn:complete-target=PENDING;"; + let mut replication_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + generation.clone(), + ); + rustfs_utils::http::insert_str( + &mut replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + timestamp.to_string(), + ); + rustfs_utils::http::insert_str( + &mut replication_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + status.to_string(), + ); + + let completed = set_disks + .clone() + .complete_multipart_upload( + bucket, + object, + &upload_id, + parts, + &ObjectOptions { + eval_metadata: Some(replication_metadata), + ..Default::default() + }, + ) + .await + .expect("multipart completion with replication admission should succeed"); + + assert_eq!( + rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,) + .as_deref(), + Some(generation.as_str()) + ); + assert_eq!( + rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,) + .as_deref(), + Some(timestamp) + ); + assert_eq!(completed.replication_status_internal.as_deref(), Some(status)); + for suffix in [ + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + ] { + assert!( + !rustfs_utils::http::contains_key_str(completed.user_defined.as_ref(), suffix), + "source completion must clear staged foreign {suffix}" + ); + } + assert!( + completed + .user_defined + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)) + .all(|(_, value)| value != "REPLICA") + ); + + let disabled_object = "disabled"; + let (upload_id, parts) = stage_upload_with_create_opts( + &set_disks, + bucket, + disabled_object, + b"multipart replication disabled body", + &ObjectOptions { + user_defined: create_replication_metadata, + ..Default::default() + }, + ) + .await; + let completed = set_disks + .clone() + .complete_multipart_upload( + bucket, + disabled_object, + &upload_id, + parts, + &ObjectOptions { + eval_metadata: Some(HashMap::new()), + ..Default::default() + }, + ) + .await + .expect("completion should clear stale create-time replication admission"); + assert!(completed.replication_status_internal.is_none()); + for suffix in [ + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + ] { + assert!( + !rustfs_utils::http::contains_key_str(completed.user_defined.as_ref(), suffix), + "disabled completion must clear stale {suffix}" + ); + } + + let inbound_object = "inbound"; + let mut inbound_replica_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut inbound_replica_metadata, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + "REPLICA".to_string(), + ); + rustfs_utils::http::insert_str( + &mut inbound_replica_metadata, + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + "authorized-inbound-time".to_string(), + ); + inbound_replica_metadata.insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string()); + let (upload_id, parts) = stage_upload_with_create_opts( + &set_disks, + bucket, + inbound_object, + b"authorized inbound multipart body", + &ObjectOptions { + user_defined: inbound_replica_metadata, + ..Default::default() + }, + ) + .await; + let mut inbound_complete_opts = ObjectOptions { + eval_metadata: Some(HashMap::new()), + ..Default::default() + }; + inbound_complete_opts.set_replica_status(ReplicationStatusType::Replica); + assert!( + !inbound_complete_opts.replication_request, + "the compatibility path must not depend on a source-request marker" + ); + let inbound = set_disks + .clone() + .complete_multipart_upload(bucket, inbound_object, &upload_id, parts, &inbound_complete_opts) + .await + .expect("authorized REPLICA-only completion must preserve replica bookkeeping"); + assert_eq!( + rustfs_utils::http::get_consistent_str(inbound.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICA_STATUS), + Some("REPLICA") + ); + assert_eq!( + rustfs_utils::http::get_consistent_str( + inbound.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP + ), + Some("authorized-inbound-time") + ); + assert_eq!( + inbound + .user_defined + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)) + .map(|(_, value)| value.as_str()), + Some("REPLICA") + ); + } + #[tokio::test] #[serial] async fn complete_multipart_upload_replaces_staged_nil_version_id() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 988090a72..e7a9f23a8 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -97,6 +97,63 @@ const GET_MID_SIZE_STREAMING_MAX_SIZE: usize = 512 * 1024; const EVENT_LIFECYCLE_TRANSITION_CLEANUP: &str = "lifecycle_transition_cleanup"; const EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER: &str = "lifecycle_transitioned_delete_cleanup_owner"; +fn replication_status_writeback_is_current(object_info: &ObjectInfo, condition: &ReplicationStatusWritebackCondition) -> bool { + let current = object_info.replication_generation_snapshot(); + !condition.expected_generation.invalid && !current.invalid && current == condition.expected_generation +} + +/// Merge storage-authored evaluated metadata without leaving compatibility +/// aliases from an older writer behind. Existing objects can contain a single +/// mixed-case MinIO key; blindly inserting lowercase dual aliases would make +/// strict generation reads observe conflicting values forever. +fn merge_evaluated_metadata(metadata: &mut HashMap, evaluated: &HashMap) -> Result { + let mut internal_updates = std::collections::BTreeMap::::new(); + let mut ordinary_updates = Vec::new(); + + for (key, value) in evaluated { + let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else { + ordinary_updates.push((key, value)); + continue; + }; + let normalized_suffix = suffix.to_ascii_lowercase(); + let rustfs_preferred = rustfs_utils::http::starts_with_ignore_ascii_case(key, rustfs_utils::http::RUSTFS_INTERNAL_PREFIX); + match internal_updates.entry(normalized_suffix) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert((suffix.to_string(), value.clone(), rustfs_preferred)); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + if entry.get().1 != *value { + return Err(Error::other("conflicting evaluated internal metadata aliases")); + } + if rustfs_preferred && !entry.get().2 { + entry.insert((suffix.to_string(), value.clone(), true)); + } + } + } + } + + for (key, value) in ordinary_updates { + metadata.insert(key.clone(), value.clone()); + } + for (suffix, value, _) in internal_updates.values() { + rustfs_utils::http::remove_str(metadata, suffix); + // `remove_str` is optimized for canonical lowercase suffixes. Retain + // this exact fallback for dynamic suffixes whose identifier casing is + // significant (for example replication-reset target ARNs). + metadata.retain(|key, _| { + !rustfs_utils::http::strip_internal_prefix_preserving_case(key) + .is_some_and(|existing| existing.eq_ignore_ascii_case(suffix)) + }); + rustfs_utils::http::insert_str(metadata, suffix, value.clone()); + } + + Ok(!internal_updates.is_empty()) +} + +fn rebuild_file_info_replication_state(fi: &mut FileInfo) { + fi.replication_state_internal = rustfs_filemeta::get_internal_replication_state(&fi.metadata); +} + fn is_get_mid_size_streaming_enabled() -> bool { #[cfg(test)] { @@ -239,7 +296,10 @@ use crate::disk::new_disk; use crate::disk::{DataDirDeleteStatus, OldCurrentSize}; use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; -use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY}; +use crate::object_api::{ + NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, + SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, +}; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; @@ -442,6 +502,253 @@ fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap Result { - self.invalidate_get_object_metadata_cache(bucket, object).await; + let replication_validate_only = opts + .replication_status_writeback + .as_ref() + .is_some_and(|condition| condition.mode == ReplicationStatusWritebackMode::ValidateOnly); + if !replication_validate_only { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } // Guard lock for metadata update #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeMetadata).await; - let _lock_guard = if !opts.no_lock { + let _lock_guard = if !opts.no_lock || opts.replication_status_writeback.is_some() { Some(self.acquire_write_lock_diag("put_object_metadata", bucket, object).await?) } else { None @@ -8263,6 +8574,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) { Ok((read_quorum, write_quorum)) => (read_quorum, write_quorum), Err(mut err) => { + // A replication terminal CAS is a validation/write-back + // attempt, never a heal operation. In particular, + // ValidateOnly must remain zero-write, and a worker whose + // outer replication lease is already lost must not delete the + // surviving xl.meta copies before the namespace fence below + // can reject it. Leave dangling cleanup to heal/scanner. + if opts.replication_status_writeback.is_some() { + return Err(to_object_err(err.into(), vec![bucket, object])); + } if err == DiskError::ErasureReadQuorum && !bucket.starts_with(RUSTFS_META_BUCKET) && self @@ -8295,27 +8615,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + if opts + .replication_status_writeback + .as_ref() + .is_some_and(|condition| !replication_status_writeback_is_current(&obj_info, condition)) + { + return Err(Error::PreconditionFailed); + } + check_object_lock_retention_update(bucket, object, &obj_info, opts)?; - for (k, v) in obj_info.user_defined.iter() { - fi.metadata.insert(k.clone(), v.clone()); - } - - if let Some(mt) = &opts.eval_metadata { - for (k, v) in mt { - fi.metadata.insert(k.clone(), v.clone()); - } - } - - fi.acknowledge_data_movement(); - - if opts.mod_time.is_some() { - fi.mod_time = opts.mod_time; - } - if let Some(ref version_id) = opts.version_id { - fi.version_id = Uuid::parse_str(version_id).ok(); - } - if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id { self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; } @@ -8338,6 +8647,29 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { }); } + if replication_validate_only { + return Ok(obj_info); + } + + for (k, v) in obj_info.user_defined.iter() { + fi.metadata.insert(k.clone(), v.clone()); + } + + if let Some(mt) = &opts.eval_metadata + && merge_evaluated_metadata(&mut fi.metadata, mt)? + { + rebuild_file_info_replication_state(&mut fi); + } + + fi.acknowledge_data_movement(); + + if opts.mod_time.is_some() { + fi.mod_time = opts.mod_time; + } + if let Some(ref version_id) = opts.version_id { + fi.version_id = Uuid::parse_str(version_id).ok(); + } + self.update_object_meta(bucket, object, fi.clone(), &online_disks) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; @@ -11409,6 +11741,693 @@ mod metadata_mutation_generation_tests { ); } + fn replication_metadata(timestamp: Option<&str>, mutation_id: Option<&str>, status: &str) -> HashMap { + let mut metadata = HashMap::new(); + if let Some(timestamp) = timestamp { + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + timestamp.to_string(), + ); + } + if let Some(mutation_id) = mutation_id { + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_REPLICATION_GENERATION, + mutation_id.to_string(), + ); + } + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_STATUS, status.to_string()); + metadata + } + + async fn seed_single_mixed_case_minio_generation( + set_disks: &Arc, + bucket: &str, + object: &str, + category_suffix: &str, + category_key: &str, + ) { + let mut reader = PutObjReader::from_vec(format!("payload for {object}").into_bytes()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let (mut fi, _, disks) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("source file metadata should be readable") + .into_owned(); + for suffix in [ + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + category_suffix, + ] { + rustfs_utils::http::remove_str(&mut fi.metadata, suffix); + } + fi.metadata + .insert("X-Minio-Internal-Replication-Timestamp".to_string(), "2026-09-03T00:00:00Z".to_string()); + fi.metadata + .insert("X-Minio-Internal-Replication-Status".to_string(), "arn:test=PENDING;".to_string()); + fi.metadata + .insert(category_key.to_string(), "2026-09-03T00:00:01Z".to_string()); + rebuild_file_info_replication_state(&mut fi); + set_disks + .update_object_meta(bucket, object, fi, &disks) + .await + .expect("mixed-case legacy metadata should be persisted"); + set_disks.invalidate_get_object_metadata_cache(bucket, object).await; + + let seeded = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("mixed-case legacy object should remain readable"); + assert!(!seeded.replication_generation_snapshot().invalid); + } + + fn assert_dual_alias_value(metadata: &HashMap, suffix: &str, expected: &str) { + let aliases = metadata + .iter() + .filter(|(key, _)| rustfs_utils::http::has_internal_suffix(key, suffix)) + .collect::>(); + assert_eq!(aliases.len(), 2, "{suffix} must be rewritten as exactly two canonical aliases"); + assert!(aliases.iter().all(|(_, value)| value.as_str() == expected)); + assert!(metadata.contains_key(&format!("x-rustfs-internal-{suffix}"))); + assert!(metadata.contains_key(&format!("x-minio-internal-{suffix}"))); + } + + #[tokio::test] + async fn replication_status_writeback_cas_accepts_suspended_null_version_snapshot() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-suspended-null-version-cas"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let timestamp = "opaque-suspended-null-version-time"; + let mutation_id = Uuid::from_u128(700).to_string(); + let mut reader = PutObjReader::from_vec(b"suspended null version".to_vec()); + let queued = set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + version_suspended: true, + user_defined: replication_metadata(Some(timestamp), Some(&mutation_id), "arn:test=PENDING;"), + ..Default::default() + }, + ) + .await + .expect("suspended-bucket null version should be written"); + assert_eq!(queued.version_id, Some(Uuid::nil())); + let expected_generation = queued.replication_generation_snapshot(); + + let reread = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("the same xl.meta should be readable without versioning flags"); + assert_eq!(reread.version_id, None); + assert_eq!(reread.replication_generation_snapshot(), expected_generation); + + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + version_id: Some(Uuid::nil().to_string()), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: expected_generation.clone(), + mode: ReplicationStatusWritebackMode::ValidateOnly, + })), + ..Default::default() + }, + ) + .await + .expect("ValidateOnly must accept the same suspended null-version identity"); + + let completed = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + version_id: Some(Uuid::nil().to_string()), + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("the queued null-version snapshot must match the storage reread"); + assert_eq!(completed.replication_status_internal.as_deref(), Some("arn:test=COMPLETED;")); + } + + #[tokio::test] + async fn replication_status_writeback_cas_applies_current_rejects_stale_and_supports_legacy_absence() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-status-writeback-cas"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let object = "generation.txt"; + let generation_a = "2026-09-02T12:34:56.123456789+08:00[Asia/Shanghai]"; + let generation_b = generation_a; + let mutation_a = Uuid::from_u128(1).to_string(); + let mutation_b = Uuid::from_u128(2).to_string(); + let mut reader = PutObjReader::from_vec(b"generation-fenced".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(Some(generation_a), Some(&mutation_a), "arn:test=PENDING;")), + ..Default::default() + }, + ) + .await + .expect("generation A should be installed"); + let generation_a_snapshot = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("generation A snapshot should be readable") + .replication_generation_snapshot(); + + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: generation_a_snapshot.clone(), + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("the worker that observed generation A should publish its terminal status"); + let current = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("generation A terminal status should be readable"); + assert_eq!(current.replication_status_internal.as_deref(), Some("arn:test=COMPLETED;")); + assert_eq!( + rustfs_utils::http::get_str(current.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,) + .as_deref(), + Some(generation_a) + ); + + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(Some(generation_b), Some(&mutation_b), "arn:test=PENDING;")), + ..Default::default() + }, + ) + .await + .expect("a newer metadata mutation should install generation B"); + let generation_b_snapshot = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("generation B snapshot should be readable") + .replication_generation_snapshot(); + let stale_error = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: generation_a_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect_err("generation A must not overwrite generation B's PENDING status"); + assert!(matches!(stale_error, Error::PreconditionFailed)); + let after_stale = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("generation B should remain readable"); + assert_eq!(after_stale.replication_status_internal.as_deref(), Some("arn:test=PENDING;")); + assert_eq!( + rustfs_utils::http::get_str(after_stale.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,) + .as_deref(), + Some(generation_b) + ); + + set_disks + .put_object_tags( + bucket, + object, + "stage=legacy-writer", + &ObjectOptions { + eval_metadata: Some(replication_metadata(Some(generation_b), None, "arn:test=PENDING;")), + ..Default::default() + }, + ) + .await + .expect("a legacy metadata writer should preserve the unknown mutation id"); + let legacy_writer_error = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: generation_b_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect_err("a preserved UUID and timestamp must not hide a legacy writer's payload change"); + assert!(matches!(legacy_writer_error, Error::PreconditionFailed)); + + let legacy_object = "legacy-without-generation.txt"; + let mut legacy_reader = PutObjReader::from_vec(b"legacy".to_vec()); + set_disks + .put_object(bucket, legacy_object, &mut legacy_reader, &ObjectOptions::default()) + .await + .expect("legacy source object should be written"); + let legacy_snapshot = set_disks + .get_object_info(bucket, legacy_object, &ObjectOptions::default()) + .await + .expect("legacy snapshot should be readable") + .replication_generation_snapshot(); + set_disks + .put_object_metadata( + bucket, + legacy_object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: legacy_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("legacy absence must compare as an exact generation value"); + let legacy = set_disks + .get_object_info(bucket, legacy_object, &ObjectOptions::default()) + .await + .expect("legacy terminal status should be readable"); + assert_eq!(legacy.replication_status_internal.as_deref(), Some("arn:test=COMPLETED;")); + assert!( + rustfs_utils::http::get_str(legacy.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,) + .is_none() + ); + + let timestamp_only_object = "legacy-timestamp-only.txt"; + let mut timestamp_only_reader = PutObjReader::from_vec(b"timestamp only".to_vec()); + set_disks + .put_object(bucket, timestamp_only_object, &mut timestamp_only_reader, &ObjectOptions::default()) + .await + .expect("timestamp-only source object should be written"); + set_disks + .put_object_metadata( + bucket, + timestamp_only_object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(Some(generation_a), None, "arn:test=PENDING;")), + ..Default::default() + }, + ) + .await + .expect("legacy timestamp-only generation should be installed"); + let timestamp_only_snapshot = set_disks + .get_object_info(bucket, timestamp_only_object, &ObjectOptions::default()) + .await + .expect("legacy timestamp-only snapshot should be readable") + .replication_generation_snapshot(); + assert_eq!(timestamp_only_snapshot.timestamp.as_deref(), Some(generation_a)); + assert_eq!(timestamp_only_snapshot.mutation_id, None); + set_disks + .put_object_metadata( + bucket, + timestamp_only_object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: timestamp_only_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("legacy timestamp-only generation should remain writable through exact CAS"); + } + + #[tokio::test] + async fn metadata_mutations_canonicalize_mixed_case_minio_generation_aliases() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-mixed-case-generation"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let tag_object = "tagging.txt"; + seed_single_mixed_case_minio_generation( + &set_disks, + bucket, + tag_object, + rustfs_utils::http::SUFFIX_TAGGING_TIMESTAMP, + "X-Minio-Internal-Tagging-Timestamp", + ) + .await; + let tag_timestamp = "2026-09-03T01:00:00Z"; + let tag_mutation = Uuid::from_u128(101).to_string(); + let mut tag_eval = replication_metadata(Some(tag_timestamp), Some(&tag_mutation), "arn:test=PENDING;"); + rustfs_utils::http::insert_str( + &mut tag_eval, + rustfs_utils::http::SUFFIX_TAGGING_TIMESTAMP, + "2026-09-03T01:00:01Z".to_string(), + ); + let tagged = set_disks + .put_object_tags( + bucket, + tag_object, + "stage=canonical", + &ObjectOptions { + eval_metadata: Some(tag_eval), + ..Default::default() + }, + ) + .await + .expect("tag mutation should canonicalize legacy aliases"); + assert_dual_alias_value( + tagged.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + tag_timestamp, + ); + assert_dual_alias_value( + tagged.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn:test=PENDING;", + ); + assert_dual_alias_value( + tagged.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_TAGGING_TIMESTAMP, + "2026-09-03T01:00:01Z", + ); + let tag_snapshot = tagged.replication_generation_snapshot(); + assert!(!tag_snapshot.invalid); + let persisted_tag_snapshot = set_disks + .get_object_info(bucket, tag_object, &ObjectOptions::default()) + .await + .expect("canonicalized tag metadata should be reread") + .replication_generation_snapshot(); + assert_eq!(tag_snapshot, persisted_tag_snapshot); + set_disks + .put_object_metadata( + bucket, + tag_object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: tag_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("the canonicalized tag generation should accept terminal CAS"); + + let metadata_object = "retention-metadata.txt"; + seed_single_mixed_case_minio_generation( + &set_disks, + bucket, + metadata_object, + rustfs_utils::http::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "X-Minio-Internal-Objectlock-Retention-Timestamp", + ) + .await; + let metadata_timestamp = "2026-09-03T02:00:00Z"; + let metadata_mutation = Uuid::from_u128(102).to_string(); + let mut metadata_eval = replication_metadata(Some(metadata_timestamp), Some(&metadata_mutation), "arn:test=PENDING;"); + rustfs_utils::http::insert_str( + &mut metadata_eval, + rustfs_utils::http::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "2026-09-03T02:00:01Z".to_string(), + ); + metadata_eval.insert("x-amz-meta-proof".to_string(), "updated".to_string()); + let updated = set_disks + .put_object_metadata( + bucket, + metadata_object, + &ObjectOptions { + eval_metadata: Some(metadata_eval), + ..Default::default() + }, + ) + .await + .expect("generic metadata mutation should canonicalize legacy aliases"); + assert_dual_alias_value( + updated.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + metadata_timestamp, + ); + assert_dual_alias_value( + updated.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn:test=PENDING;", + ); + assert_dual_alias_value( + updated.user_defined.as_ref(), + rustfs_utils::http::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "2026-09-03T02:00:01Z", + ); + let metadata_snapshot = updated.replication_generation_snapshot(); + assert!(!metadata_snapshot.invalid); + let failed = set_disks + .put_object_metadata( + bucket, + metadata_object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=FAILED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: metadata_snapshot, + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .expect("the canonicalized generic metadata generation should accept terminal CAS"); + assert_eq!(failed.replication_status_internal.as_deref(), Some("arn:test=FAILED;")); + } + + #[tokio::test] + #[serial_test::serial(metadata_cache_invalidation_probe)] + async fn replication_status_writeback_validate_only_is_zero_write_and_checks_fences() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-status-validate-only"; + let object = "failed-retry.txt"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(b"failed retry".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let timestamp = "2026-09-02T12:34:56.123456789+08:00[Asia/Shanghai]"; + let mutation_id = Uuid::from_u128(10).to_string(); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(Some(timestamp), Some(&mutation_id), "arn:test=FAILED;")), + ..Default::default() + }, + ) + .await + .expect("failed generation should be installed"); + let snapshot = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("failed generation should be readable") + .replication_generation_snapshot(); + + let invalidations = MetadataCacheInvalidationProbe::install(bucket, object); + let validated = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + no_lock: true, + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: snapshot.clone(), + mode: ReplicationStatusWritebackMode::ValidateOnly, + })), + ..Default::default() + }, + ) + .await + .expect("the current FAILED generation should validate without a rewrite"); + assert_eq!(validated.replication_status_internal.as_deref(), Some("arn:test=FAILED;")); + assert_eq!( + invalidations.count(), + 0, + "validate-only must not invalidate metadata cache or enter a write path" + ); + + let mut stale = snapshot.clone(); + stale.mutation_id = Some(Uuid::from_u128(11).to_string()); + let stale_error = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(replication_metadata(None, None, "arn:test=COMPLETED;")), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: stale, + mode: ReplicationStatusWritebackMode::ValidateOnly, + })), + ..Default::default() + }, + ) + .await + .expect_err("a stale FAILED retry must be superseded during validate-only"); + assert!(matches!(stale_error, Error::PreconditionFailed)); + assert_eq!(invalidations.count(), 0, "a rejected validate-only CAS must not invalidate cache"); + + let lost_fence_error = set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: snapshot, + mode: ReplicationStatusWritebackMode::ValidateOnly, + })), + ..Default::default() + }, + ) + .await + .expect_err("validate-only must not report success after its namespace fence is lost"); + assert!(matches!(lost_fence_error, StorageError::NamespaceLockQuorumUnavailable { .. })); + assert_eq!(invalidations.count(), 0, "a fenced validate-only path must remain side-effect free"); + } + + #[tokio::test] + #[serial_test::serial] + async fn replication_terminal_cas_quorum_failure_never_deletes_surviving_metadata() { + temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-terminal-quorum-failure"; + let object = "survivor.txt"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(b"surviving metadata".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let expected_generation = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("source generation should be readable") + .replication_generation_snapshot(); + + let meta_paths = temp_dirs + .iter() + .map(|temp_dir| temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .collect::>(); + let surviving_before = tokio::fs::read(&meta_paths[3]) + .await + .expect("the surviving xl.meta should exist before the terminal attempt"); + for path in &meta_paths[..3] { + tokio::fs::remove_file(path) + .await + .expect("three xl.meta copies should be removed to force read-quorum failure"); + } + + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation: expected_generation.clone(), + mode: ReplicationStatusWritebackMode::ValidateOnly, + })), + ..Default::default() + }, + ) + .await + .expect_err("terminal CAS must retry when current metadata lacks read quorum"); + + let surviving_after_validate = tokio::fs::read(&meta_paths[3]) + .await + .expect("terminal CAS must not delete the final xl.meta copy"); + assert_eq!( + surviving_after_validate, surviving_before, + "ValidateOnly must remain zero-write on quorum failure" + ); + + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition { + expected_generation, + mode: ReplicationStatusWritebackMode::Update, + })), + ..Default::default() + }, + ) + .await + .expect_err("a lost-fence terminal update must retry when current metadata lacks read quorum"); + + let surviving_after_lost_fence = tokio::fs::read(&meta_paths[3]) + .await + .expect("a lost-fence terminal update must not delete the final xl.meta copy"); + assert_eq!( + surviving_after_lost_fence, surviving_before, + "a lost outer fence must remain zero-write on quorum failure" + ); + }) + .await; + } + async fn persist_part_checksum_sidecar(set_disks: &Arc, bucket: &str, object: &str, value: &str) { let (fi, _, disks) = set_disks .get_object_fileinfo( diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 5ce22b3c7..0e42b1850 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -363,6 +363,25 @@ impl FileMeta { } } + // `fi.metadata` is the authoritative replacement + // for every internal suffix it carries. Remove all + // existing RustFS/MinIO and mixed-case aliases for + // those suffixes before inserting the new map; + // otherwise an old MinIO key survives this RMW and + // conflicts with newly written canonical aliases. + let replaced_internal_suffixes = fi + .metadata + .keys() + .filter_map(|key| rustfs_utils::http::strip_internal_prefix_preserving_case(key)) + .map(str::to_ascii_lowercase) + .collect::>(); + if !replaced_internal_suffixes.is_empty() { + obj.meta_sys.retain(|key, _| { + rustfs_utils::http::strip_internal_prefix_preserving_case(key) + .is_none_or(|suffix| !replaced_internal_suffixes.contains(&suffix.to_ascii_lowercase())) + }); + } + for (k, v) in fi.metadata.iter() { // Split metadata into meta_user and meta_sys based on prefix // This logic must match From for MetaObject diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 3a8d69da9..539456e5e 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -2861,7 +2861,12 @@ impl From for MetaObject { } } -fn get_internal_replication_state(metadata: &HashMap) -> Option { +/// Rebuild the structured replication state from its durable internal metadata. +/// +/// Mutation paths that update internal replication keys on an existing +/// [`FileInfo`] must use this parser before serializing xl.meta so the metadata +/// map and the structured state cannot diverge. +pub fn get_internal_replication_state(metadata: &HashMap) -> Option { let mut rs = ReplicationState::default(); let mut has = false; diff --git a/crates/replication/src/filemeta.rs b/crates/replication/src/filemeta.rs index 8dfdb26c5..7af1b3141 100644 --- a/crates/replication/src/filemeta.rs +++ b/crates/replication/src/filemeta.rs @@ -849,6 +849,20 @@ pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result, + pub mutation_id: Option, + pub payload_fingerprint: Option<[u8; 32]>, + pub invalid: bool, +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ReplicateObjectInfo { pub name: String, @@ -871,6 +885,10 @@ pub struct ReplicateObjectInfo { pub target_statuses: HashMap, pub target_purge_statuses: HashMap, pub replication_timestamp: Option, + /// Exact persisted source snapshot used to fence terminal status + /// write-back after remote I/O. + #[serde(default)] + pub replication_generation: ReplicationGenerationSnapshot, pub ssec: bool, pub user_tags: String, pub checksum: Option, diff --git a/crates/replication/src/http.rs b/crates/replication/src/http.rs index 97a45410d..e99e9fab7 100644 --- a/crates/replication/src/http.rs +++ b/crates/replication/src/http.rs @@ -85,6 +85,24 @@ pub(crate) fn get_internal_metadata(map: &HashMap, suffix: &str) }) } +/// Return a non-empty internal value only when every RustFS/MinIO alias +/// present for the suffix agrees. A single alias remains valid for rolling +/// upgrade compatibility; conflicts fail closed at security boundaries. +pub(crate) fn get_consistent_internal_metadata<'a>(map: &'a HashMap, suffix: &str) -> Option<&'a str> { + let (rustfs_key, minio_key) = internal_keys(suffix); + let mut value: Option<&String> = None; + for (key, candidate) in map { + if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) { + continue; + } + if candidate.is_empty() || value.is_some_and(|current| current != candidate) { + return None; + } + value = Some(candidate); + } + value.map(String::as_str) +} + pub(crate) fn get_header_metadata(map: &HashMap, suffix: &str) -> Option { let rustfs_key = rustfs_header_key(suffix); let minio_key = minio_header_key(suffix); @@ -114,8 +132,8 @@ pub(crate) fn insert_internal_metadata(map: &mut HashMap, suffix #[cfg(test)] mod tests { use super::{ - SUFFIX_ACTUAL_SIZE, SUFFIX_REPLICATION_RESET_STATUS, get_header_metadata, get_internal_metadata, has_prefix_fold, - insert_internal_metadata, internal_key_rustfs, trim_etag, + SUFFIX_ACTUAL_SIZE, SUFFIX_REPLICATION_RESET_STATUS, get_consistent_internal_metadata, get_header_metadata, + get_internal_metadata, has_prefix_fold, insert_internal_metadata, internal_key_rustfs, trim_etag, }; use std::collections::HashMap; @@ -136,6 +154,38 @@ mod tests { assert_eq!(get_internal_metadata(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("12")); } + #[test] + fn consistent_internal_metadata_accepts_single_or_matching_aliases_and_rejects_conflicts() { + for key in ["x-rustfs-internal-replication-status", "X-Minio-Internal-Replication-Status"] { + let metadata = HashMap::from([(key.to_string(), "arn:target=PENDING;".to_string())]); + assert_eq!( + get_consistent_internal_metadata(&metadata, "replication-status"), + Some("arn:target=PENDING;") + ); + } + + let matching = HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), "arn:target=COMPLETED;".to_string()), + ("X-Minio-Internal-Replication-Status".to_string(), "arn:target=COMPLETED;".to_string()), + ]); + assert_eq!( + get_consistent_internal_metadata(&matching, "replication-status"), + Some("arn:target=COMPLETED;") + ); + + let conflicting = HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), "arn:target=PENDING;".to_string()), + ("x-minio-internal-replication-status".to_string(), "arn:target=REPLICA;".to_string()), + ]); + assert!(get_consistent_internal_metadata(&conflicting, "replication-status").is_none()); + + let empty_alias = HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), "arn:target=PENDING;".to_string()), + ("x-minio-internal-replication-status".to_string(), String::new()), + ]); + assert!(get_consistent_internal_metadata(&empty_alias, "replication-status").is_none()); + } + #[test] fn internal_metadata_insert_writes_rustfs_and_minio_keys() { let mut metadata = HashMap::new(); diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index f000a3bf5..a8b9679de 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -48,10 +48,10 @@ pub use delete::{ pub use filemeta::{ NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING, REPLICATE_INCOMING_DELETE, REPLICATE_MRF, REPLICATE_QUEUED, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, - ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, - ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, - VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replicate_decision_for_admitted_targets, - replication_statuses_map, target_reset_header, version_purge_statuses_map, + ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, + ReplicationGenerationSnapshot, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, + ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType, get_replication_state, parse_replicate_decision, + replicate_decision_for_admitted_targets, replication_statuses_map, target_reset_header, version_purge_statuses_map, }; pub use mrf::{ MRF_ENVELOPE_FORMAT, MRF_ENVELOPE_VERSION, MRF_V2_FILE, MRF_V2_FORMAT, MRF_V2_NAMESPACE, MRF_V2_VERSION, MrfCapabilities, diff --git a/crates/replication/src/operation.rs b/crates/replication/src/operation.rs index ce566d9f8..2c8a29f4e 100644 --- a/crates/replication/src/operation.rs +++ b/crates/replication/src/operation.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use crate::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, - SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_STATUS, get_header_metadata, get_internal_metadata, + SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_STATUS, get_consistent_internal_metadata, get_header_metadata, }; use s3s::dto::ReplicationConfiguration; use time::OffsetDateTime; @@ -99,15 +99,34 @@ impl MustReplicateOptions { return true; } - get_internal_metadata(&self.meta, SUFFIX_REPLICATION_STATUS) - .as_deref() - .and_then(|statuses| { - statuses.split(';').find_map(|entry| { - let (target_arn, status) = entry.split_once('=')?; - (target_arn == arn).then(|| ReplicationStatusType::from(status)) - }) - }) - == Some(ReplicationStatusType::Completed) + let Some(statuses) = get_consistent_internal_metadata(&self.meta, SUFFIX_REPLICATION_STATUS) else { + return false; + }; + let mut admitted = None; + for entry in statuses.split(';').filter(|entry| !entry.is_empty()) { + let Some((target_arn, status)) = entry.split_once('=') else { + continue; + }; + if target_arn != arn { + continue; + } + // Per-target admission is authoritative only when the target has + // exactly one unambiguous historical status record. + if admitted.is_some() { + return false; + } + admitted = Some(ReplicationStatusType::from(status)); + } + + matches!( + admitted, + Some( + ReplicationStatusType::Pending + | ReplicationStatusType::Failed + | ReplicationStatusType::Completed + | ReplicationStatusType::CompletedLegacy + ) + ) } } @@ -376,22 +395,72 @@ mod tests { } #[test] - fn metadata_replication_requires_completed_target_state() { + fn metadata_replication_requires_prior_target_admission() { let arn = "arn:rustfs:replication:target"; - for status in ["PENDING", "FAILED"] { + for status in ["PENDING", "FAILED", "COMPLETED", "COMPLETE"] { let mut meta = HashMap::new(); insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}={status};")); let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false); - assert!(!options.metadata_target_is_eligible(arn)); + assert!( + options.metadata_target_is_eligible(arn), + "known target state {status} must remain eligible" + ); + assert!(!options.metadata_target_is_eligible("arn:rustfs:replication:missing")); } - let mut meta = HashMap::new(); - insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}=COMPLETED;")); - let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false); + for status in ["", "REPLICA", "UNKNOWN"] { + let mut meta = HashMap::new(); + insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}={status};")); + let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false); + assert!( + !options.metadata_target_is_eligible(arn), + "unsupported target state {status:?} must fail closed" + ); + } - assert!(options.metadata_target_is_eligible(arn)); - assert!(!options.metadata_target_is_eligible("arn:rustfs:replication:missing")); + for key in ["x-rustfs-internal-replication-status", "X-Minio-Internal-Replication-Status"] { + let meta = HashMap::from([(key.to_string(), format!("{arn}=PENDING;"))]); + assert!( + MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false) + .metadata_target_is_eligible(arn), + "a single compatibility alias must preserve rolling-upgrade admission" + ); + } + + let matching_aliases = HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), format!("{arn}=COMPLETED;")), + ("X-Minio-Internal-Replication-Status".to_string(), format!("{arn}=COMPLETED;")), + ]); + assert!( + MustReplicateOptions::new(&matching_aliases, String::new(), ReplicationType::Metadata, false) + .metadata_target_is_eligible(arn) + ); + + for conflicting in [ + HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), format!("{arn}=PENDING;")), + ("x-minio-internal-replication-status".to_string(), format!("{arn}=REPLICA;")), + ]), + HashMap::from([ + ("x-rustfs-internal-replication-status".to_string(), format!("{arn}=PENDING;")), + ("x-minio-internal-replication-status".to_string(), String::new()), + ]), + HashMap::from([( + "x-rustfs-internal-replication-status".to_string(), + format!("{arn}=PENDING;{arn}=REPLICA;"), + )]), + HashMap::from([( + "x-rustfs-internal-replication-status".to_string(), + format!("{arn}=PENDING;{arn}=PENDING;"), + )]), + ] { + assert!( + !MustReplicateOptions::new(&conflicting, String::new(), ReplicationType::Metadata, false) + .metadata_target_is_eligible(arn), + "conflicting, empty, or duplicate admission evidence must fail closed" + ); + } // A replica-side metadata edit (active-active) has no per-target // internal status; eligibility is left to the ReplicaModifications rule. diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index f4b2fe429..dbe41e785 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -75,6 +75,8 @@ pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; pub const SUFFIX_REPLICA_TIMESTAMP: &str = "replica-timestamp"; pub const SUFFIX_REPLICATION_STATUS: &str = "replication-status"; pub const SUFFIX_REPLICATION_TIMESTAMP: &str = "replication-timestamp"; +/// Source-local opaque mutation id fencing replication status write-back. +pub const SUFFIX_REPLICATION_GENERATION: &str = "replication-generation"; pub const SUFFIX_TAGGING_TIMESTAMP: &str = "tagging-timestamp"; pub const SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP: &str = "objectlock-retention-timestamp"; pub const SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP: &str = "objectlock-legalhold-timestamp"; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 820fcfa58..fc81b3530 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -99,11 +99,11 @@ use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; #[cfg(test)] -use rustfs_utils::http::insert_header; +use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, insert_header}; use rustfs_utils::http::{ - SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, - SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header, - get_source_scheme, + SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header, get_source_scheme, headers::{AMZ_CHECKSUM_TYPE, AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS}, insert_str, }; @@ -733,6 +733,43 @@ impl DefaultMultipartUsecase { None => None, }; + // Replication admission is decided at the object-creation commit, not + // at multipart-session creation. Configuration and matching targets + // can change while parts are uploaded, so persist the exact decision's + // generation and PENDING set atomically with the completed object and + // reuse the same decision for scheduling below. + let completion_replication_decision = must_replicate_object( + &bucket, + &key, + &multipart_info.user_defined, + "".to_string(), + opts.delete_marker_replication_status(), + opts.clone(), + ) + .await; + let mut completion_replication_metadata = HashMap::new(); + if completion_replication_decision.replicate_any() { + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_GENERATION, + Uuid::new_v4().to_string(), + ); + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_TIMESTAMP, + jiff::Zoned::now().to_string(), + ); + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_STATUS, + completion_replication_decision.pending_status().unwrap_or_default(), + ); + } + // `Some(empty)` deliberately means that completion re-evaluated the + // session as not admitted; storage removes stale Create-MPU admission + // metadata in the same final-object commit. + opts.eval_metadata = Some(completion_replication_metadata); + let complete_commit = spawn_traced_join({ let store = Arc::clone(&store); let bucket = bucket.clone(); @@ -760,20 +797,9 @@ impl DefaultMultipartUsecase { enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await; - let mt2 = obj_info.user_defined.clone(); - let dsc = must_replicate_object( - &bucket, - &key, - &mt2, - "".to_string(), - opts.delete_marker_replication_status(), - opts.clone(), - ) - .await; - - if dsc.replicate_any() { + if completion_replication_decision.replicate_any() { warn!("need multipart replication"); - schedule_object_replication(obj_info.clone(), store, dsc).await; + schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await; } rustfs_scanner::record_dirty_usage_bucket(&bucket); @@ -1037,6 +1063,7 @@ impl DefaultMultipartUsecase { must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone()) .await; if dsc.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str( &mut opts.user_defined, @@ -2499,6 +2526,87 @@ mod tests { } } + #[test] + #[serial_test::serial] + fn authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state() { + crate::app::gating_test_env::run_large_stack_test( + "authorized-replica-only-multipart-complete", + authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner, + ); + } + + async fn authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("replica-only-complete", 16_384).await; + let object = "object"; + let usecase = DefaultMultipartUsecase::from_global(); + + let create_input = CreateMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .build() + .expect("create multipart input should build"); + let mut create_request = build_request(create_input, Method::POST); + create_request + .headers + .insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA")); + create_request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + let upload_id = usecase + .execute_create_multipart_upload(create_request) + .await + .expect("authorized REPLICA-only create should succeed") + .output + .upload_id + .expect("create response should contain upload id"); + + let mut reader = PutObjReader::from_vec(b"authorized replica-only multipart body".to_vec()); + let part = store + .put_object_part(&bucket, object, &upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("replica part should be staged"); + let complete_input = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .upload_id(upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.etag.map(|etag| to_s3s_etag(&etag)), + ..Default::default() + }]), + })) + .build() + .expect("complete multipart input should build"); + let mut complete_request = build_request(complete_input, Method::POST); + complete_request + .headers + .insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA")); + complete_request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + usecase + .execute_complete_multipart_upload(complete_request) + .await + .expect("authorized REPLICA-only completion should succeed without the source-request marker"); + + let persisted = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("completed replica should be readable from storage"); + assert_eq!( + persisted + .user_defined + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)) + .map(|(_, value)| value.as_str()), + Some("REPLICA"), + "the committed xl.meta must retain REPLICA to prevent scanner cascades" + ); + } + #[tokio::test] #[serial_test::serial] async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() { diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index 097231148..fb9496c13 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -721,11 +721,7 @@ impl DefaultObjectUsecase { // exempt: the authorized replication request owns these keys (see // copy_dst_opts_with_replication_authorization above). if !dst_opts.replication_request { - user_defined.retain(|k, _| !k.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)); - remove_str(&mut user_defined, SUFFIX_REPLICATION_STATUS); - remove_str(&mut user_defined, SUFFIX_REPLICATION_TIMESTAMP); - remove_str(&mut user_defined, SUFFIX_REPLICA_STATUS); - remove_str(&mut user_defined, SUFFIX_REPLICA_TIMESTAMP); + remove_source_replication_bookkeeping(&mut user_defined); } // Compute the replication decision exactly once per copy. The same @@ -746,6 +742,7 @@ impl DefaultObjectUsecase { ) .await; if dsc.replicate_any() { + insert_str(&mut user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut user_defined, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); } @@ -914,6 +911,37 @@ mod tests { use std::sync::Arc; use tokio::io::AsyncReadExt; + #[test] + fn local_copy_does_not_inherit_source_replication_generation() { + let mut metadata = HashMap::from([ + (AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string()), + ("x-amz-meta-owner".to_string(), "source".to_string()), + ]); + for suffix in [ + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_STATUS, + SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_REPLICA_STATUS, + SUFFIX_REPLICA_TIMESTAMP, + ] { + insert_str(&mut metadata, suffix, format!("source-{suffix}")); + } + + remove_source_replication_bookkeeping(&mut metadata); + + assert_eq!(metadata.get("x-amz-meta-owner").map(String::as_str), Some("source")); + assert!(!metadata.contains_key(AMZ_BUCKET_REPLICATION_STATUS)); + for suffix in [ + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_STATUS, + SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_REPLICA_STATUS, + SUFFIX_REPLICA_TIMESTAMP, + ] { + assert!(!rustfs_utils::http::contains_key_str(&metadata, suffix)); + } + } + // A malformed bucket-default algorithm reaches this resolution only through // corrupt or hand-edited bucket metadata (PutBucketEncryption validates the // value), so the invariant is pinned here rather than end-to-end: the copy diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index 43729cbd5..e5b7fb4da 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -2535,6 +2535,7 @@ impl DefaultObjectUsecase { ) .await; if replication.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str( &mut opts.user_defined, diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs index cfeca6567..4c54ddc3f 100644 --- a/rustfs/src/app/object/internal_put.rs +++ b/rustfs/src/app/object/internal_put.rs @@ -44,7 +44,8 @@ use http::HeaderName; /// `Content-Language`, `Expires`), `user_metadata` carries `x-amz-meta-*` /// entries with the prefix stripped, `tags` is the `x-amz-tagging` query /// string and `internal_metadata` holds `x-rustfs-internal-*` / -/// `x-minio-internal-*` keys written verbatim. +/// `x-minio-internal-*` keys. Source replication bookkeeping is stripped; +/// other internal provenance is written verbatim. pub(crate) struct InternalPutContext { pub(crate) bucket: String, pub(crate) key: String, @@ -242,7 +243,7 @@ impl DefaultObjectUsecase { content_headers, user_metadata, tags, - internal_metadata, + mut internal_metadata, emit_events, principal_id, } = ctx; @@ -253,6 +254,7 @@ impl DefaultObjectUsecase { let headers = internal_put_headers(&content_headers)?; validate_internal_write_target(&key, &bucket, &headers).await?; + remove_source_replication_bookkeeping(&mut internal_metadata); let write = PutObjectWriteRequest { bucket, @@ -349,6 +351,7 @@ impl DefaultObjectUsecase { ); } metadata.extend(ctx.internal_metadata.clone()); + remove_source_replication_bookkeeping(&mut metadata); let mt2 = metadata.clone(); let mut opts = put_opts_with_replication_authorization(&ctx.bucket, &ctx.key, None, &headers, metadata, false) @@ -365,6 +368,7 @@ impl DefaultObjectUsecase { ) .await; if dsc.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str( &mut opts.user_defined, @@ -533,6 +537,11 @@ impl DefaultObjectUsecase { let capacity_scope_token = Uuid::new_v4(); opts.capacity_scope_token = Some(capacity_scope_token); + let multipart_info = store + .get_multipart_info(&bucket, &key, upload_id, &opts) + .await + .map_err(ApiError::from)?; + let current_opts = internal_object_info_lookup_opts(get_opts(&bucket, &key, None, None, &headers).await.map_err(ApiError::from)?); let object_lock_config_state = load_bucket_object_lock_config_state(&bucket) @@ -576,6 +585,44 @@ impl DefaultObjectUsecase { None => None, }; + // Internal multipart writes use the same object-creation admission + // contract as the S3 completion path. Re-evaluate against the staged + // metadata immediately before commit, persist the exact generation and + // PENDING target set atomically with the object, and reuse that same + // immutable decision for scheduling below. + let mut completion_source_metadata = multipart_info.user_defined.clone(); + remove_source_replication_bookkeeping(&mut completion_source_metadata); + let completion_replication_decision = must_replicate_object( + &bucket, + &key, + &completion_source_metadata, + "".to_string(), + opts.delete_marker_replication_status(), + opts.clone(), + ) + .await; + let mut completion_replication_metadata = HashMap::new(); + if completion_replication_decision.replicate_any() { + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_GENERATION, + Uuid::new_v4().to_string(), + ); + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_TIMESTAMP, + jiff::Zoned::now().to_string(), + ); + insert_str( + &mut completion_replication_metadata, + SUFFIX_REPLICATION_STATUS, + completion_replication_decision.pending_status().unwrap_or_default(), + ); + } + // `Some(empty)` clears a stale create-time admission when replication + // was disabled or no rule matches at completion. + opts.eval_metadata = Some(completion_replication_metadata); + let event = ctx.emit_events.then(|| { InternalPutObjectEvent::new( current_notify_interface_for_context(self.context.as_deref()), @@ -615,18 +662,8 @@ impl DefaultObjectUsecase { enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await; - let mt2 = obj_info.user_defined.clone(); - let dsc = must_replicate_object( - &bucket, - &key, - &mt2, - "".to_string(), - opts.delete_marker_replication_status(), - opts.clone(), - ) - .await; - if dsc.replicate_any() { - schedule_object_replication(obj_info.clone(), store, dsc).await; + if completion_replication_decision.replicate_any() { + schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await; } rustfs_scanner::record_dirty_usage_bucket(&bucket); @@ -663,10 +700,16 @@ impl DefaultObjectUsecase { #[cfg(test)] mod tests { use super::*; + use crate::app::storage_api::s3::{ + BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration, + ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, Tag, VersioningConfiguration, + }; use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata}; use rustfs_utils::http::{ MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_ODM_PULLED_AT, SUFFIX_ODM_SOURCE, SUFFIX_ODM_SOURCE_ETAG, - SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_str, + SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_consistent_str, get_str, + has_internal_suffix, }; const TEST_PRINCIPAL: &str = "rustfs-internal-put-test"; @@ -727,6 +770,70 @@ mod tests { (store, bucket) } + async fn install_internal_replication_config(bucket: &str, target: Option<&str>) { + install_internal_replication_config_with_tag(bucket, target, None).await; + } + + async fn install_internal_replication_config_with_tag( + bucket: &str, + target: Option<&str>, + required_tag: Option<(&str, &str)>, + ) { + use crate::app::storage_api::test::bucket::utils::serialize; + + let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized"); + let metadata = { + let sys = sys.read().await; + sys.get(bucket) + .await + .expect("bucket metadata should be cached before replication config injection") + }; + let mut metadata = (*metadata).clone(); + metadata.versioning_config_xml = b"Enabled".to_vec(); + metadata.versioning_config = Some(VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + }); + + if let Some(target) = target { + let filter = required_tag.map(|(key, value)| ReplicationRuleFilter { + tag: Some(Tag { + key: Some(key.to_string()), + value: Some(value.to_string()), + }), + ..Default::default() + }); + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![ReplicationRule { + delete_marker_replication: Some(DeleteMarkerReplication { + status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)), + }), + delete_replication: None, + destination: Destination { + bucket: target.to_string(), + ..Default::default() + }, + existing_object_replication: None, + filter, + id: Some("internal-multipart".to_string()), + prefix: Some(String::new()), + priority: Some(1), + source_selection_criteria: None, + status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED), + }], + }; + metadata.replication_config_xml = serialize(&config).expect("replication test config should serialize"); + metadata.replication_config = Some(config); + } else { + metadata.replication_config_xml.clear(); + metadata.replication_config = None; + } + set_bucket_metadata(bucket.to_string(), metadata) + .await + .expect("replication test metadata should be installed"); + } + #[tokio::test] #[serial_test::serial] async fn internal_put_object_writes_through_the_shared_put_path() { @@ -770,6 +877,120 @@ mod tests { assert_eq!(get_str(metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:source-bucket")); } + #[tokio::test] + #[serial_test::serial] + async fn internal_put_drops_foreign_admission_before_later_matching_metadata() { + let (store, bucket) = internal_put_test_bucket("internal-put-foreign-admission").await; + let target = "arn:aws:s3:::internal-tag-filter-target"; + install_internal_replication_config_with_tag(&bucket, Some(target), Some(("replicate", "yes"))).await; + + let body = b"initially unadmitted internal object".to_vec(); + let mut ctx = internal_context(&bucket, "foreign-admission.txt", &body); + ctx.tags = Some("replicate=no".to_string()); + ctx.internal_metadata + .insert("X-Minio-Internal-Replication-Status".to_string(), format!("{target}=COMPLETED;")); + ctx.internal_metadata + .insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string()); + + DefaultObjectUsecase::from_global() + .internal_put_object(ctx, body_stream(vec![Bytes::from(body)])) + .await + .expect("internal put with foreign bookkeeping should succeed after sanitization"); + let stored = store + .get_object_info(&bucket, "foreign-admission.txt", &ObjectOptions::default()) + .await + .expect("sanitized internal object should be readable"); + assert!(!contains_key_str(&stored.user_defined, SUFFIX_REPLICATION_STATUS)); + assert!( + stored + .user_defined + .keys() + .all(|key| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)), + "an unadmitted object must not retain a caller-supplied surfaced status" + ); + + let later_matching = crate::app::storage_api::object_usecase::bucket::replication::must_replicate_metadata( + &bucket, + "foreign-admission.txt", + &stored.user_defined, + "replicate=yes".to_string(), + stored.replication_status.clone(), + ObjectOptions::default(), + ) + .await; + assert!( + !later_matching.replicate_any(), + "a caller-supplied source status must not forge historical admission for a later matching tag" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn internal_put_replaces_mixed_case_foreign_generation_with_local_canonical_aliases() { + let (store, bucket) = internal_put_test_bucket("internal-put-foreign-gen").await; + let target = "arn:aws:s3:::internal-generation-target"; + install_internal_replication_config(&bucket, Some(target)).await; + + let body = b"replication-admitted internal object".to_vec(); + let mut ctx = internal_context(&bucket, "foreign-generation.txt", &body); + let foreign_generation = Uuid::from_u128(9001).to_string(); + for (key, value) in [ + ("X-Minio-Internal-Replication-Generation", foreign_generation.as_str()), + ("X-Minio-Internal-Replication-Timestamp", "foreign-replication-time"), + ("X-Minio-Internal-Replication-Status", "arn:foreign=COMPLETED;"), + ("X-Minio-Internal-Replica-Status", "REPLICA"), + ("X-Minio-Internal-Replica-Timestamp", "foreign-replica-time"), + ] { + ctx.internal_metadata.insert(key.to_string(), value.to_string()); + } + ctx.internal_metadata + .insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string()); + + DefaultObjectUsecase::from_global() + .internal_put_object(ctx, body_stream(vec![Bytes::from(body)])) + .await + .expect("replication-admitted internal put should sanitize foreign bookkeeping"); + let stored = store + .get_object_info(&bucket, "foreign-generation.txt", &ObjectOptions::default()) + .await + .expect("replication-admitted internal object should be readable"); + + let local_generation = + get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_GENERATION).expect("local generation aliases must agree"); + Uuid::parse_str(local_generation).expect("local generation must be a UUID"); + assert_ne!(local_generation, foreign_generation); + assert!(get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_TIMESTAMP).is_some()); + assert!( + get_consistent_str(&stored.user_defined, SUFFIX_REPLICATION_STATUS).is_some_and(|status| status.contains(target)) + ); + for suffix in [ + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_REPLICATION_STATUS, + ] { + assert_eq!( + stored + .user_defined + .keys() + .filter(|key| has_internal_suffix(key, suffix)) + .count(), + 2, + "{suffix} must be persisted as exactly one canonical dual-key pair" + ); + } + for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] { + assert!(!contains_key_str(&stored.user_defined, suffix), "foreign {suffix} must be removed"); + } + assert!( + stored + .user_defined + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)) + .all(|(_, value)| value != "REPLICA"), + "a local source admission may surface its own status but must not preserve foreign REPLICA state" + ); + } + #[tokio::test] #[serial_test::serial] async fn internal_put_object_preserves_the_caller_etag() { @@ -840,7 +1061,7 @@ mod tests { async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing_inner() { const FIRST_PART_SIZE: usize = 5 * 1024 * 1024; - let (store, bucket) = internal_put_test_bucket("internal-mpu").await; + let (store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu")).await; let usecase = DefaultObjectUsecase::from_global(); let first_part = vec![0x41u8; FIRST_PART_SIZE]; let last_part = b"tail of the multipart object".to_vec(); @@ -849,37 +1070,33 @@ mod tests { ctx.expected_md5_hex = None; ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-2".to_string()); - let upload_id = usecase - .internal_create_multipart_upload(&ctx) + let upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx)) .await .expect("internal multipart create must succeed"); - let part_one = usecase - .internal_upload_part( - &ctx, - &upload_id, - 1, - first_part.len() as u64, - Some(md5_hex(&first_part)), - body_stream(vec![Bytes::from(first_part.clone())]), - ) - .await - .expect("first internal part must stage"); - let part_two = usecase - .internal_upload_part( - &ctx, - &upload_id, - 2, - last_part.len() as u64, - Some(md5_hex(&last_part)), - body_stream(vec![Bytes::from(last_part.clone())]), - ) - .await - .expect("last internal part must stage"); + let part_one = Box::pin(usecase.internal_upload_part( + &ctx, + &upload_id, + 1, + first_part.len() as u64, + Some(md5_hex(&first_part)), + body_stream(vec![Bytes::from(first_part.clone())]), + )) + .await + .expect("first internal part must stage"); + let part_two = Box::pin(usecase.internal_upload_part( + &ctx, + &upload_id, + 2, + last_part.len() as u64, + Some(md5_hex(&last_part)), + body_stream(vec![Bytes::from(last_part.clone())]), + )) + .await + .expect("last internal part must stage"); assert_eq!(part_one.part_num, 1); assert_eq!(part_two.part_num, 2); - let obj_info = usecase - .internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two]) + let obj_info = Box::pin(usecase.internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two])) .await .expect("internal multipart complete must succeed"); assert_eq!(obj_info.size, (first_part.len() + last_part.len()) as i64); @@ -893,27 +1110,23 @@ mod tests { assert_eq!(stored.user_defined.get("origin").map(String::as_str), Some("unit-test")); assert!(contains_key_str(&stored.user_defined, SUFFIX_ODM_SOURCE)); - let aborted_upload_id = usecase - .internal_create_multipart_upload(&ctx) + let aborted_upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx)) .await .expect("second internal multipart create must succeed"); - usecase - .internal_upload_part( - &ctx, - &aborted_upload_id, - 1, - last_part.len() as u64, - None, - body_stream(vec![Bytes::from(last_part.clone())]), - ) - .await - .expect("part of the aborted upload must stage"); - usecase - .internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id) + Box::pin(usecase.internal_upload_part( + &ctx, + &aborted_upload_id, + 1, + last_part.len() as u64, + None, + body_stream(vec![Bytes::from(last_part.clone())]), + )) + .await + .expect("part of the aborted upload must stage"); + Box::pin(usecase.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id)) .await .expect("internal abort must succeed"); - let uploads = store - .list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100) + let uploads = Box::pin(store.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100)) .await .expect("list multipart uploads after abort"); assert!( @@ -923,10 +1136,137 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn internal_multipart_completion_recomputes_replication_admission_atomically() { + let (store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu-replication")).await; + let usecase = DefaultObjectUsecase::from_global(); + let target_a = "arn:aws:s3:::internal-target-a"; + let target_b = "arn:aws:s3:::internal-target-b"; + Box::pin(install_internal_replication_config(&bucket, Some(target_a))).await; + + let payload = b"internal multipart replication body".to_vec(); + let mut ctx = internal_context(&bucket, "multipart/replicated.bin", &[]); + ctx.size = None; + ctx.expected_md5_hex = None; + let foreign_generation = Uuid::from_u128(9002).to_string(); + for (key, value) in [ + ("X-Minio-Internal-Replication-Generation", foreign_generation.as_str()), + ("X-Minio-Internal-Replication-Timestamp", "foreign-multipart-time"), + ("X-Minio-Internal-Replication-Status", "arn:foreign=COMPLETED;"), + ("X-Minio-Internal-Replica-Status", "REPLICA"), + ("X-Minio-Internal-Replica-Timestamp", "foreign-replica-time"), + ] { + ctx.internal_metadata.insert(key.to_string(), value.to_string()); + } + ctx.internal_metadata + .insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string()); + let upload_id = Box::pin(usecase.internal_create_multipart_upload(&ctx)) + .await + .expect("internal multipart create must succeed"); + let staged = Box::pin(store.get_multipart_info(&bucket, &ctx.key, &upload_id, &ObjectOptions::default())) + .await + .expect("staged internal multipart metadata must be readable"); + let staged_generation = get_str(&staged.user_defined, SUFFIX_REPLICATION_GENERATION) + .expect("replication-admitted internal create must persist a generation"); + Uuid::parse_str(&staged_generation).expect("staged replication generation must be a UUID"); + assert_ne!(staged_generation, foreign_generation); + let staged_status = get_str(&staged.user_defined, SUFFIX_REPLICATION_STATUS) + .expect("replication-admitted internal create must persist PENDING"); + assert!(staged_status.contains(target_a)); + for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] { + assert!(!contains_key_str(&staged.user_defined, suffix), "staged foreign {suffix} must be removed"); + } + assert!( + staged + .user_defined + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)) + .all(|(_, value)| value != "REPLICA") + ); + + let part = Box::pin(usecase.internal_upload_part( + &ctx, + &upload_id, + 1, + payload.len() as u64, + Some(md5_hex(&payload)), + body_stream(vec![Bytes::from(payload)]), + )) + .await + .expect("internal multipart part must stage"); + + Box::pin(install_internal_replication_config(&bucket, Some(target_b))).await; + let completed = Box::pin(usecase.internal_complete_multipart_upload(&ctx, &upload_id, vec![part])) + .await + .expect("internal multipart completion must succeed"); + let completion_generation = get_str(&completed.user_defined, SUFFIX_REPLICATION_GENERATION) + .expect("completion must persist a new replication generation"); + Uuid::parse_str(&completion_generation).expect("completion replication generation must be a UUID"); + assert_ne!( + completion_generation, staged_generation, + "completion must replace the create-time generation" + ); + let completion_status = completed + .replication_status_internal + .as_deref() + .expect("completion must persist its PENDING target set"); + assert!(completion_status.contains(target_b)); + assert!(!completion_status.contains(target_a)); + for suffix in [SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP] { + assert!( + !contains_key_str(&completed.user_defined, suffix), + "completed foreign {suffix} must remain removed" + ); + } + assert!( + completed + .user_defined + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)) + .all(|(_, value)| value != "REPLICA") + ); + + let disabled_payload = b"internal multipart replication disabled".to_vec(); + let mut disabled_ctx = internal_context(&bucket, "multipart/disabled.bin", &[]); + disabled_ctx.size = None; + disabled_ctx.expected_md5_hex = None; + Box::pin(install_internal_replication_config(&bucket, Some(target_a))).await; + let disabled_upload_id = Box::pin(usecase.internal_create_multipart_upload(&disabled_ctx)) + .await + .expect("replication-admitted internal multipart create must succeed"); + let disabled_part = Box::pin(usecase.internal_upload_part( + &disabled_ctx, + &disabled_upload_id, + 1, + disabled_payload.len() as u64, + Some(md5_hex(&disabled_payload)), + body_stream(vec![Bytes::from(disabled_payload)]), + )) + .await + .expect("disabled-case internal multipart part must stage"); + Box::pin(install_internal_replication_config(&bucket, None)).await; + let disabled = + Box::pin(usecase.internal_complete_multipart_upload(&disabled_ctx, &disabled_upload_id, vec![disabled_part])) + .await + .expect("internal multipart completion with replication disabled must succeed"); + assert!(disabled.replication_status_internal.is_none()); + for suffix in [ + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_REPLICATION_STATUS, + ] { + assert!( + !contains_key_str(&disabled.user_defined, suffix), + "disabled completion must clear stale {suffix}" + ); + } + } + #[tokio::test] #[serial_test::serial] async fn internal_complete_multipart_upload_rejects_unordered_parts() { - let (_store, bucket) = internal_put_test_bucket("internal-mpu-order").await; + let (_store, bucket) = Box::pin(internal_put_test_bucket("internal-mpu-order")).await; let ctx = internal_context(&bucket, "unordered.bin", &[]); let parts = vec![ CompletePart { @@ -938,8 +1278,8 @@ mod tests { ..Default::default() }, ]; - let err = DefaultObjectUsecase::from_global() - .internal_complete_multipart_upload(&ctx, "upload", parts) + let usecase = DefaultObjectUsecase::from_global(); + let err = Box::pin(usecase.internal_complete_multipart_upload(&ctx, "upload", parts)) .await .expect_err("unordered parts must be rejected before touching the store"); assert_eq!(err.code, S3ErrorCode::InvalidRequest); diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index 367c2aff0..74cb3e12d 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -146,8 +146,8 @@ use rustfs_utils::http::insert_header; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICA_STATUS, - SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_RESTORE_OPERATION_ID, - SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, get_header, + SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_RESTORE_OPERATION_ID, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, get_header, headers::{ AMZ_CONTENT_SHA256, AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, @@ -186,6 +186,22 @@ fn object_s3_error_default(code: S3ErrorCode) -> S3Error { S3Error::new(code) } +/// Remove replication history owned by another source object before a local +/// object-creation decision is made. Compatibility keys are case-insensitive, +/// so use the shared removers rather than deleting only canonical spellings. +fn remove_source_replication_bookkeeping(user_defined: &mut HashMap) { + user_defined.retain(|key, _| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)); + for suffix in [ + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICATION_STATUS, + SUFFIX_REPLICATION_TIMESTAMP, + SUFFIX_REPLICA_STATUS, + SUFFIX_REPLICA_TIMESTAMP, + ] { + remove_str(user_defined, suffix); + } +} + mod copy; mod delete; mod extract; diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs index 000d76852..c05fc5ef1 100644 --- a/rustfs/src/app/object/on_demand_migration_put.rs +++ b/rustfs/src/app/object/on_demand_migration_put.rs @@ -280,14 +280,14 @@ mod tests { use crate::app::storage_api::object_usecase::on_demand_migration::{PullFailureReason, SourceSse}; use crate::app::storage_api::s3::{ BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration, - ReplicationRule, ReplicationRuleStatus, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, - ServerSideEncryptionRule, VersioningConfiguration, + ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, }; use crate::app::storage_api::test::bucket::utils::serialize; use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata}; use http::Method; - use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, get_str}; + use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, contains_key_str, get_str}; use sha2::{Digest as Sha256Digest, Sha256}; use std::time::SystemTime; use tokio::io::AsyncReadExt; @@ -614,9 +614,16 @@ mod tests { .expect("install bucket default SSE"); } - #[tokio::test] + #[test] #[serial_test::serial] - async fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag() { + fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag() { + crate::app::gating_test_env::run_large_stack_test( + "odm-write-back-under-bucket-default-sse", + write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag_inner, + ); + } + + async fn write_back_under_bucket_default_sse_stores_ciphertext_and_records_source_etag_inner() { let local_sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); temp_env::async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(local_sse_master_key))], async { let (store, bucket) = write_back_test_bucket("odm-wb-sse", false).await; @@ -677,6 +684,10 @@ mod tests { } async fn install_replication_rule(bucket: &str) { + install_replication_rule_with_tag(bucket, None).await; + } + + async fn install_replication_rule_with_tag(bucket: &str, required_tag: Option<(&str, &str)>) { let sys = get_global_bucket_metadata_sys().expect("bucket metadata system"); let metadata = { let sys = sys.read().await; @@ -700,7 +711,13 @@ mod tests { ..Default::default() }, existing_object_replication: None, - filter: None, + filter: required_tag.map(|(key, value)| ReplicationRuleFilter { + tag: Some(Tag { + key: Some(key.to_string()), + value: Some(value.to_string()), + }), + ..Default::default() + }), id: Some("odm".to_string()), prefix: Some(String::new()), priority: Some(1), @@ -750,6 +767,75 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn write_back_namespaces_source_user_metadata_that_looks_like_replication_bookkeeping() { + let (store, bucket) = write_back_test_bucket("odm-wb-user-metadata", true).await; + let target = "arn:aws:s3:::target-bucket"; + install_replication_rule_with_tag(&bucket, Some(("replicate", "yes"))).await; + + let body = b"initially unadmitted ODM object".to_vec(); + let mut head = source_head(&body); + let forged_status_key = "X-Minio-Internal-Replication-Status"; + let forged_generation_key = "X-RuStFs-InTeRnAl-RePlIcAtIoN-GeNeRaTiOn"; + let forged_replica_key = "X-Minio-Internal-Replica-Status"; + let forged_status = format!("{target}=COMPLETED;"); + let forged_generation = Uuid::from_u128(9003).to_string(); + head.user_metadata + .insert(forged_status_key.to_string(), forged_status.clone()); + head.user_metadata + .insert(forged_generation_key.to_string(), forged_generation.clone()); + head.user_metadata + .insert(forged_replica_key.to_string(), ReplicationStatusType::Replica.as_str().to_string()); + + let mut write_request = request(&bucket, "unadmitted.txt", head); + write_request.tags = Some(HashMap::from([("replicate".to_string(), "no".to_string())])); + OnDemandMigrationWriteBack::new() + .put_object(&write_request, body_stream(&body)) + .await + .expect("ODM write-back with reserved-looking user metadata must commit safely"); + + let stored = stored_object(&store, &bucket, "unadmitted.txt").await; + for suffix in [ + SUFFIX_REPLICATION_STATUS, + SUFFIX_REPLICATION_GENERATION, + SUFFIX_REPLICA_STATUS, + ] { + assert!( + !contains_key_str(&stored.user_defined, suffix), + "source user metadata must not become trusted {suffix} bookkeeping" + ); + } + for (key, value) in [ + (forged_status_key, forged_status.as_str()), + (forged_generation_key, forged_generation.as_str()), + (forged_replica_key, ReplicationStatusType::Replica.as_str()), + ] { + let namespaced = format!("x-amz-meta-{key}"); + assert!( + stored + .user_defined + .iter() + .any(|(stored_key, stored_value)| stored_key.eq_ignore_ascii_case(&namespaced) && stored_value == value), + "reserved-looking source metadata must survive only in the user namespace: {namespaced}" + ); + } + + let later_matching = crate::app::storage_api::object_usecase::bucket::replication::must_replicate_metadata( + &bucket, + "unadmitted.txt", + &stored.user_defined, + "replicate=yes".to_string(), + stored.replication_status.clone(), + ObjectOptions::default(), + ) + .await; + assert!( + !later_matching.replicate_any(), + "source user metadata must not forge historical target admission after a later matching tag" + ); + } + #[test] fn content_headers_follow_the_allowlist() { let head = SourceHead { diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 6fe1c06d7..d108b1885 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1854,6 +1854,7 @@ impl DefaultObjectUsecase { rustfs_io_metrics::record_put_object_stage_duration_from("app_replication_decision", replication_decision_stage_start); if dsc.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str( &mut opts.user_defined, diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 3a903e385..d7e6837a2 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -30,8 +30,8 @@ pub(crate) mod s3 { #[cfg(test)] pub(crate) use s3s::dto::{ BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration, - ReplicationRule, ReplicationRuleStatus, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, - ServerSideEncryptionRule, VersioningConfiguration, + ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, }; } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index aedd3e4d5..e62cd6d82 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -45,7 +45,9 @@ use rustfs_targets::EventName; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, }; -use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str}; +use rustfs_utils::http::{ + SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str, +}; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; use std::collections::HashMap; use std::fmt::Debug; @@ -679,6 +681,7 @@ impl S3 for FS { .await; if dsc.replicate_any() { let mut eval_metadata = HashMap::new(); + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); insert_str( @@ -1602,6 +1605,7 @@ impl S3 for FS { let mut eval_metadata = parse_object_lock_legal_hold(legal_hold)?; if dsc.replicate_any() { + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); } @@ -1820,6 +1824,7 @@ impl S3 for FS { let mut eval_metadata = parse_object_lock_retention(retention)?; if dsc.replicate_any() { + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); } @@ -1908,6 +1913,7 @@ impl S3 for FS { .await; if dsc.replicate_any() { let mut eval_metadata = HashMap::new(); + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); insert_str( diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 3c2959634..c723c4237 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -1876,6 +1876,10 @@ mod tests { let authorized = get_complete_multipart_upload_opts_with_replication_authorization(&headers, true) .expect("authorized replica status header should parse"); + assert!( + !authorized.replication_request, + "REPLICA-only compatibility requests intentionally omit the source-request marker" + ); assert_eq!(authorized.delete_marker_replication_status(), ReplicationStatusType::Replica); // For multipart the on-disk stamp actually comes from the SAME header // at initiate time (create-multipart builds its options through