Merge branch 'main' into overtrue/activate-group-e2e

* test(kms): bound Vault failover progress wait

* ci(nightly): honor manual dispatch ref

* test(kms): preserve Vault worker failures

* test(kms): validate Vault circuit recovery

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
-sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
+sha256-darwin=f832043fcca8c0b616c5d820a3a652da7544298ef5812a8668a3a9a3e4607b8b
+sha256-linux=93b94adb110b86a41d0b7313909e0bf53cb1515e2d08e8f105652b29b249990f

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/e2e_test/src/group_delete_test.rs

@@ -14,7 +14,7 @@
 
 //! E2E tests for group management (fixes #2028).
 
-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
 
 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
     env.start_rustfs_server(vec![]).await?;
 
     // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
     let user_body = serde_json::json!({
         "secretKey": "testuser1secret",
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created testuser1");
 
     // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": "testgroup",
         "members": ["testuser1"],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added testuser1 to testgroup");
 
     // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
     info!("Delete of non-empty group correctly rejected");
 
     // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
         "isRemove": true,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
     info!("Removed testuser1 from testgroup");
 
     // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
     info!("Deleted empty testgroup successfully");
 
     // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
     info!("Confirmed testgroup no longer exists");
 
     Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
         "Statement": [{
             "Effect": "Allow",
             "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
         }]
     });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
     info!("Created canned policy {}", policy_name);
 
     // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {} with no explicit policy", user_name);
 
     // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
     info!("Attached policy {} to group {}", policy_name, group_name);
 
     // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
     let group_name = "soledeletegroup";
 
     // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {}", user_name);
 
     // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
     info!("Deleted user {}", user_name);
 
     // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
     info!("Deleted group {} after user was removed", group_name);
 
     Ok(())

crates/e2e_test/src/lib.rs

@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
-// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
-// replication metadata must not overwrite a newer local category state.
-#[cfg(test)]
-mod replication_lww_receiver_test;
-
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;

crates/e2e_test/src/replication_lww_receiver_test.rs

@@ -1,150 +0,0 @@
-#![cfg(test)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
-//! A4/P1-6).
-//!
-//! In an active-active topology both sites' metadata states arrive at the
-//! peer as authorized replication PUTs carrying per-category source
-//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
-//! family). Before the fix the receiver applied them unconditionally, so a
-//! stale delivery overwrote a newer local state and the two sites diverged
-//! permanently while both reported COMPLETED. This test drives one live
-//! `rustfs` server with simulated inbound replication PUTs for the same
-//! object version and asserts the newer tagging state wins regardless of
-//! delivery order, while a stale delivery still succeeds at the object level
-//! (a failure would loop through MRF re-delivering the stale value).
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
-const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
-const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
-const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
-
-const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
-const T_STALE: &str = "2026-01-01T00:00:01Z";
-const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-const T_NEWER: &str = "2026-03-01T00:00:00Z";
-
-/// Simulated inbound authorized replication PUT: same object version, tags and
-/// the source-authored tagging timestamp carried in transport headers.
-async fn inbound_replication_put(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-    version_id: &str,
-    tags: &str,
-    tagging_timestamp: &str,
-) -> TestResult {
-    let version_id = version_id.to_string();
-    let tagging_timestamp = tagging_timestamp.to_string();
-    client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from_static(b"lww-e2e-body"))
-        .tagging(tags)
-        .customize()
-        .mutate_request(move |req| {
-            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
-            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
-            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
-            req.headers_mut()
-                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
-        })
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
-    let tagging = client
-        .get_object_tagging()
-        .bucket(bucket)
-        .key(key)
-        .version_id(version_id)
-        .send()
-        .await
-        .expect("object tagging should be readable");
-    tagging
-        .tag_set()
-        .iter()
-        .find(|tag| tag.key() == tag_key)
-        .map(|tag| tag.value().to_string())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let client = env.create_s3_client();
-
-    let bucket = "replication-lww-receiver";
-    let key = "object";
-    client.create_bucket().bucket(bucket).send().await?;
-    client
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Enabled)
-                .build(),
-        )
-        .send()
-        .await?;
-
-    // First delivery establishes version V with tags stamped T_LOCAL.
-    let version_id = uuid::Uuid::new_v4().to_string();
-    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "the first delivery must establish the tagged version"
-    );
-
-    // A stale delivery (older source timestamp) must succeed at the object
-    // level but must NOT overwrite the newer tags.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
-    );
-
-    // A newer delivery still converges the version onto the newest state.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("newer"),
-        "a newer inbound delivery must overwrite older tags"
-    );
-
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(key)
-        .version_id(&version_id)
-        .send()
-        .await?;
-    env.delete_test_bucket(bucket).await.ok();
-    Ok(())
-}

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -3868,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                 actual_size,
                 object_info.etag.clone().unwrap_or_default(),
                 object_info.mod_time,
-                &put_opts.internal,
             ),
         )
         .await

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
     actual_size: String,
     source_etag: String,
     source_mtime: Option<OffsetDateTime>,
-    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
             // mtime must degrade to epoch so header() suppresses the header
             // instead of asserting the replication time as the object's mtime.
             source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            // Carry the per-category LWW timestamps on the complete request as
-            // well: the receiver's CompleteMultipartUpload options builder
-            // parses the same headers, so the multipart transport gets the
-            // same receiver-side LWW as the single-PUT transport
-            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
-            tagging_timestamp: source_internal.tagging_timestamp,
-            retention_timestamp: source_internal.retention_timestamp,
-            legalhold_timestamp: source_internal.legalhold_timestamp,
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -672,39 +663,20 @@ mod tests {
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
         let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
-        let source_internal = AdvancedPutOptions {
-            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
-            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
-            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
-            ..Default::default()
-        };
         let options = replication_complete_multipart_options(
             "1024".to_string(),
             "0123456789abcdef0123456789abcdef-3".to_string(),
             Some(source_mtime),
-            &source_internal,
         );
         assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
         assert_eq!(options.internal.source_mtime, source_mtime);
 
-        // The complete request must carry the same per-category LWW timestamps
-        // as the initiate request; the receiver reads them from the complete
-        // headers (rustfs/backlog#1953).
-        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
-        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
-        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
-
         // Absent source mtime must degrade to epoch (header suppressed), not
         // the AdvancedPutOptions default of now_utc() — that default would
         // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence. Unset category timestamps stay epoch so
-        // header() keeps suppressing them.
-        let options_no_mtime =
-            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
+        // multipart HEAD convergence.
+        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
         assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),

crates/ecstore/src/io_support/bitrot.rs

@@ -784,6 +784,24 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 ///
 /// # Returns
 /// A Result containing the BitrotWriterWrapper or an error
+/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
+///
+/// A known length is grown by one checksum per shard so the on-disk file size
+/// matches what the bitrot writer emits. A negative length is the
+/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
+/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
+/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
+/// a fixed body length when locating the authenticated trailer. Clamping it
+/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
+/// because a genuinely empty object still means an empty body.
+fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
+    if length <= 0 {
+        return length;
+    }
+    let length = length as usize;
+    (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+}
+
 pub async fn create_bitrot_writer(
     is_inline_buffer: bool,
     disk: Option<&DiskStore>,
@@ -796,12 +814,7 @@ pub async fn create_bitrot_writer(
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
-        let length = if length > 0 {
-            let length = length as usize;
-            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
-        } else {
-            0
-        };
+        let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
 
         let file = disk.create_file("", volume, path, length).await?;
         #[cfg(feature = "hotpath")]
@@ -820,6 +833,25 @@ mod tests {
     use rustfs_rio::ChunkReader;
     use std::collections::VecDeque;
 
+    #[test]
+    fn bitrot_create_file_size_grows_known_length_by_checksums() {
+        // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
+    }
+
+    #[test]
+    fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
+        assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
+        // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
+        // put_file_stream receiver relies on a non-positive size to parse the auth
+        // trailer from the stream tail, so the sentinel must survive untouched.
+        assert_eq!(
+            bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
+            rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
+        );
+    }
+
     struct TestChunkReader {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2318,57 +2318,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             fi.set_data_moved();
         }
 
-        // Receiver-side LWW (rustfs/backlog#1953): the multipart replication
-        // transport carries the category values at CreateMultipartUpload (in
-        // the staged upload metadata) and the source category timestamps on
-        // the complete request. Read the destination version under the held
-        // object write lock and keep any category this site modified more
-        // recently. Read failures (version absent on first replication, quorum
-        // errors) keep today's overwrite semantics: failing the complete would
-        // loop through MRF, re-delivering the stale value forever.
-        if crate::set_disk::ops::object::replication_lww_applicable(opts)
-            && let Some(version_id) = fi.version_id
-        {
-            match self
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        version_id: Some(version_id.to_string()),
-                        no_lock: true,
-                        metadata_cache_safe: false,
-                        versioned: opts.versioned,
-                        version_suspended: opts.version_suspended,
-                        ..Default::default()
-                    },
-                )
-                .await
-            {
-                Ok(existing) => {
-                    let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
-                    crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
-                }
-                // Version absent: first replication of this version, nothing
-                // local to compare — the normal path, not a degraded one.
-                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
-                Err(err) => {
-                    // Degraded path: without the stored state the inbound
-                    // metadata is applied unchanged — exactly the overwrite
-                    // LWW exists to prevent — so this must be operator-visible.
-                    warn!(
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        bucket,
-                        object,
-                        version_id = %version_id,
-                        error = %err,
-                        state = "replication_lww_read_unavailable",
-                        "SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
-                    );
-                }
-            }
-        }
-
         for meta in parts_metadatas.iter_mut() {
             if meta.has_valid_erasure_geometry() {
                 meta.size = fi.size;
@@ -7106,97 +7055,6 @@ mod tests {
                 .await
         }
 
-        /// Receiver-side LWW on the multipart replication transport
-        /// (rustfs/backlog#1953): a metadata-only replication of a multipart
-        /// source object rides CreateMultipartUpload (category values in the
-        /// upload metadata) + CompleteMultipartUpload (category timestamps in
-        /// the complete options). A stale inbound tagging timestamp must not
-        /// overwrite a newer locally-tagged destination version.
-        #[tokio::test]
-        #[serial]
-        async fn complete_multipart_upload_stale_replication_tags_keep_local() {
-            use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
-            use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
-            use time::format_description::well_known::Rfc3339;
-
-            const T_OLD: &str = "2026-01-01T00:00:00Z";
-            const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-            let bucket = "multipart-replication-lww-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-
-            // Local destination version with newer tags.
-            let version_id = Uuid::new_v4();
-            let mut local_metadata = HashMap::new();
-            local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-            rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-            let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
-            set_disks
-                .put_object(
-                    bucket,
-                    object,
-                    &mut local_reader,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        user_defined: local_metadata,
-                        // Explicit-version PUTs require the bucket Object Lock snapshot.
-                        object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
-                            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-                        ))),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("local versioned put should commit");
-
-            // Inbound replication upload carrying older tags for the same version.
-            let mut inbound_metadata = HashMap::new();
-            inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-            rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-            let create_opts = ObjectOptions {
-                versioned: true,
-                user_defined: inbound_metadata,
-                ..Default::default()
-            };
-            let (upload_id, parts) =
-                stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
-            rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
-
-            let complete_opts = ObjectOptions {
-                versioned: true,
-                replication_request: true,
-                replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
-                ..Default::default()
-            };
-            set_disks
-                .clone()
-                .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
-                .await
-                .expect("replication multipart completion should succeed even when a category keeps local values");
-
-            let info = set_disks
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("completed version should be readable");
-            assert_eq!(
-                info.user_tags.as_str(),
-                "site=local",
-                "older inbound multipart tags must not overwrite newer local tags"
-            );
-            assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-        }
-
         #[tokio::test]
         #[serial]
         async fn complete_multipart_upload_assigns_completion_version_id() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -1881,110 +1881,6 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
     transported
 }
 
-/// True when an authorized replication write carries at least one per-category
-/// source timestamp, i.e. receiver-side LWW has something to judge.
-pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
-    opts.replication_request
-        && (opts.replication_tagging_timestamp.is_some()
-            || opts.replication_retention_timestamp.is_some()
-            || opts.replication_legalhold_timestamp.is_some())
-}
-
-/// The stored per-category state of a destination version, as compared by
-/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
-/// externalizes tags into `user_tags` (stripping the metadata key), so the
-/// tag value is folded back into map form here.
-pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
-    let mut stored = (*existing.user_defined).clone();
-    if !existing.user_tags.is_empty() {
-        stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
-    }
-    stored
-}
-
-/// Receiver-side last-writer-wins for authorized replication writes
-/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
-/// whole-object transports, so in active-active topologies an inbound write
-/// carries the source's tags / retention / legal hold verbatim and would
-/// otherwise overwrite a category the destination modified more recently —
-/// both sites end up permanently diverged while reporting COMPLETED.
-///
-/// Judged per category, only when the inbound request carries that category's
-/// source timestamp (`ObjectOptions::replication_*_timestamp`):
-/// - stored timestamp newer than inbound: the local category values and
-///   timestamp are kept; the rest of the write proceeds per the inbound
-///   metadata and the object-level result stays successful (failing the write
-///   instead would loop through MRF, re-delivering the stale value forever);
-/// - otherwise the inbound category wins and its internal timestamp key is
-///   pinned to the source-authored time — the PUT path re-stamps the
-///   object-lock timestamps with the receiver's clock
-///   (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
-///   `now()` via `eval_metadata`), which would make the replica's clock the
-///   LWW authority and wedge later convergence;
-/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
-///   overwrite behavior is preserved.
-///
-/// Returns whether `inbound` was modified. Callers must hold the object write
-/// lock so the stored values compared here are the ones being replaced.
-pub(in crate::set_disk) fn merge_replication_metadata_lww(
-    inbound: &mut HashMap<String, String>,
-    existing: &HashMap<String, String>,
-    opts: &ObjectOptions,
-) -> bool {
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::metadata_compat::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        remove_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
-        (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
-        (
-            opts.replication_retention_timestamp,
-            SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
-        ),
-        (
-            opts.replication_legalhold_timestamp,
-            SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
-        ),
-    ];
-
-    let mut changed = false;
-    for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
-        let Some(inbound_timestamp) = inbound_timestamp else { continue };
-        let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
-        let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
-        if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
-            inbound.retain(|key, _| !is_category_value_key(key));
-            remove_str(inbound, timestamp_suffix);
-            for (key, value) in existing {
-                if is_category_value_key(key) {
-                    inbound.insert(key.clone(), value.clone());
-                }
-            }
-            // Restore the winning timestamp via insert_str, not a verbatim key
-            // copy: a MinIO-written version may carry only the
-            // x-minio-internal- key, and the dual-key invariant requires every
-            // write to produce both keys.
-            if let Some(stored_value) = get_str(existing, timestamp_suffix) {
-                rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
-            }
-            changed = true;
-        } else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
-            && get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
-        {
-            rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
-            changed = true;
-        }
-    }
-    changed
-}
-
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
         &self,
@@ -2666,22 +2562,6 @@ impl SetDisks {
                         if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
                             return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
                         }
-                        // Receiver-side LWW (rustfs/backlog#1953): reuse this
-                        // commit-lock read of the destination version so a
-                        // category (tags / retention / legal hold) modified
-                        // more recently on this site is kept instead of being
-                        // overwritten by the inbound replication metadata.
-                        if replication_lww_applicable(opts) {
-                            let stored = stored_replication_category_metadata(&existing);
-                            let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
-                            if merge_replication_metadata_lww(&mut merged, &stored, opts) {
-                                for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
-                                    if disk.is_some() {
-                                        pfi.metadata = merged.clone();
-                                    }
-                                }
-                            }
-                        }
                     }
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
@@ -8186,357 +8066,6 @@ mod replication_quota_safety_tests {
     }
 }
 
-#[cfg(test)]
-mod replication_lww_tests {
-    //! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
-    //! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
-    //! (tags / retention / legal hold) is older than the destination version's
-    //! stored timestamp must keep the local category values instead of
-    //! overwriting them; categories are judged independently and the write
-    //! itself still succeeds.
-
-    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
-    use super::*;
-    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        insert_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    const T_OLD: &str = "2026-01-01T00:00:00Z";
-    const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-    const T_NEW: &str = "2026-03-01T00:00:00Z";
-
-    fn parse_ts(value: &str) -> OffsetDateTime {
-        OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
-    }
-
-    async fn make_bucket(disks: &[DiskStore], bucket: &str) {
-        for disk in disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-    }
-
-    async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
-        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
-        set_disks
-            .put_object(bucket, object, &mut reader, opts)
-            .await
-            .expect("versioned put should commit");
-        assert_eq!(opts.version_id.as_deref(), Some(version_id));
-    }
-
-    fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
-        ObjectOptions {
-            versioned: true,
-            version_id: Some(version_id.to_string()),
-            user_defined,
-            // Explicit-version PUTs require the bucket Object Lock snapshot.
-            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
-                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-            ))),
-            ..Default::default()
-        }
-    }
-
-    /// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
-    async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
-    }
-
-    fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
-        ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(timestamp)),
-            ..versioned_opts(version_id, user_defined)
-        }
-    }
-
-    async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
-        set_disks
-            .get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
-            .await
-            .expect("version should be readable")
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_tagging_keeps_newer_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=local",
-            "older inbound tags must not overwrite newer local tags"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the winning local tagging timestamp must be preserved"
-        );
-    }
-
-    #[tokio::test]
-    async fn inbound_newer_tagging_overwrites_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-newer";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_NEW),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "newer inbound tags must overwrite older local tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_wins_when_local_has_no_tagging_timestamp() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-no-local-ts";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Pre-P1-6 data: local tags without a stored tagging timestamp.
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "without a local timestamp the inbound category must win (pre-LWW data compatibility)"
-        );
-    }
-
-    #[tokio::test]
-    async fn categories_are_judged_independently() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-category-independent";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
-        // timestamp key only, the shape a replicated retention clear stores.
-        // (An active local retention would already block the overwrite at the
-        // WORM gate; the LWW-reachable retention states are cleared/expired.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound: older tags (T_OLD), newer retention (T_NEW).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            replication_retention_timestamp: Some(parse_ts(T_NEW)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
-            Some("COMPLIANCE"),
-            "the newer retention category must be applied in the same write"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_legal_hold_keeps_local_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-legalhold-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
-        // still ON already blocks the overwrite at the WORM gate; the
-        // LWW-reachable divergence is a stale inbound ON resurrecting a hold
-        // that was released more recently on this site.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
-            Some("OFF"),
-            "a stale inbound legal hold must not resurrect a hold released more recently"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
-            Some(T_LOCAL)
-        );
-    }
-
-    /// Dual-key invariant under LWW: a MinIO-written destination version may
-    /// carry only the x-minio-internal timestamp key; when the local category
-    /// wins, the restored map must still hold BOTH compatibility keys.
-    #[test]
-    fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        let existing = HashMap::from([
-            (AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
-            ("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
-        ]);
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            ..Default::default()
-        };
-
-        assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
-        assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
-        assert_eq!(
-            inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
-            Some(T_LOCAL),
-            "the RustFS twin key must be materialized even when the source version only had the MinIO key"
-        );
-        assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
-    }
-
-    /// When the inbound category wins, the stored timestamp must be the
-    /// source-authored one: the PUT path's eval_metadata stamps the
-    /// object-lock timestamps with the receiver's clock
-    /// (`parse_object_lock_retention`), which would otherwise make this
-    /// replica's clock the LWW authority and wedge later convergence.
-    #[tokio::test]
-    async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-retention-ts-pinned";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local cleared retention at T_OLD.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound newer retention: the source authored T_LOCAL, but the PUT
-        // path's eval_metadata stomped the metadata key with receiver-now
-        // (simulated by T_NEW here).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the stored category timestamp must be the source-authored time, not the receiver's clock"
-        );
-        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
-    }
-
-    #[tokio::test]
-    async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-deleted";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert!(
-            info.user_tags.is_empty(),
-            "a newer local tag deletion must not be resurrected by older inbound tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-    }
-}
-
 #[cfg(test)]
 mod inline_put_commit_path_tests {
     use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

docs/testing/e2e-suite-inventory.md

@@ -52,7 +52,7 @@
 | fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | get_stream_failure_observability_test | 1 |  |
-| group_delete_test | 1 |  |
+| group_delete_test | 4 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
 | heal_erasure_disk_rebuild_test | 4 | 🌙 |
@@ -99,4 +99,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
+**Total listed: 578 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 456 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.

rustfs/src/storage/options.rs

@@ -554,11 +554,10 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
 
     // Persist into the internal metadata keys so a later outbound replication
     // pass (replication_target_boundary) reads the source's modification
-    // times instead of falling back to mod_time. Receiver-side LWW happens at
-    // the set layer under the object write lock
-    // (ecstore set_disk::ops::object::merge_replication_metadata_lww,
-    // rustfs/backlog#1953): a category whose stored timestamp is newer than
-    // the inbound one keeps the local values.
+    // times instead of falling back to mod_time.
+    // TODO(P1-6): receiver-side LWW is still missing — when the stored
+    // per-category timestamp is newer than the inbound one, the existing
+    // tags/retention/legal-hold should win instead of being overwritten.
     for (timestamp, suffix) in [
         (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
         (opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!