Merge branch 'main' into overtrue/require-mc-e2e

* test(kms): bound Vault failover progress wait

* ci(nightly): honor manual dispatch ref

* test(kms): preserve Vault worker failures

* test(kms): validate Vault circuit recovery

.config/nextest.toml

@@ -325,8 +325,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #
 # Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
 # workflow_dispatch), which builds the rustfs binary once, installs awscurl so
-# the STS dual-node test actually exercises its path (it skips gracefully with
-# a visible log line when awscurl is absent), and routes scheduled failures
+# the STS dual-node test actually exercises its path (the test fails when
+# awscurl is absent), and routes scheduled failures
 # through .github/actions/schedule-failure-issue (ci-8). Explicit division of
 # labor with e2e-full: these tests run only in the consolidated nightly
 # workflow, not in the merge/main lane.

.github/workflows/ci.yml

@@ -681,6 +681,19 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
@@ -803,6 +816,20 @@ jobs:
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
+      - name: Install mc
+        env:
+          MC_VERSION: RELEASE.2025-08-13T08-35-41Z
+          MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
+        run: |
+          MC_BINARY="mc.linux-amd64.${MC_VERSION}"
+          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
+          echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
+          chmod +x "$RUNNER_TEMP/mc"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Verify mc
+        run: mc --version
+
       - name: Install Vault
         run: |
           VAULT_VERSION="1.17.6"

.github/workflows/e2e-replication-nightly.yml

@@ -75,11 +75,7 @@ jobs:
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: 'false'
 
-      # awscurl lets the STS dual-node test actually exercise its path. Without
-      # it the test skips gracefully with a visible log line
-      # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
-      # still passes — installing it just upgrades that one test from skip to
-      # real coverage.
+      # The STS dual-node test requires awscurl and fails if it is unavailable.
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
@@ -87,7 +83,7 @@ jobs:
 
       - name: Install awscurl
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
           echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
 
       - name: Verify awscurl

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/e2e_test/README.md

@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `find_available_port` | Random free port (isolation primitive) |
 | `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
-| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
+| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
 | `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
 | `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
-# Replication nightly lane; install awscurl so STS paths do not skip
+# Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
 job) to dodge this; local single-node tests already use random ports, so a
 lingering orphan is usually the cause of a spurious bind failure.
 
-**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
-visible log line (`awscurl_available()`); install `awscurl` to actually run
-them.
+**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
+spawn error. Install the pinned CI version before running their profiles.
 
 ## Related
 
@@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is:
 2. **Single-node** — spawns its own server via
    `RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
    isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
-3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
-   no fixed protocol ports. Tools that may be absent on the runner (e.g.
-   `awscurl`) are acceptable only when the test skips gracefully with a
-   visible log line (see `bucket_policy_check_test.rs`).
+3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
+   Vault, and no fixed protocol ports. Any required CLI must be pinned and
+   installed by the workflow; a missing CLI must fail the test.
 4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
    ci-13 / backlog#1148 ilm-3), not smoke candidates.
 

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[tokio::test]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
-        return Ok(());
-    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/common.rs

@@ -494,17 +494,6 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
-pub fn awscurl_available() -> bool {
-    let path = awscurl_binary_path();
-    if path.components().count() > 1 || path.is_absolute() {
-        return path.is_file();
-    }
-
-    std::env::var_os("PATH")
-        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
-        .unwrap_or(false)
-}
-
 // Global initialization
 static INIT: Once = Once::new();
 

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -16,9 +16,7 @@
 //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
 //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
@@ -175,11 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
 #[tokio::test]
 async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2eiamtag-{suffix}");
     let user_secret = "longSecretKeyForTest123!";
@@ -233,11 +226,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
 #[tokio::test]
 async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2ebptag-{suffix}");
     let user_secret = "longSecretKeyForTest456!";
@@ -294,11 +282,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
 #[tokio::test]
 async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-par-{suffix}");
     let parent_secret = "longSecretKeyForParentSts99!";
@@ -370,11 +353,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
 #[tokio::test]
 async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-del-par-{suffix}");
     let parent_secret = "longSecretKeyForParentDelete99!";

crates/e2e_test/src/kms/common.rs

@@ -22,9 +22,7 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
@@ -59,15 +57,6 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
-pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
-    if awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping {} because awscurl is not available in PATH", test_name);
-    true
-}
-
 pub fn sse_customer_key_md5_base64(key: &str) -> String {
     let mut hasher = Md5::new();
     hasher.update(key.as_bytes());
@@ -490,10 +479,6 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
-        return Ok(());
-    }
-
     info!("Testing KMS key management APIs");
 
     // Test CreateKey

crates/e2e_test/src/kms/kms_local_test.rs

@@ -20,8 +20,7 @@
 //! - Complete encryption/decryption lifecycle
 
 use super::common::{
-    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    test_kms_key_management, test_sse_c_encryption,
+    LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
@@ -29,9 +28,6 @@ use tracing::{error, info};
 #[tokio::test]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -22,8 +22,8 @@ use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
+    test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
     test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
@@ -62,9 +62,6 @@ impl VaultKmsTestContext {
 #[tokio::test]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -117,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
-        return Ok(());
-    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -203,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[tokio::test]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
-        return Ok(());
-    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -267,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
-        return Ok(());
-    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -297,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[tokio::test]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
-        return Ok(());
-    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/lib.rs

@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
-// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
-// replication metadata must not overwrite a newer local category state.
-#[cfg(test)]
-mod replication_lww_receiver_test;
-
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;

crates/e2e_test/src/mc_mirror_small_bucket_test.rs

@@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
     Ok(())
 }
 
-fn mc_available() -> bool {
-    Command::new("mc")
-        .arg("--version")
-        .output()
-        .is_ok_and(|output| output.status.success())
-}
-
 fn run_mc(args: &[&str]) -> TestResult {
     let output = Command::new("mc").args(args).output()?;
     if !output.status.success() {
@@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize {
 async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
     crate::common::init_logging();
     info!("Starting issue #3107 mc mirror regression test");
-    if !mc_available() {
-        info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
-        return Ok(());
-    }
+    run_mc(&["--version"])?;
 
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;

crates/e2e_test/src/multipart_auth_test.rs

@@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
 async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        return Ok(());
-    }
-
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/e2e_test/src/quota_test.rs

@@ -18,15 +18,6 @@ use http::{Method, StatusCode};
 use tokio::time::{Duration, sleep, timeout};
 use tracing::{debug, info};
 
-fn skip_without_awscurl() -> bool {
-    if crate::common::awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping quota test because awscurl is not available");
-    true
-}
-
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -276,9 +267,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -320,9 +308,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -371,9 +356,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -406,9 +388,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -442,9 +421,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -480,9 +456,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -513,9 +486,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -553,9 +523,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -593,9 +560,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -628,9 +592,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -689,9 +650,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -744,9 +702,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -789,9 +744,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -847,9 +799,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;

crates/e2e_test/src/replication_extension_test.rs

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -7281,11 +7280,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
 async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
     init_logging();
 
-    if !awscurl_available() {
-        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
-        return Ok(());
-    }
-
     let mut source_env = RustFSTestEnvironment::new().await?;
     source_env
         .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)

crates/e2e_test/src/replication_lww_receiver_test.rs

@@ -1,150 +0,0 @@
-#![cfg(test)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
-//! A4/P1-6).
-//!
-//! In an active-active topology both sites' metadata states arrive at the
-//! peer as authorized replication PUTs carrying per-category source
-//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
-//! family). Before the fix the receiver applied them unconditionally, so a
-//! stale delivery overwrote a newer local state and the two sites diverged
-//! permanently while both reported COMPLETED. This test drives one live
-//! `rustfs` server with simulated inbound replication PUTs for the same
-//! object version and asserts the newer tagging state wins regardless of
-//! delivery order, while a stale delivery still succeeds at the object level
-//! (a failure would loop through MRF re-delivering the stale value).
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
-const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
-const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
-const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
-
-const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
-const T_STALE: &str = "2026-01-01T00:00:01Z";
-const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-const T_NEWER: &str = "2026-03-01T00:00:00Z";
-
-/// Simulated inbound authorized replication PUT: same object version, tags and
-/// the source-authored tagging timestamp carried in transport headers.
-async fn inbound_replication_put(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-    version_id: &str,
-    tags: &str,
-    tagging_timestamp: &str,
-) -> TestResult {
-    let version_id = version_id.to_string();
-    let tagging_timestamp = tagging_timestamp.to_string();
-    client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from_static(b"lww-e2e-body"))
-        .tagging(tags)
-        .customize()
-        .mutate_request(move |req| {
-            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
-            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
-            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
-            req.headers_mut()
-                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
-        })
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
-    let tagging = client
-        .get_object_tagging()
-        .bucket(bucket)
-        .key(key)
-        .version_id(version_id)
-        .send()
-        .await
-        .expect("object tagging should be readable");
-    tagging
-        .tag_set()
-        .iter()
-        .find(|tag| tag.key() == tag_key)
-        .map(|tag| tag.value().to_string())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let client = env.create_s3_client();
-
-    let bucket = "replication-lww-receiver";
-    let key = "object";
-    client.create_bucket().bucket(bucket).send().await?;
-    client
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Enabled)
-                .build(),
-        )
-        .send()
-        .await?;
-
-    // First delivery establishes version V with tags stamped T_LOCAL.
-    let version_id = uuid::Uuid::new_v4().to_string();
-    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "the first delivery must establish the tagged version"
-    );
-
-    // A stale delivery (older source timestamp) must succeed at the object
-    // level but must NOT overwrite the newer tags.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
-    );
-
-    // A newer delivery still converges the version onto the newest state.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("newer"),
-        "a newer inbound delivery must overwrite older tags"
-    );
-
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(key)
-        .version_id(&version_id)
-        .send()
-        .await?;
-    env.delete_test_bucket(bucket).await.ok();
-    Ok(())
-}

crates/e2e_test/src/security_boundary_test.rs

@@ -21,7 +21,7 @@
 //! - SSRF prevention (internal/private endpoints rejected for tiering)
 //! - Race condition handling (concurrent writes converge without corruption)
 
-use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
@@ -225,16 +225,11 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
 /// outcome — the internal endpoint is not accepted — is asserted here.
 ///
 /// The admin API is exercised via signed `awscurl` requests, matching the
-/// pattern used by the other admin-API E2E tests in this crate; the test is
-/// skipped when `awscurl` is not installed.
+/// pattern used by the other admin-API E2E tests in this crate. The full E2E
+/// lane installs and verifies the pinned `awscurl` prerequisite.
 #[tokio::test]
 async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping tiering URL validation test because awscurl is not available");
-        return Ok(());
-    }
-
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -3868,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                 actual_size,
                 object_info.etag.clone().unwrap_or_default(),
                 object_info.mod_time,
-                &put_opts.internal,
             ),
         )
         .await

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
     actual_size: String,
     source_etag: String,
     source_mtime: Option<OffsetDateTime>,
-    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
             // mtime must degrade to epoch so header() suppresses the header
             // instead of asserting the replication time as the object's mtime.
             source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            // Carry the per-category LWW timestamps on the complete request as
-            // well: the receiver's CompleteMultipartUpload options builder
-            // parses the same headers, so the multipart transport gets the
-            // same receiver-side LWW as the single-PUT transport
-            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
-            tagging_timestamp: source_internal.tagging_timestamp,
-            retention_timestamp: source_internal.retention_timestamp,
-            legalhold_timestamp: source_internal.legalhold_timestamp,
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -672,39 +663,20 @@ mod tests {
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
         let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
-        let source_internal = AdvancedPutOptions {
-            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
-            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
-            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
-            ..Default::default()
-        };
         let options = replication_complete_multipart_options(
             "1024".to_string(),
             "0123456789abcdef0123456789abcdef-3".to_string(),
             Some(source_mtime),
-            &source_internal,
         );
         assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
         assert_eq!(options.internal.source_mtime, source_mtime);
 
-        // The complete request must carry the same per-category LWW timestamps
-        // as the initiate request; the receiver reads them from the complete
-        // headers (rustfs/backlog#1953).
-        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
-        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
-        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
-
         // Absent source mtime must degrade to epoch (header suppressed), not
         // the AdvancedPutOptions default of now_utc() — that default would
         // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence. Unset category timestamps stay epoch so
-        // header() keeps suppressing them.
-        let options_no_mtime =
-            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
+        // multipart HEAD convergence.
+        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
         assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),

crates/ecstore/src/io_support/bitrot.rs

@@ -784,6 +784,24 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 ///
 /// # Returns
 /// A Result containing the BitrotWriterWrapper or an error
+/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
+///
+/// A known length is grown by one checksum per shard so the on-disk file size
+/// matches what the bitrot writer emits. A negative length is the
+/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
+/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
+/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
+/// a fixed body length when locating the authenticated trailer. Clamping it
+/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
+/// because a genuinely empty object still means an empty body.
+fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
+    if length <= 0 {
+        return length;
+    }
+    let length = length as usize;
+    (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+}
+
 pub async fn create_bitrot_writer(
     is_inline_buffer: bool,
     disk: Option<&DiskStore>,
@@ -796,12 +814,7 @@ pub async fn create_bitrot_writer(
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
-        let length = if length > 0 {
-            let length = length as usize;
-            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
-        } else {
-            0
-        };
+        let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
 
         let file = disk.create_file("", volume, path, length).await?;
         #[cfg(feature = "hotpath")]
@@ -820,6 +833,25 @@ mod tests {
     use rustfs_rio::ChunkReader;
     use std::collections::VecDeque;
 
+    #[test]
+    fn bitrot_create_file_size_grows_known_length_by_checksums() {
+        // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
+    }
+
+    #[test]
+    fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
+        assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
+        // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
+        // put_file_stream receiver relies on a non-positive size to parse the auth
+        // trailer from the stream tail, so the sentinel must survive untouched.
+        assert_eq!(
+            bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
+            rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
+        );
+    }
+
     struct TestChunkReader {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2318,57 +2318,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             fi.set_data_moved();
         }
 
-        // Receiver-side LWW (rustfs/backlog#1953): the multipart replication
-        // transport carries the category values at CreateMultipartUpload (in
-        // the staged upload metadata) and the source category timestamps on
-        // the complete request. Read the destination version under the held
-        // object write lock and keep any category this site modified more
-        // recently. Read failures (version absent on first replication, quorum
-        // errors) keep today's overwrite semantics: failing the complete would
-        // loop through MRF, re-delivering the stale value forever.
-        if crate::set_disk::ops::object::replication_lww_applicable(opts)
-            && let Some(version_id) = fi.version_id
-        {
-            match self
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        version_id: Some(version_id.to_string()),
-                        no_lock: true,
-                        metadata_cache_safe: false,
-                        versioned: opts.versioned,
-                        version_suspended: opts.version_suspended,
-                        ..Default::default()
-                    },
-                )
-                .await
-            {
-                Ok(existing) => {
-                    let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
-                    crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
-                }
-                // Version absent: first replication of this version, nothing
-                // local to compare — the normal path, not a degraded one.
-                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
-                Err(err) => {
-                    // Degraded path: without the stored state the inbound
-                    // metadata is applied unchanged — exactly the overwrite
-                    // LWW exists to prevent — so this must be operator-visible.
-                    warn!(
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        bucket,
-                        object,
-                        version_id = %version_id,
-                        error = %err,
-                        state = "replication_lww_read_unavailable",
-                        "SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
-                    );
-                }
-            }
-        }
-
         for meta in parts_metadatas.iter_mut() {
             if meta.has_valid_erasure_geometry() {
                 meta.size = fi.size;
@@ -7106,97 +7055,6 @@ mod tests {
                 .await
         }
 
-        /// Receiver-side LWW on the multipart replication transport
-        /// (rustfs/backlog#1953): a metadata-only replication of a multipart
-        /// source object rides CreateMultipartUpload (category values in the
-        /// upload metadata) + CompleteMultipartUpload (category timestamps in
-        /// the complete options). A stale inbound tagging timestamp must not
-        /// overwrite a newer locally-tagged destination version.
-        #[tokio::test]
-        #[serial]
-        async fn complete_multipart_upload_stale_replication_tags_keep_local() {
-            use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
-            use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
-            use time::format_description::well_known::Rfc3339;
-
-            const T_OLD: &str = "2026-01-01T00:00:00Z";
-            const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-            let bucket = "multipart-replication-lww-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-
-            // Local destination version with newer tags.
-            let version_id = Uuid::new_v4();
-            let mut local_metadata = HashMap::new();
-            local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-            rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-            let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
-            set_disks
-                .put_object(
-                    bucket,
-                    object,
-                    &mut local_reader,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        user_defined: local_metadata,
-                        // Explicit-version PUTs require the bucket Object Lock snapshot.
-                        object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
-                            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-                        ))),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("local versioned put should commit");
-
-            // Inbound replication upload carrying older tags for the same version.
-            let mut inbound_metadata = HashMap::new();
-            inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-            rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-            let create_opts = ObjectOptions {
-                versioned: true,
-                user_defined: inbound_metadata,
-                ..Default::default()
-            };
-            let (upload_id, parts) =
-                stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
-            rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
-
-            let complete_opts = ObjectOptions {
-                versioned: true,
-                replication_request: true,
-                replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
-                ..Default::default()
-            };
-            set_disks
-                .clone()
-                .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
-                .await
-                .expect("replication multipart completion should succeed even when a category keeps local values");
-
-            let info = set_disks
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("completed version should be readable");
-            assert_eq!(
-                info.user_tags.as_str(),
-                "site=local",
-                "older inbound multipart tags must not overwrite newer local tags"
-            );
-            assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-        }
-
         #[tokio::test]
         #[serial]
         async fn complete_multipart_upload_assigns_completion_version_id() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -1881,110 +1881,6 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
     transported
 }
 
-/// True when an authorized replication write carries at least one per-category
-/// source timestamp, i.e. receiver-side LWW has something to judge.
-pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
-    opts.replication_request
-        && (opts.replication_tagging_timestamp.is_some()
-            || opts.replication_retention_timestamp.is_some()
-            || opts.replication_legalhold_timestamp.is_some())
-}
-
-/// The stored per-category state of a destination version, as compared by
-/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
-/// externalizes tags into `user_tags` (stripping the metadata key), so the
-/// tag value is folded back into map form here.
-pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
-    let mut stored = (*existing.user_defined).clone();
-    if !existing.user_tags.is_empty() {
-        stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
-    }
-    stored
-}
-
-/// Receiver-side last-writer-wins for authorized replication writes
-/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
-/// whole-object transports, so in active-active topologies an inbound write
-/// carries the source's tags / retention / legal hold verbatim and would
-/// otherwise overwrite a category the destination modified more recently —
-/// both sites end up permanently diverged while reporting COMPLETED.
-///
-/// Judged per category, only when the inbound request carries that category's
-/// source timestamp (`ObjectOptions::replication_*_timestamp`):
-/// - stored timestamp newer than inbound: the local category values and
-///   timestamp are kept; the rest of the write proceeds per the inbound
-///   metadata and the object-level result stays successful (failing the write
-///   instead would loop through MRF, re-delivering the stale value forever);
-/// - otherwise the inbound category wins and its internal timestamp key is
-///   pinned to the source-authored time — the PUT path re-stamps the
-///   object-lock timestamps with the receiver's clock
-///   (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
-///   `now()` via `eval_metadata`), which would make the replica's clock the
-///   LWW authority and wedge later convergence;
-/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
-///   overwrite behavior is preserved.
-///
-/// Returns whether `inbound` was modified. Callers must hold the object write
-/// lock so the stored values compared here are the ones being replaced.
-pub(in crate::set_disk) fn merge_replication_metadata_lww(
-    inbound: &mut HashMap<String, String>,
-    existing: &HashMap<String, String>,
-    opts: &ObjectOptions,
-) -> bool {
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::metadata_compat::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        remove_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
-        (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
-        (
-            opts.replication_retention_timestamp,
-            SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
-        ),
-        (
-            opts.replication_legalhold_timestamp,
-            SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
-        ),
-    ];
-
-    let mut changed = false;
-    for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
-        let Some(inbound_timestamp) = inbound_timestamp else { continue };
-        let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
-        let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
-        if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
-            inbound.retain(|key, _| !is_category_value_key(key));
-            remove_str(inbound, timestamp_suffix);
-            for (key, value) in existing {
-                if is_category_value_key(key) {
-                    inbound.insert(key.clone(), value.clone());
-                }
-            }
-            // Restore the winning timestamp via insert_str, not a verbatim key
-            // copy: a MinIO-written version may carry only the
-            // x-minio-internal- key, and the dual-key invariant requires every
-            // write to produce both keys.
-            if let Some(stored_value) = get_str(existing, timestamp_suffix) {
-                rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
-            }
-            changed = true;
-        } else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
-            && get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
-        {
-            rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
-            changed = true;
-        }
-    }
-    changed
-}
-
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
         &self,
@@ -2666,22 +2562,6 @@ impl SetDisks {
                         if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
                             return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
                         }
-                        // Receiver-side LWW (rustfs/backlog#1953): reuse this
-                        // commit-lock read of the destination version so a
-                        // category (tags / retention / legal hold) modified
-                        // more recently on this site is kept instead of being
-                        // overwritten by the inbound replication metadata.
-                        if replication_lww_applicable(opts) {
-                            let stored = stored_replication_category_metadata(&existing);
-                            let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
-                            if merge_replication_metadata_lww(&mut merged, &stored, opts) {
-                                for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
-                                    if disk.is_some() {
-                                        pfi.metadata = merged.clone();
-                                    }
-                                }
-                            }
-                        }
                     }
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
@@ -8186,357 +8066,6 @@ mod replication_quota_safety_tests {
     }
 }
 
-#[cfg(test)]
-mod replication_lww_tests {
-    //! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
-    //! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
-    //! (tags / retention / legal hold) is older than the destination version's
-    //! stored timestamp must keep the local category values instead of
-    //! overwriting them; categories are judged independently and the write
-    //! itself still succeeds.
-
-    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
-    use super::*;
-    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        insert_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    const T_OLD: &str = "2026-01-01T00:00:00Z";
-    const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-    const T_NEW: &str = "2026-03-01T00:00:00Z";
-
-    fn parse_ts(value: &str) -> OffsetDateTime {
-        OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
-    }
-
-    async fn make_bucket(disks: &[DiskStore], bucket: &str) {
-        for disk in disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-    }
-
-    async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
-        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
-        set_disks
-            .put_object(bucket, object, &mut reader, opts)
-            .await
-            .expect("versioned put should commit");
-        assert_eq!(opts.version_id.as_deref(), Some(version_id));
-    }
-
-    fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
-        ObjectOptions {
-            versioned: true,
-            version_id: Some(version_id.to_string()),
-            user_defined,
-            // Explicit-version PUTs require the bucket Object Lock snapshot.
-            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
-                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-            ))),
-            ..Default::default()
-        }
-    }
-
-    /// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
-    async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
-    }
-
-    fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
-        ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(timestamp)),
-            ..versioned_opts(version_id, user_defined)
-        }
-    }
-
-    async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
-        set_disks
-            .get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
-            .await
-            .expect("version should be readable")
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_tagging_keeps_newer_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=local",
-            "older inbound tags must not overwrite newer local tags"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the winning local tagging timestamp must be preserved"
-        );
-    }
-
-    #[tokio::test]
-    async fn inbound_newer_tagging_overwrites_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-newer";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_NEW),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "newer inbound tags must overwrite older local tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_wins_when_local_has_no_tagging_timestamp() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-no-local-ts";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Pre-P1-6 data: local tags without a stored tagging timestamp.
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "without a local timestamp the inbound category must win (pre-LWW data compatibility)"
-        );
-    }
-
-    #[tokio::test]
-    async fn categories_are_judged_independently() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-category-independent";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
-        // timestamp key only, the shape a replicated retention clear stores.
-        // (An active local retention would already block the overwrite at the
-        // WORM gate; the LWW-reachable retention states are cleared/expired.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound: older tags (T_OLD), newer retention (T_NEW).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            replication_retention_timestamp: Some(parse_ts(T_NEW)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
-            Some("COMPLIANCE"),
-            "the newer retention category must be applied in the same write"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_legal_hold_keeps_local_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-legalhold-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
-        // still ON already blocks the overwrite at the WORM gate; the
-        // LWW-reachable divergence is a stale inbound ON resurrecting a hold
-        // that was released more recently on this site.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
-            Some("OFF"),
-            "a stale inbound legal hold must not resurrect a hold released more recently"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
-            Some(T_LOCAL)
-        );
-    }
-
-    /// Dual-key invariant under LWW: a MinIO-written destination version may
-    /// carry only the x-minio-internal timestamp key; when the local category
-    /// wins, the restored map must still hold BOTH compatibility keys.
-    #[test]
-    fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        let existing = HashMap::from([
-            (AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
-            ("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
-        ]);
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            ..Default::default()
-        };
-
-        assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
-        assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
-        assert_eq!(
-            inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
-            Some(T_LOCAL),
-            "the RustFS twin key must be materialized even when the source version only had the MinIO key"
-        );
-        assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
-    }
-
-    /// When the inbound category wins, the stored timestamp must be the
-    /// source-authored one: the PUT path's eval_metadata stamps the
-    /// object-lock timestamps with the receiver's clock
-    /// (`parse_object_lock_retention`), which would otherwise make this
-    /// replica's clock the LWW authority and wedge later convergence.
-    #[tokio::test]
-    async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-retention-ts-pinned";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local cleared retention at T_OLD.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound newer retention: the source authored T_LOCAL, but the PUT
-        // path's eval_metadata stomped the metadata key with receiver-now
-        // (simulated by T_NEW here).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the stored category timestamp must be the source-authored time, not the receiver's clock"
-        );
-        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
-    }
-
-    #[tokio::test]
-    async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-deleted";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert!(
-            info.user_tags.is_empty(),
-            "a newer local tag deletion must not be resurrected by older inbound tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-    }
-}
-
 #[cfg(test)]
 mod inline_put_commit_path_tests {
     use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

rustfs/src/storage/options.rs

@@ -554,11 +554,10 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
 
     // Persist into the internal metadata keys so a later outbound replication
     // pass (replication_target_boundary) reads the source's modification
-    // times instead of falling back to mod_time. Receiver-side LWW happens at
-    // the set layer under the object write lock
-    // (ecstore set_disk::ops::object::merge_replication_metadata_lww,
-    // rustfs/backlog#1953): a category whose stored timestamp is newer than
-    // the inbound one keeps the local values.
+    // times instead of falling back to mod_time.
+    // TODO(P1-6): receiver-side LWW is still missing — when the stored
+    // per-category timestamp is newer than the inbound one, the existing
+    // tags/retention/legal-hold should win instead of being overwritten.
     for (timestamp, suffix) in [
         (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
         (opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!