Merge branch 'main' into overtrue/report-security-coverage-ratchet

* test(kms): bound Vault failover progress wait

* ci(nightly): honor manual dispatch ref

* test(kms): preserve Vault worker failures

* test(kms): validate Vault circuit recovery

.config/coverage-baselines.toml

@@ -0,0 +1,20 @@
+# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
+# Update counts only with a linked coverage run and a reviewed explanation.
+phase = "report-only"
+allowed_drop_percentage_points = 1.0
+
+[crates."crates/iam"]
+covered = 5149
+count = 8131
+
+[crates."crates/kms"]
+covered = 2950
+count = 4200
+
+[crates."crates/policy"]
+covered = 4636
+count = 5464
+
+[crates."crates/crypto"]
+covered = 469
+count = 494

.config/make/tests.mak

@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
 	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh

.github/workflows/coverage.yml

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+# Workspace line-coverage baseline and security-crate calibration
+# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# trend, while relevant pull requests run a report-only security-crate
+# comparison. Neither job is a required check during calibration.
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,6 +29,17 @@
 name: coverage
 
 on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "crates/iam/**"
+      - "crates/kms/**"
+      - "crates/policy/**"
+      - "crates/crypto/**"
+      - ".config/coverage-baselines.toml"
+      - "scripts/coverage_per_crate.py"
+      - "scripts/check_security_coverage.py"
+      - ".github/workflows/coverage.yml"
   workflow_dispatch:
   schedule:
     # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -39,6 +48,10 @@ on:
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
     - cron: "43 7 * * 0"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
 permissions:
@@ -46,12 +59,13 @@ permissions:
 
 jobs:
   coverage:
-    name: Workspace coverage (weekly)
+    name: Workspace line coverage
     runs-on: sm-standard-4
     # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
+    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # full suite. Exact-head run 32573798257 needed 119m42s including reports
+    # and artifact upload, so keep a bounded 30-minute publication margin.
+    timeout-minutes: 150
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +105,9 @@ jobs:
           cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
 
       - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload coverage artifact
         if: always()

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/e2e_test/src/lib.rs

@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
-// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
-// replication metadata must not overwrite a newer local category state.
-#[cfg(test)]
-mod replication_lww_receiver_test;
-
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;

crates/e2e_test/src/replication_lww_receiver_test.rs

@@ -1,150 +0,0 @@
-#![cfg(test)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
-//! A4/P1-6).
-//!
-//! In an active-active topology both sites' metadata states arrive at the
-//! peer as authorized replication PUTs carrying per-category source
-//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
-//! family). Before the fix the receiver applied them unconditionally, so a
-//! stale delivery overwrote a newer local state and the two sites diverged
-//! permanently while both reported COMPLETED. This test drives one live
-//! `rustfs` server with simulated inbound replication PUTs for the same
-//! object version and asserts the newer tagging state wins regardless of
-//! delivery order, while a stale delivery still succeeds at the object level
-//! (a failure would loop through MRF re-delivering the stale value).
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
-const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
-const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
-const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
-
-const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
-const T_STALE: &str = "2026-01-01T00:00:01Z";
-const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-const T_NEWER: &str = "2026-03-01T00:00:00Z";
-
-/// Simulated inbound authorized replication PUT: same object version, tags and
-/// the source-authored tagging timestamp carried in transport headers.
-async fn inbound_replication_put(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-    version_id: &str,
-    tags: &str,
-    tagging_timestamp: &str,
-) -> TestResult {
-    let version_id = version_id.to_string();
-    let tagging_timestamp = tagging_timestamp.to_string();
-    client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from_static(b"lww-e2e-body"))
-        .tagging(tags)
-        .customize()
-        .mutate_request(move |req| {
-            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
-            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
-            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
-            req.headers_mut()
-                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
-        })
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
-    let tagging = client
-        .get_object_tagging()
-        .bucket(bucket)
-        .key(key)
-        .version_id(version_id)
-        .send()
-        .await
-        .expect("object tagging should be readable");
-    tagging
-        .tag_set()
-        .iter()
-        .find(|tag| tag.key() == tag_key)
-        .map(|tag| tag.value().to_string())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let client = env.create_s3_client();
-
-    let bucket = "replication-lww-receiver";
-    let key = "object";
-    client.create_bucket().bucket(bucket).send().await?;
-    client
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Enabled)
-                .build(),
-        )
-        .send()
-        .await?;
-
-    // First delivery establishes version V with tags stamped T_LOCAL.
-    let version_id = uuid::Uuid::new_v4().to_string();
-    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "the first delivery must establish the tagged version"
-    );
-
-    // A stale delivery (older source timestamp) must succeed at the object
-    // level but must NOT overwrite the newer tags.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("local"),
-        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
-    );
-
-    // A newer delivery still converges the version onto the newest state.
-    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
-    assert_eq!(
-        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
-        Some("newer"),
-        "a newer inbound delivery must overwrite older tags"
-    );
-
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(key)
-        .version_id(&version_id)
-        .send()
-        .await?;
-    env.delete_test_bucket(bucket).await.ok();
-    Ok(())
-}

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -3868,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                 actual_size,
                 object_info.etag.clone().unwrap_or_default(),
                 object_info.mod_time,
-                &put_opts.internal,
             ),
         )
         .await

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
     actual_size: String,
     source_etag: String,
     source_mtime: Option<OffsetDateTime>,
-    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
             // mtime must degrade to epoch so header() suppresses the header
             // instead of asserting the replication time as the object's mtime.
             source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            // Carry the per-category LWW timestamps on the complete request as
-            // well: the receiver's CompleteMultipartUpload options builder
-            // parses the same headers, so the multipart transport gets the
-            // same receiver-side LWW as the single-PUT transport
-            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
-            tagging_timestamp: source_internal.tagging_timestamp,
-            retention_timestamp: source_internal.retention_timestamp,
-            legalhold_timestamp: source_internal.legalhold_timestamp,
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -672,39 +663,20 @@ mod tests {
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
         let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
-        let source_internal = AdvancedPutOptions {
-            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
-            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
-            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
-            ..Default::default()
-        };
         let options = replication_complete_multipart_options(
             "1024".to_string(),
             "0123456789abcdef0123456789abcdef-3".to_string(),
             Some(source_mtime),
-            &source_internal,
         );
         assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
         assert_eq!(options.internal.source_mtime, source_mtime);
 
-        // The complete request must carry the same per-category LWW timestamps
-        // as the initiate request; the receiver reads them from the complete
-        // headers (rustfs/backlog#1953).
-        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
-        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
-        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
-
         // Absent source mtime must degrade to epoch (header suppressed), not
         // the AdvancedPutOptions default of now_utc() — that default would
         // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence. Unset category timestamps stay epoch so
-        // header() keeps suppressing them.
-        let options_no_mtime =
-            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
+        // multipart HEAD convergence.
+        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
         assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
-        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),

crates/ecstore/src/io_support/bitrot.rs

@@ -784,6 +784,24 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 ///
 /// # Returns
 /// A Result containing the BitrotWriterWrapper or an error
+/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
+///
+/// A known length is grown by one checksum per shard so the on-disk file size
+/// matches what the bitrot writer emits. A negative length is the
+/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
+/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
+/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
+/// a fixed body length when locating the authenticated trailer. Clamping it
+/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
+/// because a genuinely empty object still means an empty body.
+fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
+    if length <= 0 {
+        return length;
+    }
+    let length = length as usize;
+    (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+}
+
 pub async fn create_bitrot_writer(
     is_inline_buffer: bool,
     disk: Option<&DiskStore>,
@@ -796,12 +814,7 @@ pub async fn create_bitrot_writer(
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
-        let length = if length > 0 {
-            let length = length as usize;
-            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
-        } else {
-            0
-        };
+        let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
 
         let file = disk.create_file("", volume, path, length).await?;
         #[cfg(feature = "hotpath")]
@@ -820,6 +833,25 @@ mod tests {
     use rustfs_rio::ChunkReader;
     use std::collections::VecDeque;
 
+    #[test]
+    fn bitrot_create_file_size_grows_known_length_by_checksums() {
+        // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
+    }
+
+    #[test]
+    fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
+        assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
+        // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
+        // put_file_stream receiver relies on a non-positive size to parse the auth
+        // trailer from the stream tail, so the sentinel must survive untouched.
+        assert_eq!(
+            bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
+            rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
+        );
+    }
+
     struct TestChunkReader {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2318,57 +2318,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             fi.set_data_moved();
         }
 
-        // Receiver-side LWW (rustfs/backlog#1953): the multipart replication
-        // transport carries the category values at CreateMultipartUpload (in
-        // the staged upload metadata) and the source category timestamps on
-        // the complete request. Read the destination version under the held
-        // object write lock and keep any category this site modified more
-        // recently. Read failures (version absent on first replication, quorum
-        // errors) keep today's overwrite semantics: failing the complete would
-        // loop through MRF, re-delivering the stale value forever.
-        if crate::set_disk::ops::object::replication_lww_applicable(opts)
-            && let Some(version_id) = fi.version_id
-        {
-            match self
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        version_id: Some(version_id.to_string()),
-                        no_lock: true,
-                        metadata_cache_safe: false,
-                        versioned: opts.versioned,
-                        version_suspended: opts.version_suspended,
-                        ..Default::default()
-                    },
-                )
-                .await
-            {
-                Ok(existing) => {
-                    let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
-                    crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
-                }
-                // Version absent: first replication of this version, nothing
-                // local to compare — the normal path, not a degraded one.
-                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
-                Err(err) => {
-                    // Degraded path: without the stored state the inbound
-                    // metadata is applied unchanged — exactly the overwrite
-                    // LWW exists to prevent — so this must be operator-visible.
-                    warn!(
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        bucket,
-                        object,
-                        version_id = %version_id,
-                        error = %err,
-                        state = "replication_lww_read_unavailable",
-                        "SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
-                    );
-                }
-            }
-        }
-
         for meta in parts_metadatas.iter_mut() {
             if meta.has_valid_erasure_geometry() {
                 meta.size = fi.size;
@@ -7106,97 +7055,6 @@ mod tests {
                 .await
         }
 
-        /// Receiver-side LWW on the multipart replication transport
-        /// (rustfs/backlog#1953): a metadata-only replication of a multipart
-        /// source object rides CreateMultipartUpload (category values in the
-        /// upload metadata) + CompleteMultipartUpload (category timestamps in
-        /// the complete options). A stale inbound tagging timestamp must not
-        /// overwrite a newer locally-tagged destination version.
-        #[tokio::test]
-        #[serial]
-        async fn complete_multipart_upload_stale_replication_tags_keep_local() {
-            use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
-            use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
-            use time::format_description::well_known::Rfc3339;
-
-            const T_OLD: &str = "2026-01-01T00:00:00Z";
-            const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-            let bucket = "multipart-replication-lww-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-
-            // Local destination version with newer tags.
-            let version_id = Uuid::new_v4();
-            let mut local_metadata = HashMap::new();
-            local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-            rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-            let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
-            set_disks
-                .put_object(
-                    bucket,
-                    object,
-                    &mut local_reader,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        user_defined: local_metadata,
-                        // Explicit-version PUTs require the bucket Object Lock snapshot.
-                        object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
-                            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-                        ))),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("local versioned put should commit");
-
-            // Inbound replication upload carrying older tags for the same version.
-            let mut inbound_metadata = HashMap::new();
-            inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-            rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-            let create_opts = ObjectOptions {
-                versioned: true,
-                user_defined: inbound_metadata,
-                ..Default::default()
-            };
-            let (upload_id, parts) =
-                stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
-            rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
-
-            let complete_opts = ObjectOptions {
-                versioned: true,
-                replication_request: true,
-                replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
-                ..Default::default()
-            };
-            set_disks
-                .clone()
-                .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
-                .await
-                .expect("replication multipart completion should succeed even when a category keeps local values");
-
-            let info = set_disks
-                .get_object_info(
-                    bucket,
-                    object,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: Some(version_id.to_string()),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("completed version should be readable");
-            assert_eq!(
-                info.user_tags.as_str(),
-                "site=local",
-                "older inbound multipart tags must not overwrite newer local tags"
-            );
-            assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-        }
-
         #[tokio::test]
         #[serial]
         async fn complete_multipart_upload_assigns_completion_version_id() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -1881,110 +1881,6 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
     transported
 }
 
-/// True when an authorized replication write carries at least one per-category
-/// source timestamp, i.e. receiver-side LWW has something to judge.
-pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
-    opts.replication_request
-        && (opts.replication_tagging_timestamp.is_some()
-            || opts.replication_retention_timestamp.is_some()
-            || opts.replication_legalhold_timestamp.is_some())
-}
-
-/// The stored per-category state of a destination version, as compared by
-/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
-/// externalizes tags into `user_tags` (stripping the metadata key), so the
-/// tag value is folded back into map form here.
-pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
-    let mut stored = (*existing.user_defined).clone();
-    if !existing.user_tags.is_empty() {
-        stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
-    }
-    stored
-}
-
-/// Receiver-side last-writer-wins for authorized replication writes
-/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
-/// whole-object transports, so in active-active topologies an inbound write
-/// carries the source's tags / retention / legal hold verbatim and would
-/// otherwise overwrite a category the destination modified more recently —
-/// both sites end up permanently diverged while reporting COMPLETED.
-///
-/// Judged per category, only when the inbound request carries that category's
-/// source timestamp (`ObjectOptions::replication_*_timestamp`):
-/// - stored timestamp newer than inbound: the local category values and
-///   timestamp are kept; the rest of the write proceeds per the inbound
-///   metadata and the object-level result stays successful (failing the write
-///   instead would loop through MRF, re-delivering the stale value forever);
-/// - otherwise the inbound category wins and its internal timestamp key is
-///   pinned to the source-authored time — the PUT path re-stamps the
-///   object-lock timestamps with the receiver's clock
-///   (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
-///   `now()` via `eval_metadata`), which would make the replica's clock the
-///   LWW authority and wedge later convergence;
-/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
-///   overwrite behavior is preserved.
-///
-/// Returns whether `inbound` was modified. Callers must hold the object write
-/// lock so the stored values compared here are the ones being replaced.
-pub(in crate::set_disk) fn merge_replication_metadata_lww(
-    inbound: &mut HashMap<String, String>,
-    existing: &HashMap<String, String>,
-    opts: &ObjectOptions,
-) -> bool {
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::metadata_compat::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        remove_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
-        (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
-        (
-            opts.replication_retention_timestamp,
-            SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
-        ),
-        (
-            opts.replication_legalhold_timestamp,
-            SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
-            &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
-        ),
-    ];
-
-    let mut changed = false;
-    for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
-        let Some(inbound_timestamp) = inbound_timestamp else { continue };
-        let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
-        let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
-        if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
-            inbound.retain(|key, _| !is_category_value_key(key));
-            remove_str(inbound, timestamp_suffix);
-            for (key, value) in existing {
-                if is_category_value_key(key) {
-                    inbound.insert(key.clone(), value.clone());
-                }
-            }
-            // Restore the winning timestamp via insert_str, not a verbatim key
-            // copy: a MinIO-written version may carry only the
-            // x-minio-internal- key, and the dual-key invariant requires every
-            // write to produce both keys.
-            if let Some(stored_value) = get_str(existing, timestamp_suffix) {
-                rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
-            }
-            changed = true;
-        } else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
-            && get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
-        {
-            rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
-            changed = true;
-        }
-    }
-    changed
-}
-
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
         &self,
@@ -2666,22 +2562,6 @@ impl SetDisks {
                         if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
                             return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
                         }
-                        // Receiver-side LWW (rustfs/backlog#1953): reuse this
-                        // commit-lock read of the destination version so a
-                        // category (tags / retention / legal hold) modified
-                        // more recently on this site is kept instead of being
-                        // overwritten by the inbound replication metadata.
-                        if replication_lww_applicable(opts) {
-                            let stored = stored_replication_category_metadata(&existing);
-                            let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
-                            if merge_replication_metadata_lww(&mut merged, &stored, opts) {
-                                for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
-                                    if disk.is_some() {
-                                        pfi.metadata = merged.clone();
-                                    }
-                                }
-                            }
-                        }
                     }
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
@@ -8186,357 +8066,6 @@ mod replication_quota_safety_tests {
     }
 }
 
-#[cfg(test)]
-mod replication_lww_tests {
-    //! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
-    //! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
-    //! (tags / retention / legal hold) is older than the destination version's
-    //! stored timestamp must keep the local category values instead of
-    //! overwriting them; categories are judged independently and the write
-    //! itself still succeeds.
-
-    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
-    use super::*;
-    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
-    use rustfs_utils::http::headers::{
-        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
-    };
-    use rustfs_utils::http::{
-        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
-        insert_str,
-    };
-    use time::format_description::well_known::Rfc3339;
-
-    const T_OLD: &str = "2026-01-01T00:00:00Z";
-    const T_LOCAL: &str = "2026-02-01T00:00:00Z";
-    const T_NEW: &str = "2026-03-01T00:00:00Z";
-
-    fn parse_ts(value: &str) -> OffsetDateTime {
-        OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
-    }
-
-    async fn make_bucket(disks: &[DiskStore], bucket: &str) {
-        for disk in disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-    }
-
-    async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
-        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
-        set_disks
-            .put_object(bucket, object, &mut reader, opts)
-            .await
-            .expect("versioned put should commit");
-        assert_eq!(opts.version_id.as_deref(), Some(version_id));
-    }
-
-    fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
-        ObjectOptions {
-            versioned: true,
-            version_id: Some(version_id.to_string()),
-            user_defined,
-            // Explicit-version PUTs require the bucket Object Lock snapshot.
-            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
-                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-            ))),
-            ..Default::default()
-        }
-    }
-
-    /// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
-    async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
-    }
-
-    fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
-        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
-        ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(timestamp)),
-            ..versioned_opts(version_id, user_defined)
-        }
-    }
-
-    async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
-        set_disks
-            .get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
-            .await
-            .expect("version should be readable")
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_tagging_keeps_newer_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=local",
-            "older inbound tags must not overwrite newer local tags"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the winning local tagging timestamp must be preserved"
-        );
-    }
-
-    #[tokio::test]
-    async fn inbound_newer_tagging_overwrites_local_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-newer";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_NEW),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "newer inbound tags must overwrite older local tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_wins_when_local_has_no_tagging_timestamp() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-no-local-ts";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Pre-P1-6 data: local tags without a stored tagging timestamp.
-        let mut user_defined = HashMap::new();
-        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_tags.as_str(),
-            "site=remote",
-            "without a local timestamp the inbound category must win (pre-LWW data compatibility)"
-        );
-    }
-
-    #[tokio::test]
-    async fn categories_are_judged_independently() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-category-independent";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
-        // timestamp key only, the shape a replicated retention clear stores.
-        // (An active local retention would already block the overwrite at the
-        // WORM gate; the LWW-reachable retention states are cleared/expired.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound: older tags (T_OLD), newer retention (T_NEW).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            replication_retention_timestamp: Some(parse_ts(T_NEW)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
-            Some("COMPLIANCE"),
-            "the newer retention category must be applied in the same write"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
-    }
-
-    #[tokio::test]
-    async fn inbound_stale_legal_hold_keeps_local_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-legalhold-stale";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
-        // still ON already blocks the overwrite at the WORM gate; the
-        // LWW-reachable divergence is a stale inbound ON resurrecting a hold
-        // that was released more recently on this site.)
-        let mut local = HashMap::new();
-        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
-            Some("OFF"),
-            "a stale inbound legal hold must not resurrect a hold released more recently"
-        );
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
-            Some(T_LOCAL)
-        );
-    }
-
-    /// Dual-key invariant under LWW: a MinIO-written destination version may
-    /// carry only the x-minio-internal timestamp key; when the local category
-    /// wins, the restored map must still hold BOTH compatibility keys.
-    #[test]
-    fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
-        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
-        let existing = HashMap::from([
-            (AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
-            ("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
-        ]);
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
-            ..Default::default()
-        };
-
-        assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
-        assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
-        assert_eq!(
-            inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
-            Some(T_LOCAL),
-            "the RustFS twin key must be materialized even when the source version only had the MinIO key"
-        );
-        assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
-    }
-
-    /// When the inbound category wins, the stored timestamp must be the
-    /// source-authored one: the PUT path's eval_metadata stamps the
-    /// object-lock timestamps with the receiver's clock
-    /// (`parse_object_lock_retention`), which would otherwise make this
-    /// replica's clock the LWW authority and wedge later convergence.
-    #[tokio::test]
-    async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-retention-ts-pinned";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-
-        // Local cleared retention at T_OLD.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        // Inbound newer retention: the source authored T_LOCAL, but the PUT
-        // path's eval_metadata stomped the metadata key with receiver-now
-        // (simulated by T_NEW here).
-        let mut inbound = HashMap::new();
-        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
-        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
-        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
-        let opts = ObjectOptions {
-            replication_request: true,
-            replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
-            ..versioned_opts(&version_id, inbound)
-        };
-        put_version(&set_disks, bucket, object, &version_id, &opts).await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert_eq!(
-            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
-            Some(T_LOCAL),
-            "the stored category timestamp must be the source-authored time, not the receiver's clock"
-        );
-        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
-    }
-
-    #[tokio::test]
-    async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "lww-tagging-deleted";
-        let object = "object";
-        let version_id = Uuid::new_v4().to_string();
-        make_bucket(&disk_stores, bucket).await;
-        // Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
-        let mut local = HashMap::new();
-        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
-        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
-
-        put_version(
-            &set_disks,
-            bucket,
-            object,
-            &version_id,
-            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
-        )
-        .await;
-
-        let info = version_info(&set_disks, bucket, object, &version_id).await;
-        assert!(
-            info.user_tags.is_empty(),
-            "a newer local tag deletion must not be resurrected by older inbound tags"
-        );
-        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
-    }
-}
-
 #[cfg(test)]
 mod inline_put_commit_path_tests {
     use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

docs/testing/README.md

@@ -158,10 +158,11 @@ added by backlog#1153 infra-4.
 
 ## Coverage
 
-Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
-exists for visibility and trend, never as a required check. Per-crate ratchets
-for the security-critical crates (iam / kms / policy / crypto) build on this
-baseline later (backlog#1153 infra-6, report-only first).
+Workspace line coverage is measured weekly. Pull requests that touch iam, kms,
+policy, or crypto also run a non-required, report-only comparison against
+`.config/coverage-baselines.toml`. During calibration, a regression is recorded
+in the job summary without failing the job; missing or malformed coverage
+evidence still fails closed (backlog#1153 infra-6).
 
 - **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
   dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
@@ -174,6 +175,12 @@ baseline later (backlog#1153 infra-6, report-only first).
   plus the full suite). It prints the same per-crate table via
   `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
   `coverage.json`.
+- **Security-critical ratchet**: relevant pull requests compare iam / kms /
+  policy / crypto line coverage with the versioned baseline. Drops greater than
+  the configured one-percentage-point calibration threshold are marked
+  `REGRESSION (report-only)`. The weekly summary runs the same comparison so
+  calibration continues even when no relevant pull request is open. Baseline
+  changes require a linked coverage run and a reviewed explanation.
 - **Trend comparison**: each run's job summary is the weekly per-crate
   snapshot — open two runs from the Actions history (workflow "coverage") and
   compare their tables. For line-level diffs, download the two runs'

rustfs/src/storage/options.rs

@@ -554,11 +554,10 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
 
     // Persist into the internal metadata keys so a later outbound replication
     // pass (replication_target_boundary) reads the source's modification
-    // times instead of falling back to mod_time. Receiver-side LWW happens at
-    // the set layer under the object write lock
-    // (ecstore set_disk::ops::object::merge_replication_metadata_lww,
-    // rustfs/backlog#1953): a category whose stored timestamp is newer than
-    // the inbound one keeps the local values.
+    // times instead of falling back to mod_time.
+    // TODO(P1-6): receiver-side LWW is still missing — when the stored
+    // per-category timestamp is newer than the inbound one, the existing
+    // tags/retention/legal-hold should win instead of being overwritten.
     for (timestamp, suffix) in [
         (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
         (opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),

scripts/check_security_coverage.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare security-critical crate line coverage with the report-only baseline."""
+
+import argparse
+import json
+import math
+import os
+import sys
+import tempfile
+import tomllib
+from pathlib import Path
+
+from coverage_per_crate import fmt_pct, load_coverage
+
+
+SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto")
+
+
+def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]:
+    with open(path, "rb") as fh:
+        config = tomllib.load(fh)
+
+    if config.get("phase") != "report-only":
+        raise ValueError("coverage baseline phase must be report-only")
+
+    raw_allowed_drop = config["allowed_drop_percentage_points"]
+    if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)):
+        raise ValueError("allowed_drop_percentage_points must be a number")
+    allowed_drop = float(raw_allowed_drop)
+    if not math.isfinite(allowed_drop) or allowed_drop < 0:
+        raise ValueError("allowed_drop_percentage_points must be finite and non-negative")
+
+    baselines: dict[str, tuple[int, int]] = {}
+    for crate, values in config["crates"].items():
+        covered = values["covered"]
+        count = values["count"]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid baseline for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid baseline for {crate}: {covered}/{count}")
+        baselines[crate] = (covered, count)
+    missing = [crate for crate in SECURITY_CRATES if crate not in baselines]
+    unexpected = sorted(set(baselines).difference(SECURITY_CRATES))
+    if missing or unexpected:
+        raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}")
+    return allowed_drop, baselines
+
+
+def compare(
+    current: dict[str, list[int]],
+    baselines: dict[str, tuple[int, int]],
+    allowed_drop: float,
+) -> list[tuple[str, int, int, int, int, float, bool]]:
+    rows = []
+    for crate, (baseline_covered, baseline_count) in baselines.items():
+        if crate not in current:
+            raise ValueError(f"coverage report is missing {crate}")
+        covered, count = current[crate]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid coverage for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid coverage for {crate}: {covered}/{count}")
+        current_pct = 100.0 * covered / count
+        baseline_pct = 100.0 * baseline_covered / baseline_count
+        delta = current_pct - baseline_pct
+        rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop))
+    return rows
+
+
+def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None:
+    print("## Security-critical coverage ratchet (report-only)")
+    print()
+    print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.")
+    print()
+    print("| Crate | Current | Baseline | Delta | Status |")
+    print("|---|---:|---:|---:|---|")
+    for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows:
+        status = "REGRESSION (report-only)" if regressed else "OK"
+        print(
+            f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) "
+            f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) "
+            f"| {delta:+.2f} pp | {status} |"
+        )
+    print()
+    print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.")
+
+
+def self_test() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        coverage = root / "coverage.json"
+        baseline = root / "baseline.toml"
+        coverage_data = {
+            "data": [
+                {
+                    "files": [
+                        {
+                            "filename": str(root / "crates/iam/src/lib.rs"),
+                            "summary": {"lines": {"covered": 80, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/kms/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/policy/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/crypto/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                    ],
+                    "totals": {"lines": {"covered": 350, "count": 400}},
+                }
+            ]
+        }
+        coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+        baseline_text = """phase = "report-only"
+allowed_drop_percentage_points = 1.0
+[crates."crates/iam"]
+covered = 90
+count = 100
+[crates."crates/kms"]
+covered = 85
+count = 100
+[crates."crates/policy"]
+covered = 90
+count = 100
+[crates."crates/crypto"]
+covered = 90
+count = 100
+"""
+        baseline.write_text(baseline_text, encoding="utf-8")
+        current, _ = load_coverage(str(coverage), str(root))
+        allowed_drop, baselines = load_baselines(str(baseline))
+        rows = compare(current, baselines, allowed_drop)
+        assert [row[-1] for row in rows] == [True, False, False, False]
+        try:
+            compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "coverage report is missing crates/kms"
+        else:
+            raise AssertionError("missing crate must fail closed")
+        try:
+            compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "invalid coverage for crates/iam: 101/100"
+        else:
+            raise AssertionError("invalid coverage must fail closed")
+        for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"):
+            baseline.write_text(
+                baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"),
+                encoding="utf-8",
+            )
+            try:
+                load_baselines(str(baseline))
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed")
+        for field, invalid_values in (
+            ("covered", ("true", '"90"', "90.0", "90.5")),
+            ("count", ("true", '"100"', "100.0", "100.5")),
+        ):
+            for invalid_value in invalid_values:
+                baseline.write_text(
+                    baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1),
+                    encoding="utf-8",
+                )
+                try:
+                    load_baselines(str(baseline))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed")
+        for covered, count in (
+            (True, 100),
+            (80, True),
+            (80.0, 100),
+            (80, 100.0),
+            (float("nan"), 100),
+            (80, float("inf")),
+        ):
+            try:
+                compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop)
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed")
+        lines = coverage_data["data"][0]["files"][0]["summary"]["lines"]
+        for field, invalid_values in (
+            ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))),
+            ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))),
+        ):
+            original = lines[field]
+            for invalid_value in invalid_values:
+                lines[field] = invalid_value
+                coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+                try:
+                    load_coverage(str(coverage), str(root))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed")
+            lines[field] = original
+        baseline.write_text(
+            baseline_text.replace(
+                '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n',
+                "",
+            ),
+            encoding="utf-8",
+        )
+        try:
+            load_baselines(str(baseline))
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("missing security-crate baseline must fail closed")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("coverage_json", nargs="?")
+    parser.add_argument("--baseline", default=".config/coverage-baselines.toml")
+    parser.add_argument("--repo-root", default=os.getcwd())
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        self_test()
+        print("security coverage self-test passed")
+        return 0
+    if not args.coverage_json:
+        parser.error("coverage_json is required unless --self-test is used")
+
+    try:
+        current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root))
+        allowed_drop, baselines = load_baselines(args.baseline)
+        rows = compare(current, baselines, allowed_drop)
+    except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    print_report(rows, allowed_drop)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/coverage_per_crate.py

@@ -47,6 +47,31 @@ def fmt_pct(covered: int, count: int) -> str:
     return f"{100.0 * covered / count:.2f}%" if count else "—"
 
 
+def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]:
+    covered = lines["covered"]
+    count = lines["count"]
+    if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count:
+        raise ValueError(f"invalid line coverage for {source}: {covered}/{count}")
+    return covered, count
+
+
+def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]:
+    with open(path, encoding="utf-8") as fh:
+        export = json.load(fh)
+
+    data = export["data"][0]
+    files = data["files"]
+    total_covered, total_count = _line_counts(data["totals"]["lines"], "totals")
+
+    crates: dict[str, list[int]] = {}
+    for f in files:
+        covered, count = _line_counts(f["summary"]["lines"], f["filename"])
+        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
+        acc[0] += covered
+        acc[1] += count
+    return crates, {"covered": total_covered, "count": total_count}
+
+
 def main() -> int:
     if len(sys.argv) < 2 or len(sys.argv) > 3:
         print(__doc__.strip(), file=sys.stderr)
@@ -54,24 +79,12 @@ def main() -> int:
     path = sys.argv[1]
     root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
 
-    with open(path, encoding="utf-8") as fh:
-        export = json.load(fh)
-
     try:
-        data = export["data"][0]
-        files = data["files"]
-        totals = data["totals"]["lines"]
-    except (KeyError, IndexError) as exc:
+        crates, totals = load_coverage(path, root)
+    except (KeyError, IndexError, ValueError) as exc:
         print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
         return 1
 
-    crates: dict[str, list[int]] = {}
-    for f in files:
-        lines = f["summary"]["lines"]
-        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
-        acc[0] += lines["covered"]
-        acc[1] += lines["count"]
-
     rows = sorted(
         crates.items(),
         key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!