fix(ecstore): preserve restore fileinfo errors

## Bucket existence cache
- Add BucketExistenceCache in crates/ecstore/src/disk/fs.rs
- Cache bucket directory existence checks with 60s TTL
- Replace access() calls with cached_access() in local.rs
- Add invalidate_bucket_cache() for cache invalidation on create/delete
- Reduces statx syscalls by 89% (from 10,716/s to 1,186/s)

## Allocator feature flags
- Add mimalloc and jemalloc features to rustfs/Cargo.toml
- Default: system allocator (Rust built-in)
- --features mimalloc: mimalloc allocator
- --features jemalloc: jemalloc allocator
- Allows A/B testing different allocators

## Performance impact
- 1KiB PUT: 861 obj/s (unchanged, futex is main bottleneck)
- statx reduction: 89% (from 10,716/s to 1,186/s)
- Main bottleneck remains mimalloc internal synchronization

Ref: rustfs/backlog#2005
Ref: microsoft/mimalloc#1372

Co-authored-by: hector <hetor@rustfs.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>

.config/coverage-baselines.toml

@@ -0,0 +1,20 @@
+# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
+# Update counts only with a linked coverage run and a reviewed explanation.
+phase = "report-only"
+allowed_drop_percentage_points = 1.0
+
+[crates."crates/iam"]
+covered = 5149
+count = 8131
+
+[crates."crates/kms"]
+covered = 2950
+count = 4200
+
+[crates."crates/policy"]
+covered = 4636
+count = 5464
+
+[crates."crates/crypto"]
+covered = 469
+count = 494

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
-sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
+sha256-darwin=88ee9684ece0e27294f2b3f0c9c8fe62890feff76aa47279d42dab0af3196fe2
+sha256-linux=d13337936af6778b1d2b2b255ae7fd350fdec94034be46daf738bd577653f799

.config/make/tests.mak

@@ -36,6 +36,8 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test

.config/nextest.toml

@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# The durable ILM decommission regressions build isolated multi-pool stores and
+# deliberately take source or target disks offline while checking fencing.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -107,9 +113,10 @@ filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
-# does not cross nextest process boundaries, so keep these tests in one group.
+# does not cross nextest process boundaries, so keep every Vault-backed test in
+# one group.
 [[profile.default.overrides]]
-filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
 
 # ---------------------------------------------------------------------------
@@ -190,6 +197,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -305,7 +316,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #     the target incl. multipart and the resync path, SSE-C and
 #     target-without-KMS stay fail-closed), and one guards event/history
 #     observers.
-#   * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
+#   * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
 #   * 1 `_real_three_node` site-replication test.
 #   * 1 `_real_single_node` service-account round-trip test.
@@ -325,8 +336,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #
 # Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
 # workflow_dispatch), which builds the rustfs binary once, installs awscurl so
-# the STS dual-node test actually exercises its path (it skips gracefully with
-# a visible log line when awscurl is absent), and routes scheduled failures
+# the STS dual-node test actually exercises its path (the test fails when
+# awscurl is absent), and routes scheduled failures
 # through .github/actions/schedule-failure-issue (ci-8). Explicit division of
 # labor with e2e-full: these tests run only in the consolidated nightly
 # workflow, not in the merge/main lane.
@@ -396,10 +407,10 @@ path = "junit.xml"
 #     object_lambda) — too heavy for the merge budget; they run in the
 #     e2e-nightly serial cluster-fault lane.
 #   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
-#     manual-localhost:9000 reliant/policy tests are ci-13's migration.
+#     manual-localhost:9000 reliant tests are ci-13's migration.
 #
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
@@ -443,5 +454,5 @@ filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
 [[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'

.github/actions/schedule-failure-issue/action.yml

@@ -14,9 +14,10 @@
 
 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
@@ -38,6 +39,30 @@ inputs:
       Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
+  details-file:
+    description: "Optional Markdown file appended to the issue body."
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -48,17 +73,22 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
+        DETAILS_FILE: ${{ inputs.details-file }}
       run: |
         set -euo pipefail
 
         title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
 
-        # Failed job names for this run attempt. The alert job runs while the
-        # run as a whole is still in progress, so inspect the jobs that have
-        # already completed with a non-success conclusion.
+        # Inspect the reported run attempt. It can be the current in-workflow
+        # failure or a completed run observed by the external watchdog.
         failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
                 | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
           failed_jobs="- (failed job not recorded yet — see the run page)"
         fi
 
+        details=""
+        if [ -n "${DETAILS_FILE}" ]; then
+          if [ -f "${DETAILS_FILE}" ]; then
+            details="$(cat "${DETAILS_FILE}")"
+          else
+            details="Details file was not available: \`${DETAILS_FILE}\`"
+          fi
+        fi
+
         body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Run of **${WORKFLOW_NAME}** did not complete successfully.
 
-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
+        - Event: \`${SOURCE_EVENT}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
 
-        Failed jobs:
+        Non-success jobs:
         ${failed_jobs}
+
+        ${details}
         EOF
         )"
 

.github/scheduled-validations.json

@@ -0,0 +1,14 @@
+[
+  { "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+]

.github/workflows/audit.yml

@@ -46,7 +46,7 @@ on:
     # advisory could sit unnoticed for seven days. The check list is unchanged —
     # splitting it into a light daily advisories-only run and a weekly full run
     # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '23 3 * * *' # Daily 03:23 UTC
   workflow_dispatch:
 
 permissions:

.github/workflows/build.yml

@@ -52,7 +52,7 @@ on:
       - ".dockerignore"
       - "flake.lock"
   schedule:
-    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
+    - cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
   workflow_dispatch:
     inputs:
       build_docker:
@@ -1032,3 +1032,23 @@ jobs:
 
           echo "🎉 Released $TAG successfully!"
           echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cache-warm.yml

@@ -94,6 +94,9 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
+  # ci.yml or the writer and readers use disjoint cache keys.
+  RUST_BACKTRACE: 1
 
 jobs:
   # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
@@ -101,7 +104,7 @@ jobs:
   warm-ci-dev:
     name: Warm ci-dev
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -191,7 +194,7 @@ jobs:
   warm-ci-feat-rio:
     name: Warm ci-feat-rio
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -219,7 +222,7 @@ jobs:
   warm-ci-feat-proto:
     name: Warm ci-feat-proto
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:

.github/workflows/ci-docs-only.yml

@@ -126,7 +126,10 @@ jobs:
         run: ./scripts/check_embedded_secrets.sh
 
       - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py
 
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh

.github/workflows/ci.yml

@@ -59,7 +59,7 @@ on:
   merge_group:
     types: [ checks_requested ]
   schedule:
-    - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
+    - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
   workflow_dispatch:
 
 permissions:
@@ -161,7 +161,10 @@ jobs:
         run: ./scripts/check_embedded_secrets.sh
 
       - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py
 
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
@@ -678,6 +681,19 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
@@ -800,6 +816,20 @@ jobs:
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
+      - name: Install mc
+        env:
+          MC_VERSION: RELEASE.2025-08-13T08-35-41Z
+          MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
+        run: |
+          MC_BINARY="mc.linux-amd64.${MC_VERSION}"
+          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
+          echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
+          chmod +x "$RUNNER_TEMP/mc"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Verify mc
+        run: mc --version
+
       - name: Install Vault
         run: |
           VAULT_VERSION="1.17.6"
@@ -1032,3 +1062,37 @@ jobs:
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/coverage.yml

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+# Workspace line-coverage baseline and security-crate calibration
+# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# trend, while relevant pull requests run a report-only security-crate
+# comparison. Neither job is a required check during calibration.
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,13 +29,28 @@
 name: coverage
 
 on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "crates/iam/**"
+      - "crates/kms/**"
+      - "crates/policy/**"
+      - "crates/crypto/**"
+      - ".config/coverage-baselines.toml"
+      - "scripts/coverage_per_crate.py"
+      - "scripts/check_security_coverage.py"
+      - ".github/workflows/coverage.yml"
   workflow_dispatch:
   schedule:
     # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
     # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
     # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
+    - cron: "43 7 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
 
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -46,12 +59,14 @@ permissions:
 
 jobs:
   coverage:
-    name: Workspace coverage (weekly)
+    name: Workspace line coverage
     runs-on: sm-standard-4
     # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
+    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # full suite. Two later exact-head runs exhausted 150 minutes before the
+    # report steps, so allow one additional 90-minute cold-run margin while
+    # keeping the calibration job bounded.
+    timeout-minutes: 240
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +106,9 @@ jobs:
           cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
 
       - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload coverage artifact
         if: always()

.github/workflows/e2e-replication-nightly.yml

@@ -40,7 +40,7 @@ on:
   schedule:
     # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
     # stale (01:30) and performance-ab (06:00).
-    - cron: "0 4 * * *"
+    - cron: "29 4 * * *"
 
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -75,11 +75,7 @@ jobs:
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: 'false'
 
-      # awscurl lets the STS dual-node test actually exercise its path. Without
-      # it the test skips gracefully with a visible log line
-      # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
-      # still passes — installing it just upgrades that one test from skip to
-      # real coverage.
+      # The STS dual-node test requires awscurl and fails if it is unavailable.
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
@@ -87,7 +83,7 @@ jobs:
 
       - name: Install awscurl
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
           echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
 
       - name: Verify awscurl
@@ -196,6 +192,12 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Install and verify protocol socket oracle
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq iproute2
+          ss -tn state CLOSE-WAIT >/dev/null
+
       # The suite owns fixed protocol ports and serializes its internal cases.
       - name: Verify protocol e2e membership
         env:

.github/workflows/e2e-s3tests.yml

@@ -21,6 +21,9 @@
 #     suite and reports promotion candidates. Regressions, unclassified tests,
 #     incomplete execution, and infrastructure errors fail the job; classified
 #     failures for not-yet-implemented features remain informational.
+#   - Non-blocking upstream HEAD canary: collects current upstream node IDs and
+#     reports new, removed, duplicate, or overlapping classifications without
+#     making upstream drift a release gate.
 #   - Manual runs (workflow_dispatch): same, with configurable mode/scope.
 #
 # All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -97,7 +100,7 @@ on:
   schedule:
     # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
     # single-node and the 4-node distributed topologies (matrix below).
-    - cron: "0 2 * * 0"
+    - cron: "19 2 * * 0"
 
 env:
   # main user
@@ -178,9 +181,14 @@ jobs:
 
       - name: Install Python tools
         run: |
-          python3 -m pip install --user --upgrade pip awscurl tox
+          python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
+      - name: Verify Python tools
+        run: |
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
+
       - name: Enable buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
@@ -287,6 +295,7 @@ jobs:
 
           frontend fe_s3
             bind *:9000
+            option http-buffer-request
             default_backend be_s3
 
           backend be_s3
@@ -302,7 +311,7 @@ jobs:
       - name: Wait for RustFS ready
         run: |
           for _ in {1..120}; do
-            if curl -sf "http://${S3_HOST}:${S3_PORT}/health" >/dev/null 2>&1; then
+            if curl -sf "http://${S3_HOST}:${S3_PORT}/health/ready" >/dev/null 2>&1; then
               echo "RustFS is ready"
               exit 0
             fi
@@ -353,6 +362,85 @@ jobs:
           name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
           path: artifacts/**
 
+  upstream-head-canary:
+    name: Upstream HEAD classification canary
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install collection tool
+        run: |
+          python3 -m pip install --user "tox==4.60.0"
+          python3 - <<'PY'
+          from importlib.metadata import version
+
+          assert version("tox") == "4.60.0"
+          PY
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Compare upstream HEAD classifications
+        id: upstream-compare
+        run: |
+          ARTIFACT_DIR="artifacts/s3tests-upstream-head"
+          UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
+          mkdir -p "${ARTIFACT_DIR}"
+          git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
+          git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
+          cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
+          (
+            cd "${UPSTREAM_DIR}"
+            S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
+              -q --collect-only s3tests/functional/test_s3.py \
+              -m "not rustfs_never_marker"
+          ) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
+          grep -E '^s3tests/functional/test_s3\.py::' \
+            "${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
+          python3 scripts/s3-tests/report_compat.py \
+            --lists-dir scripts/s3-tests \
+            --collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
+            --check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
+
+      - name: Publish canary report
+        if: always()
+        env:
+          CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
+        run: |
+          {
+            echo "## ceph/s3-tests upstream HEAD canary"
+            echo
+            if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
+              echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
+            fi
+            echo
+            echo '```text'
+            if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
+              cat artifacts/s3tests-upstream-head/classification-drift.txt
+            elif [ "${CANARY_OUTCOME}" != "success" ]; then
+              echo "Canary did not complete; inspect the collection log artifact."
+            else
+              echo "No classification drift detected."
+            fi
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload canary artifacts
+        if: always() && env.ACT != 'true'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: s3tests-upstream-head
+          path: artifacts/s3tests-upstream-head/**
+          retention-days: 14
+
   alert-on-failure:
     name: Alert on scheduled failure
     needs: [s3tests]

.github/workflows/fuzz.yml

@@ -30,7 +30,7 @@ on:
       - "Cargo.lock"
       - ".github/workflows/fuzz.yml"
   schedule:
-    - cron: "0 2 * * *"
+    - cron: "17 2 * * *"
   workflow_dispatch:
     inputs:
       profile:

.github/workflows/minio-interop.yml

@@ -121,3 +121,21 @@ jobs:
           cargo nextest run --run-ignored ignored-only --no-tests=fail \
             -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
             -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/mint.yml

@@ -45,13 +45,6 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: mint
 
 on:
@@ -70,13 +63,13 @@ on:
           - core
           - full
       mint-image:
-        description: "Mint image reference"
+        description: "Mint image reference (empty = pinned default)"
         required: false
-        default: "minio/mint:edge"
+        default: ""
   schedule:
     # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
     # 3h) has finished, so the two never contend for the same runner pool.
-    - cron: "0 6 * * 0"
+    - cron: "41 6 * * 0"
 
 env:
   S3_ACCESS_KEY: rustfsadmin-ci
@@ -162,7 +155,7 @@ jobs:
       - name: Wait for RustFS ready
         run: |
           for _ in {1..60}; do
-            if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then
+            if curl -sf http://127.0.0.1:9000/health/ready >/dev/null 2>&1; then
               echo "RustFS is ready"
               exit 0
             fi

.github/workflows/nightly-gnu.yml

@@ -16,7 +16,7 @@ name: Nightly GNU Build
 
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "7 0 * * *"
       timezone: "Asia/Shanghai"
   workflow_dispatch:
 
@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -194,3 +191,23 @@ jobs:
 
       - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
         run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/performance-ab.yml

@@ -22,22 +22,15 @@
 # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
 # blocked (rustfs/backlog#935 correction 1).
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Performance A/B
 
 on:
   schedule:
-    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
+    - cron: "31 6 * * *" # 06:31 UTC nightly, against main
   workflow_dispatch:
     inputs:
       duration:
-        description: "warp duration per round (short by default to fit the double-build budget)"
+        description: "warp duration per round"
         required: false
         default: "12s"
         type: string
@@ -46,12 +39,8 @@ on:
         required: false
         default: false
         type: boolean
-  push:
-    # Every main commit pre-builds and caches its release binary (perf-3) so the
-    # nightly A/B restores a ready baseline instead of paying the double build.
-    branches: [main]
-
 permissions:
+  actions: read
   contents: read
 
 env:
@@ -59,83 +48,19 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # perf-3: on every push to main, build the release binary once and cache it
-  # keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
-  # restore this instead of paying the ~32min-per-side source
-  # build. That double build is what pushed the expanded 24-cell nightly past its
-  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
-  # builds off the shared cargo cache keep each push cheap, and building on the
-  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
-  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
-  # are cancelled ~7/8 of the time and are not a reliable baseline.
-  build-baseline-cache:
-    name: Build + cache baseline binary
-    if: github.event_name == 'push'
-    runs-on: sm-standard-2
-    # Latest-wins: consumers only ever restore the binary for the *current*
-    # origin/main tip, so when pushes land faster than the ~65min build, a
-    # superseded build's output is dead weight — cancel it instead of stacking
-    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
-    # most costs one same-commit self-heal in the A/B job.
-    concurrency:
-      group: perf-baseline-build-main
-      cancel-in-progress: true
-    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
-    # single release build past 60min on this runner — every cache build on
-    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
-    # time of 1h0m0s") and the cache never populated. The measured binary must
-    # keep the production profile, so the budget absorbs the build instead.
-    timeout-minutes: 100
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Build release rustfs
-        run: cargo build --release --bin rustfs
-
-      - name: Stage binary for cache
-        run: |
-          set -euo pipefail
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-
-      - name: Cache baseline binary by SHA
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ github.sha }}
-
   warp-ab:
     name: Warp A/B budget gate
-    # Always run on schedule / manual dispatch. Never on push — that event only
-    # feeds build-baseline-cache above.
-    if: >-
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch'
     runs-on: sm-standard-2
-    # With perf-3's cached baseline binary the common (cache-hit) nightly is
-    # measurement-only and finishes well under 50min. This ceiling stays
-    # generous only to absorb the same-commit cache-miss self-heal (~65min
-    # single build with the post-#4806 LTO profile + measurement). A timeout
-    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
-    # not just failure). perf-6 recalibrates the budget once the noise study
-    # lands.
-    timeout-minutes: 120
+    # A normal nightly restores the last successful binary and builds only the
+    # candidate; daily access keeps that cache warm. A cache miss may build both
+    # and needs room for the A/B run plus artifact and cache publication.
+    timeout-minutes: 180
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          fetch-depth: 0 # baseline is built from origin/main
+          fetch-depth: 0 # baseline may be an earlier successful scheduled head
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -163,24 +88,55 @@ jobs:
           fi
           echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
 
-      # perf-3: resolve the commits so the cache can be keyed by SHA. The
-      # baseline is origin/main; the candidate is the checked-out ref. On the
-      # nightly (checkout == main) they are the same commit, so one cached binary
-      # serves both phases and the run does zero source builds.
+      # A failed regression run must keep comparing against the last known-good
+      # scheduled head. Otherwise the next nightly would absorb the regression
+      # into its baseline and turn green without a fix.
+      - name: Find last successful scheduled baseline
+        id: scheduled_baseline
+        if: github.event_name == 'schedule'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        with:
+          result-encoding: string
+          script: |
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "performance-ab.yml",
+              event: "schedule",
+              status: "success",
+              per_page: 1,
+            });
+            return data.workflow_runs[0]?.head_sha ?? "";
+
+      # Manual runs compare a selected ref with current main. Scheduled runs
+      # compare current main with the last successful scheduled head. With no
+      # history, the first run measures the candidate against itself and seeds
+      # that head only if the complete rig succeeds.
       - name: Resolve baseline / candidate commits
         id: commits
+        env:
+          SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
         run: |
           set -euo pipefail
-          baseline_sha="$(git rev-parse origin/main)"
           candidate_sha="$(git rev-parse HEAD)"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
+          else
+            baseline_sha="$(git rev-parse origin/main)"
+          fi
+          git cat-file -e "${baseline_sha}^{commit}"
+          if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+            echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
+            exit 1
+          fi
           echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
           echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
           echo "baseline commit: $baseline_sha"
           echo "candidate commit: $candidate_sha"
 
-      # Exact-key restore of the baseline binary built by build-baseline-cache
-      # when origin/main last landed. A miss (binary evicted or not built yet)
-      # leaves cache-hit unset and the rig falls back to a source build.
+      # Exact-key restore of the candidate binary saved by its successful
+      # scheduled run. A miss leaves cache-hit unset and falls back to a source
+      # build of that known-good head.
       - name: Restore cached baseline binary
         id: baseline_cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
@@ -270,11 +226,11 @@ jobs:
             elif [[ "$selfheal_built" == "true" ]]; then
               base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
             else
-              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
+              base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
             fi
             if [[ "$candidate_sha" == "$baseline_sha" ]]; then
-              # Nightly on main: the candidate is the same commit as the baseline,
-              # so reuse the one binary for both phases and skip all builds.
+              # No commits landed since the last successful baseline, so reuse
+              # the one binary for both phases and measure only rig drift.
               args+=(--candidate-bin "$base_bin")
               cand_src="same binary as baseline (same commit)"
             elif [[ "$candidate_built" == "true" ]]; then
@@ -362,6 +318,23 @@ jobs:
             fi
           } >> "$GITHUB_STEP_SUMMARY"
 
+      - name: Stage successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        run: |
+          set -euo pipefail
+          cp candidate-bin/rustfs baseline-bin/rustfs
+
+      - name: Cache successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
+
       # Scheduled failure alerting is handled by the alert-on-failure job below
       # (perf-2 consuming ci-8's schedule-failure-issue composite action).
 
@@ -369,6 +342,10 @@ jobs:
         if: always()
         run: |
           status="${{ steps.ab.outputs.status }}"
+          if [[ -z "$status" ]]; then
+            echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
+            exit 1
+          fi
           if [[ "$status" != "0" ]]; then
             echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
             exit "$status"

.github/workflows/runner-hygiene.yml

@@ -30,7 +30,7 @@ name: Runner Hygiene
 
 on:
   schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+    - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
   workflow_dispatch:
 
 permissions:

.github/workflows/scheduled-validation-freshness.yml

@@ -0,0 +1,57 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Freshness
+
+on:
+  schedule:
+    - cron: "47 23 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scheduled-validation-freshness
+  cancel-in-progress: false
+
+jobs:
+  check-freshness:
+    name: Check scheduled validation freshness
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check latest scheduled runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set +e
+          python3 scripts/check_scheduled_validation_freshness.py \
+            --report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
+          status=$?
+          cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
+          exit "${status}"
+      - name: Open or update freshness issue
+        if: failure()
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md

.github/workflows/scheduled-validation-watchdog.yml

@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}

CLAUDE.md

@@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04
 - Crate membership: `Cargo.toml` `[workspace].members`
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
-- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
+- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
+  matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
 - Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
   policy: [docs/testing/README.md](docs/testing/README.md)
 - Tier/ILM transition debugging (xl.meta inspection, versionId tracing):

CONTRIBUTING.md

@@ -70,6 +70,8 @@ make pre-pr
 
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
+> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
+
 ### 🔒 Automated Pre-commit Hooks
 #### What `make pre-commit` and `make pre-pr` actually run
 

Cargo.lock

@@ -68,16 +68,16 @@ dependencies = [
 
 [[package]]
 name = "aes-gcm"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
+checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
 dependencies = [
  "aead",
  "aes 0.9.2",
  "cipher 0.5.2",
  "ctr",
+ "ctutils",
  "ghash",
- "subtle",
  "zeroize",
 ]
 
@@ -333,6 +333,12 @@ dependencies = [
  "password-hash",
 ]
 
+[[package]]
+name = "array-init"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.8"
@@ -813,11 +819,12 @@ dependencies = [
 
 [[package]]
 name = "async_zip"
-version = "0.0.18"
+version = "0.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
+checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
 dependencies = [
  "async-compression",
+ "binrw",
  "crc32fast",
  "futures-lite",
  "pin-project",
@@ -869,9 +876,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-config"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
+checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -964,9 +971,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-kms"
-version = "1.115.0"
+version = "1.116.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
+checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -990,9 +997,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.142.0"
+version = "1.143.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
+checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1027,9 +1034,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.106.0"
+version = "1.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
+checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1053,9 +1060,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.108.0"
+version = "1.109.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
+checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1079,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.111.0"
+version = "1.112.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
+checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1544,6 +1551,30 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "binrw"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
+dependencies = [
+ "array-init",
+ "binrw_derive",
+ "bytemuck",
+]
+
+[[package]]
+name = "binrw_derive"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
+dependencies = [
+ "either",
+ "owo-colors",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -1631,9 +1662,9 @@ dependencies = [
 
 [[package]]
 name = "blocking"
-version = "1.6.2"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
+checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
 dependencies = [
  "async-channel",
  "async-task",
@@ -1722,6 +1753,12 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "bytemuck"
+version = "1.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -1858,9 +1895,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.4.3"
+version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
+checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -2325,9 +2362,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
 dependencies = [
  "cfg-if",
 ]
@@ -2522,12 +2559,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "cty"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
-
 [[package]]
 name = "curve25519-dalek"
 version = "4.1.3"
@@ -3940,9 +3971,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
 
 [[package]]
 name = "elliptic-curve"
@@ -4476,6 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
 dependencies = [
  "polyval",
+ "zeroize",
 ]
 
 [[package]]
@@ -5058,9 +5090,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
+checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
 dependencies = [
  "arc-swap",
  "async-channel",
@@ -5092,9 +5124,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
+checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5103,15 +5135,15 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
+checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
 
 [[package]]
 name = "hotpath-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
+checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
 dependencies = [
  "hotpath-macros-meta",
 ]
@@ -5378,9 +5410,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
 
 [[package]]
 name = "icu_provider"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
+checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -5767,9 +5799,9 @@ dependencies = [
 
 [[package]]
 name = "keccak"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
+checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
 dependencies = [
  "cfg-if",
  "cpufeatures 0.3.0",
@@ -5988,15 +6020,6 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
-[[package]]
-name = "libmimalloc-sys"
-version = "0.1.49"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
-dependencies = [
- "cc",
- "cty",
-]
-
 [[package]]
 name = "libredox"
 version = "0.1.20"
@@ -6090,9 +6113,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
 
 [[package]]
 name = "lru"
@@ -6282,7 +6305,7 @@ dependencies = [
  "hashbrown 0.16.1",
  "indexmap 2.14.0",
  "metrics",
- "ordered-float 5.3.0",
+ "ordered-float 5.5.0",
  "quanta",
  "radix_trie",
  "rand 0.9.5",
@@ -6397,14 +6420,6 @@ dependencies = [
  "synstructure 0.13.2",
 ]
 
-[[package]]
-name = "mimalloc"
-version = "0.1.52"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
-dependencies = [
- "libmimalloc-sys",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -7238,9 +7253,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "5.3.0"
+version = "5.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
 dependencies = [
  "num-traits",
 ]
@@ -7275,6 +7290,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
+[[package]]
+name = "owo-colors"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
+
 [[package]]
 name = "p12-keystore"
 version = "0.2.1"
@@ -7817,6 +7838,7 @@ dependencies = [
  "cpubits",
  "cpufeatures 0.3.0",
  "universal-hash",
+ "zeroize",
 ]
 
 [[package]]
@@ -8287,6 +8309,16 @@ name = "quick-xml"
 version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
 dependencies = [
  "encoding_rs",
  "memchr",
@@ -8951,9 +8983,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.62.7"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
+checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
 dependencies = [
  "aes 0.9.2",
  "aws-lc-rs",
@@ -9162,22 +9194,20 @@ dependencies = [
  "insta",
  "jiff",
  "libc",
- "libmimalloc-sys",
  "libsystemd",
  "matchit 0.9.2",
  "md-5 0.11.0",
  "metrics",
  "metrics-util",
- "mimalloc",
  "mime_guess",
  "opentelemetry",
  "opentelemetry_sdk",
- "p256 0.13.2",
+ "p256 0.14.0",
  "parking_lot",
  "percent-encoding",
  "pin-project-lite",
  "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rand 0.10.2",
  "rcgen",
  "regex",
@@ -9204,6 +9234,8 @@ dependencies = [
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
+ "rustfs-mimalloc",
+ "rustfs-mimalloc-sys",
  "rustfs-notify",
  "rustfs-object-capacity",
  "rustfs-object-data-cache",
@@ -9441,7 +9473,7 @@ dependencies = [
  "path-absolutize",
  "pin-project-lite",
  "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rand 0.10.2",
  "ratelimit",
  "rcgen",
@@ -9587,6 +9619,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serial_test",
+ "sha2 0.11.0",
  "temp-env",
  "tempfile",
  "thiserror 2.0.20",
@@ -9875,6 +9908,24 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "rustfs-mimalloc"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
+dependencies = [
+ "rustfs-mimalloc-sys",
+]
+
+[[package]]
+name = "rustfs-mimalloc-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "rustfs-notify"
 version = "1.0.0-rc.3"
@@ -9889,7 +9940,7 @@ dependencies = [
  "jiff",
  "metrics",
  "percent-encoding",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rayon",
  "rustc-hash",
  "rustfs-config",
@@ -10662,9 +10713,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.14"
+version = "0.103.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -10708,7 +10759,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
+source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -10735,7 +10786,7 @@ dependencies = [
  "nom 8.0.0",
  "numeric_cast",
  "pin-project-lite",
- "quick-xml",
+ "quick-xml 0.41.0",
  "regex",
  "serde",
  "serde_json",
@@ -12680,14 +12731,15 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.24.1"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
+checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
 dependencies = [
  "getrandom 0.4.3",
  "js-sys",
  "rand 0.10.2",
  "serde_core",
+ "sha1_smol",
  "wasm-bindgen",
 ]
 
@@ -13401,9 +13453,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
+checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async_zip = { default-features = false, version = "0.0.18" }
+async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.4"
-quick-xml = "0.41.0"
+quick-xml = "0.42.0"
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.0" }
+aes-gcm = { version = "=0.11.1" }
 argon2 = { version = "=0.6.0-rc.8" }
 blake2 = "=0.11.0-rc.6"
 chacha20poly1305 = { version = "=0.11.0" }
@@ -200,6 +200,7 @@ hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.0.0" }
 openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
+p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
 rustls = { default-features = false, version = "0.23.43" }
 rustls-native-certs = "0.8"
@@ -209,6 +210,7 @@ sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
 zeroize = { version = "1.9.0" }
+proptest = "1"
 
 # Time and Date
 chrono = { version = "0.4.45" }
@@ -227,11 +229,11 @@ arc-swap = "1.9.2"
 astral-tokio-tar = "0.6.4"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.10.1" }
+aws-config = { version = "1.11.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.115.0" }
-aws-sdk-s3 = { default-features = false, version = "1.142.0" }
-aws-sdk-sts = { default-features = false, version = "1.111.0" }
+aws-sdk-kms = { default-features = false, version = "1.116.0" }
+aws-sdk-s3 = { default-features = false, version = "1.143.0" }
+aws-sdk-sts = { default-features = false, version = "1.112.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
 aws-smithy-runtime-api = { version = "1.15.0" }
 aws-smithy-types = { version = "1.6.2" }
@@ -291,7 +293,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -314,7 +316,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.24.1" }
+uuid = { version = "1.25.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -343,16 +345,16 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.2" }
 rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.62.7" }
+russh = { version = "0.63.0" }
 russh-sftp = "2.4.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
-libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
-hotpath = { version = "0.23.3", default-features = false }
+rustfs-mimalloc = { version = "0.5.0" }
+rustfs-mimalloc-sys = { version = "0.5.0" }
+hotpath = { version = "0.24.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

crates/common/src/metrics.rs

@@ -901,6 +901,10 @@ pub struct Metrics {
     scanner_cycle_max_duration_millis: AtomicU64,
     scanner_cycle_max_objects: AtomicU64,
     scanner_cycle_max_directories: AtomicU64,
+    scanner_cycle_timeout_total: AtomicU64,
+    scanner_cycle_recovery_required_total: AtomicU64,
+    scanner_cycle_last_progress_age_seconds: AtomicU64,
+    scanner_leader_lease_without_progress: AtomicBool,
     scanner_bitrot_cycle_enabled: AtomicBool,
     scanner_bitrot_cycle_millis: AtomicU64,
     scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -914,7 +918,15 @@ pub struct Metrics {
     scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
     scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
     scanner_usage_last_save_unix_secs: AtomicU64,
+    scanner_usage_last_durable_success_unix_secs: AtomicU64,
+    scanner_usage_last_publication_unix_secs: AtomicU64,
+    scanner_usage_last_publication_state: Mutex<String>,
+    scanner_usage_last_publication_reason: Mutex<String>,
     scanner_usage_last_save_result: AtomicU8,
+    scanner_usage_deferred_pending: AtomicBool,
+    scanner_usage_deferred_total: AtomicU64,
+    scanner_usage_last_deferred_unix_secs: AtomicU64,
+    scanner_usage_last_deferred_reason: Mutex<String>,
     scanner_source_work: Vec<ScannerSourceWorkCounters>,
     current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
     last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1212,6 +1224,22 @@ pub struct ScannerUsageFreshnessSnapshot {
     pub last_usage_save_unix_secs: u64,
     pub last_usage_save_result: String,
     pub last_usage_save_result_code: u64,
+    #[serde(default)]
+    pub last_durable_success_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_state: String,
+    #[serde(default)]
+    pub last_publication_reason: String,
+    #[serde(default)]
+    pub deferred_pending: bool,
+    #[serde(default)]
+    pub deferred_total: u64,
+    #[serde(default)]
+    pub last_deferred_unix_secs: u64,
+    #[serde(default)]
+    pub last_deferred_reason: String,
 }
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1370,6 +1398,14 @@ pub struct ScannerMetricsReport {
     #[serde(default)]
     pub cycle_max_directories: u64,
     #[serde(default)]
+    pub cycle_timeout_total: u64,
+    #[serde(default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(default)]
+    pub leader_lease_without_progress: bool,
+    #[serde(default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(default)]
     pub bitrot_cycle_seconds: f64,
@@ -1430,6 +1466,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
+const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
+const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
 
 fn scan_cycle_result_label(result: u8) -> &'static str {
     match result {
@@ -1913,6 +1952,10 @@ impl Metrics {
             scanner_cycle_max_duration_millis: AtomicU64::new(0),
             scanner_cycle_max_objects: AtomicU64::new(0),
             scanner_cycle_max_directories: AtomicU64::new(0),
+            scanner_cycle_timeout_total: AtomicU64::new(0),
+            scanner_cycle_recovery_required_total: AtomicU64::new(0),
+            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
+            scanner_leader_lease_without_progress: AtomicBool::new(false),
             scanner_bitrot_cycle_enabled: AtomicBool::new(false),
             scanner_bitrot_cycle_millis: AtomicU64::new(0),
             scanner_checkpoint: Mutex::new(None),
@@ -1926,7 +1969,15 @@ impl Metrics {
             scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
             scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
             scanner_usage_last_save_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_state: Mutex::new(String::new()),
+            scanner_usage_last_publication_reason: Mutex::new(String::new()),
             scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
+            scanner_usage_deferred_pending: AtomicBool::new(false),
+            scanner_usage_deferred_total: AtomicU64::new(0),
+            scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_deferred_reason: Mutex::new(String::new()),
             scanner_source_work: ScannerWorkSource::all()
                 .iter()
                 .map(|_| ScannerSourceWorkCounters::default())
@@ -2251,6 +2302,44 @@ impl Metrics {
             .store(unix_now_secs(), Ordering::Relaxed);
     }
 
+    /// Record an intentional retryable usage publication deferral separately
+    /// from the last durable save result.
+    pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
+        let reason = reason.into();
+        self.record_scanner_usage_publication("deferred", reason.clone());
+        self.scanner_usage_deferred_pending.store(true, Ordering::Release);
+        self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
+        self.scanner_usage_last_deferred_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *last_reason = reason;
+    }
+
+    pub fn record_scanner_usage_durable_success(&self) {
+        self.record_scanner_usage_publication("success", "");
+        self.scanner_usage_last_durable_success_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        self.scanner_usage_deferred_pending.store(false, Ordering::Release);
+    }
+
+    pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
+        self.scanner_usage_last_publication_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_state = state.to_string();
+        let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_reason = reason.into();
+    }
+
     pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
         if let Some(counters) = self.scanner_source_work.get(source.index()) {
             counters.add(work);
@@ -2412,12 +2501,29 @@ impl Metrics {
             .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
         self.scanner_cycle_max_directories
             .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
+        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
         self.scanner_bitrot_cycle_enabled
             .store(bitrot_cycle.is_some(), Ordering::Relaxed);
         self.scanner_bitrot_cycle_millis
             .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
     }
 
+    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
+        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
+        if recovery_required {
+            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
+        }
+        self.scanner_cycle_last_progress_age_seconds
+            .store(progress_age.as_secs(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
+        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
+    }
+
     pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
         if let Some(concurrency_limit) = concurrency_limit {
             self.scanner_set_scan_concurrency_limit
@@ -3256,6 +3362,23 @@ impl Metrics {
             last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
             last_usage_save_result: usage_save_result.as_str().to_string(),
             last_usage_save_result_code: usage_save_result as u8 as u64,
+            last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
+            last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
+            last_publication_state: match self.scanner_usage_last_publication_state.lock() {
+                Ok(state) => state.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
+            deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
+            last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
+            last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
         };
         m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
         m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -3265,6 +3388,10 @@ impl Metrics {
         m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
         m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
+        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
+        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
+        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
+        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
         m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
         m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4623,6 +4750,7 @@ mod tests {
         metrics.record_scanner_dirty_usage_cycle_snapshot(1);
         metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
         metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
+        metrics.record_scanner_usage_deferred("data_movement");
 
         let report = metrics.report().await;
 
@@ -4634,6 +4762,22 @@ mod tests {
         assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
         assert_eq!(report.usage_freshness.last_usage_save_result, "success");
         assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
+        assert!(report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_deferred_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
+
+        metrics.record_scanner_usage_durable_success();
+        let report = metrics.report().await;
+        assert!(!report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_publication_state, "success");
+
+        metrics.record_scanner_usage_publication("no_update", "no_update");
+        let report = metrics.report().await;
+        assert_eq!(report.usage_freshness.last_publication_state, "no_update");
+        assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
     }
 
     #[tokio::test]
@@ -4926,4 +5070,20 @@ mod tests {
         assert!(!report.bitrot_cycle_enabled);
         assert_eq!(report.bitrot_cycle_seconds, 0.0);
     }
+
+    #[tokio::test]
+    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
+        let metrics = Metrics::new();
+        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
+        let timed_out = metrics.report().await;
+        assert_eq!(timed_out.cycle_timeout_total, 1);
+        assert_eq!(timed_out.cycle_last_progress_age, 17);
+        assert!(timed_out.leader_lease_without_progress);
+
+        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
+        let current = metrics.report().await;
+        assert_eq!(current.cycle_timeout_total, 1);
+        assert_eq!(current.cycle_last_progress_age, 0);
+        assert!(!current.leader_lease_without_progress);
+    }
 }

crates/common/src/mrf_channel.rs

@@ -23,21 +23,32 @@
 //! unconsumed intents is the consumer's job (see `rustfs-heal`
 //! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
 
+use std::collections::HashMap;
+use std::collections::hash_map::RandomState;
+use std::hash::{BuildHasher, Hash};
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::AtomicUsize;
 use std::sync::{
-    Arc, OnceLock,
+    Arc, Mutex, OnceLock,
     atomic::{AtomicBool, Ordering},
 };
+use std::time::{Duration, Instant};
 use tokio::sync::mpsc;
 use uuid::Uuid;
 
 /// Bounded capacity of the global MRF channel. Backpressure is resolved by
 /// dropping (and counting) intents, never by blocking the producer.
 const MRF_CHANNEL_CAPACITY: usize = 8192;
+const MRF_COALESCER_SHARDS: usize = 16;
+const MRF_COALESCER_MAX_KEYS: usize = 8192;
+const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
+const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
+const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
 
 /// Why an intent was produced. Drives the heal priority mapping on the
 /// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
 /// PartialWrite -> Normal).
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum MrfKind {
     /// Erasure decode failed while serving a read (read path).
     DecodeFailure,
@@ -67,12 +78,52 @@ pub struct MrfIntent {
     /// Version the intent targets, as raw UUID bytes.
     pub version_id: Option<[u8; 16]>,
     pub kind: MrfKind,
+    /// Stable erasure-set scope when the producer has it. Kept optional so
+    /// metadata corruption and legacy producers do not invent a scope.
+    pub scope: Option<MrfScope>,
+    /// Generation of the node-local ingress lease. It is not persisted in
+    /// the journal; replayed records acquire a fresh lease when re-enqueued.
+    pub lease: Option<MrfIngressLease>,
     pub enqueued_at_ms: u64,
     /// Times this intent has already been offered to the heal manager.
     /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
     pub attempts: u8,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfScope {
+    pub pool_index: u32,
+    pub set_index: u32,
+}
+
+/// Opaque generation used to release exactly the admission that created an
+/// ingress entry. A generation prevents a late terminal callback from
+/// deleting a newer retry for the same identity (ABA).
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfIngressLease(u64);
+
+impl MrfIngressLease {
+    const fn new(value: u64) -> Self {
+        Self(value)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfDropReason {
+    Disabled,
+    Uninitialized,
+    Full,
+    OversizedIdentity,
+    CoalescerFull,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfIngressResult {
+    Enqueued,
+    Coalesced,
+    Dropped(MrfDropReason),
+}
+
 /// Consumer-side retry ceiling before an intent is given up on.
 pub const MRF_MAX_ATTEMPTS: u8 = 3;
 
@@ -87,6 +138,159 @@ impl MrfIntent {
 
 static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct MrfIdentityKey {
+    kind: MrfKind,
+    bucket: Arc<str>,
+    object: Arc<str>,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+}
+
+#[derive(Debug)]
+struct IngressEntry {
+    lease: MrfIngressLease,
+    expires_at: Instant,
+    bytes: usize,
+}
+
+type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
+type MrfCoalescer = Box<[MrfCoalescerShard]>;
+
+static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
+static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
+static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
+static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
+static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
+
+fn coalescer() -> &'static [MrfCoalescerShard] {
+    MRF_COALESCER.get_or_init(|| {
+        (0..MRF_COALESCER_SHARDS)
+            .map(|_| Mutex::new(HashMap::new()))
+            .collect::<Vec<_>>()
+            .into_boxed_slice()
+    })
+}
+
+fn key_shard(key: &MrfIdentityKey) -> usize {
+    let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
+    usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
+}
+
+fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
+    version_id
+        .filter(|version| !version.is_nil())
+        .map(|version| *version.as_bytes())
+}
+
+fn canonical_identity(
+    kind: MrfKind,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+) -> (Option<[u8; 16]>, Option<MrfScope>) {
+    let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
+    match kind {
+        MrfKind::MetadataCorruption => (None, None),
+        MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
+    }
+}
+
+fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
+    64usize
+        .saturating_add(key.bucket.len())
+        .saturating_add(key.object.len())
+        .saturating_add(key.version_id.map_or(0, |_| 16))
+        .saturating_add(key.scope.map_or(0, |_| 8))
+}
+
+fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
+    let mut current = counter.load(Ordering::Relaxed);
+    loop {
+        let Some(next) = current.checked_add(amount) else {
+            return false;
+        };
+        if next > limit {
+            return false;
+        }
+        match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return true,
+            Err(observed) => current = observed,
+        }
+    }
+}
+
+fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
+    let shard = key_shard(&key);
+    let mut entries = coalescer()[shard]
+        .lock()
+        .map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
+    let now = Instant::now();
+    let before = entries.len();
+    let mut expired_bytes = 0usize;
+    entries.retain(|_, entry| {
+        if entry.expires_at > now {
+            true
+        } else {
+            expired_bytes = expired_bytes.saturating_add(entry.bytes);
+            false
+        }
+    });
+    let evicted = before.saturating_sub(entries.len());
+    if evicted > 0 {
+        MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
+        let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
+        metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
+        metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
+    }
+    if entries.contains_key(&key) {
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    let bytes = identity_estimated_bytes(&key);
+    let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
+    let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
+    if !count_reserved || !bytes_reserved {
+        if count_reserved {
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        }
+        metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
+        return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
+    }
+    let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
+    if entries
+        .insert(
+            key,
+            IngressEntry {
+                lease,
+                expires_at: now + MRF_COALESCER_TTL,
+                bytes,
+            },
+        )
+        .is_some()
+    {
+        MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    Ok(lease)
+}
+
+fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
+    let Some(lease) = lease else {
+        return;
+    };
+    if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
+        let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
+        if should_remove {
+            let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+            MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        }
+    }
+}
+
 /// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
 /// this before touching the channel so the disabled path stays allocation- and
 /// sync-free.
@@ -122,21 +326,90 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
 /// This runs on IO error paths, so it stays synchronous and cheap: one
 /// bounded allocation for the two `Arc<str>` handles plus the channel slot.
 pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
+    matches!(
+        try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
+        MrfIngressResult::Enqueued
+    )
+}
+
+/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
+/// intent already exists; it is not a second executable or durable admission.
+pub fn try_send_mrf_intent_typed(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<Uuid>,
+    scope: Option<MrfScope>,
+) -> MrfIngressResult {
     if !mrf_delivery_enabled() {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Disabled);
     }
     let Some(sender) = GLOBAL_MRF_SENDER.get() else {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
     };
-    let intent = MrfIntent {
+    if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
+        return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
+    }
+    let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
+    let key = MrfIdentityKey {
+        kind,
         bucket: Arc::from(bucket),
         object: Arc::from(object),
-        version_id: version_id.map(|vid| *vid.as_bytes()),
+        version_id,
+        scope,
+    };
+    let lease = match coalescer_admit(key.clone()) {
+        Ok(lease) => lease,
+        Err(result) => return result,
+    };
+    let intent = MrfIntent {
+        bucket: key.bucket.clone(),
+        object: key.object.clone(),
+        version_id: key.version_id,
         kind,
+        scope,
+        lease: Some(lease),
         enqueued_at_ms: unix_now_ms(),
         attempts: 0,
     };
-    sender.try_send(intent).is_ok()
+    match sender.try_send(intent) {
+        Ok(()) => MrfIngressResult::Enqueued,
+        Err(mpsc::error::TrySendError::Full(_)) => {
+            coalescer_release(&key, Some(lease));
+            metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
+            MrfIngressResult::Dropped(MrfDropReason::Full)
+        }
+        Err(mpsc::error::TrySendError::Closed(_)) => {
+            coalescer_release(&key, Some(lease));
+            MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
+        }
+    }
+}
+
+/// Release the ingress key once the consumer owns the intent.
+pub fn release_mrf_intent(intent: &MrfIntent) {
+    release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
+}
+
+pub fn release_mrf_identity(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+    lease: Option<MrfIngressLease>,
+) {
+    let (version_id, scope) = canonical_identity(kind, version_id, scope);
+    coalescer_release(
+        &MrfIdentityKey {
+            kind,
+            bucket: Arc::from(bucket),
+            object: Arc::from(object),
+            version_id,
+            scope,
+        },
+        lease,
+    );
 }
 
 fn unix_now_ms() -> u64 {
@@ -144,7 +417,8 @@ fn unix_now_ms() -> u64 {
     // failure would be a bug rather than something to handle here.
     std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
-        .map(|d| d.as_millis() as u64)
+        .ok()
+        .and_then(|d| u64::try_from(d.as_millis()).ok())
         .unwrap_or(0)
 }
 
@@ -215,12 +489,60 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0u8; 16]),
             kind: MrfKind::DecodeFailure,
+            scope: None,
+            lease: None,
             enqueued_at_ms: 0,
             attempts: 0,
         };
         assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
     }
 
+    #[test]
+    fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
+        let key = MrfIdentityKey {
+            kind: MrfKind::DecodeFailure,
+            bucket: Arc::from("ingress-test-bucket"),
+            object: Arc::from("ingress-test-object"),
+            version_id: Some([9; 16]),
+            scope: Some(MrfScope {
+                pool_index: 3,
+                set_index: 4,
+            }),
+        };
+        let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
+        for _ in 0..999 {
+            assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
+        }
+        coalescer_release(&key, Some(lease));
+        let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
+        coalescer_release(&key, Some(retry_lease));
+    }
+
+    #[test]
+    fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
+        let (nil_version, nil_scope) = canonical_identity(
+            MrfKind::DecodeFailure,
+            Some([0; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
+        assert!(nil_scope.is_some());
+
+        let (metadata_version, metadata_scope) = canonical_identity(
+            MrfKind::MetadataCorruption,
+            Some([7; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(metadata_version, None);
+        assert_eq!(metadata_scope, None);
+    }
+
     #[tokio::test]
     async fn try_send_delivers_and_respects_capacity() {
         let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -230,6 +552,7 @@ mod tests {
         let intent = receiver.recv().await.expect("intent should arrive");
         assert_eq!(intent.kind, MrfKind::DecodeFailure);
         assert_eq!(intent.bucket.as_ref(), "b");
+        release_mrf_intent(&intent);
 
         // Disable delivery: producers become no-ops.
         set_mrf_delivery_enabled(false);
@@ -239,8 +562,8 @@ mod tests {
         // Fill the bounded channel past capacity: excess intents are dropped,
         // never blocking.
         let mut accepted = 0;
-        for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
-            if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
+        for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
+            if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
                 accepted += 1;
             }
         }

crates/config/README.md

@@ -84,6 +84,12 @@ Current guidance:
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
 
+Scanner cycle budget controls:
+
+- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
+- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
+- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
+
 ## Mmap read environment aliases
 
 - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)

crates/config/src/constants/object.rs

@@ -168,6 +168,19 @@ pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
 const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
 
+/// Request writing pool metadata version 2.
+///
+/// This remains ineffective until [`ENV_POOL_META_V2_FLEET_CONFIRMED`] is also enabled.
+pub const ENV_POOL_META_V2_WRITE: &str = "RUSTFS_POOL_META_V2_WRITE";
+pub const DEFAULT_POOL_META_V2_WRITE: bool = false;
+
+/// Operator-attested confirmation that every pool metadata reader and writer understands version 2.
+pub const ENV_POOL_META_V2_FLEET_CONFIRMED: &str = "RUSTFS_POOL_META_V2_FLEET_CONFIRMED";
+pub const DEFAULT_POOL_META_V2_FLEET_CONFIRMED: bool = false;
+
+const _: () = assert!(!DEFAULT_POOL_META_V2_WRITE);
+const _: () = assert!(!DEFAULT_POOL_META_V2_FLEET_CONFIRMED);
+
 // =============================================================================
 // Concurrent Request Fix - Timeout and Backpressure Configuration
 // =============================================================================
@@ -736,4 +749,10 @@ mod remote_version_state_tests {
             "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
         );
     }
+
+    #[test]
+    fn pool_meta_v2_gate_uses_stable_environment_names() {
+        assert_eq!(super::ENV_POOL_META_V2_WRITE, "RUSTFS_POOL_META_V2_WRITE");
+        assert_eq!(super::ENV_POOL_META_V2_FLEET_CONFIRMED, "RUSTFS_POOL_META_V2_FLEET_CONFIRMED");
+    }
 }

crates/config/src/constants/runtime.rs

@@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 /// Dedicated blocking thread pool for fsync/fdatasync operations.
 /// When > 1, fsync operations are isolated from the main blocking pool to
 /// prevent device-bound fsync from starving read operations (pread/stat/open).
-/// Default 0 means auto (no isolation, use main runtime).
+/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O.
 pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
-pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
+pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64;
 
 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default

crates/config/src/constants/scanner.rs

@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";
 
-/// Default scanner cycle runtime budget.
-/// `0` keeps the existing unbounded per-cycle behavior.
-pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
+/// Default scanner cycle runtime budget when no override is configured.
+///
+/// An explicit `0` remains the compatibility escape hatch for an unbounded
+/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
+/// operation from holding the leader lease forever.
+pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
 
 /// Default scanner per-cycle object budget.
 /// `0` keeps the existing unbounded per-cycle behavior.

crates/e2e_test/README.md

@@ -72,7 +72,7 @@ The reason string on each attribute is the classifier. Current classes:
 
 - **Needs a pre-started server** — `"requires running RustFS server at
   localhost:9000"` / `"Connects to existing rustfs server"`. These are the
-  `reliant/*` and `policy/test_runner` tests; start a server first (e.g.
+  `reliant/*` tests; start a server first (e.g.
   [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh)) or use
   `--run-ignored`.
 - **Heavy / external tool** — `"Starts a rustfs server; enable when running
@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `find_available_port` | Random free port (isolation primitive) |
 | `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
-| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
+| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
 | `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
 | `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
-# Replication nightly lane; install awscurl so STS paths do not skip
+# Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
 job) to dodge this; local single-node tests already use random ports, so a
 lingering orphan is usually the cause of a spurious bind failure.
 
-**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
-visible log line (`awscurl_available()`); install `awscurl` to actually run
-them.
+**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
+spawn error. Install the pinned CI version before running their profiles.
 
 ## Related
 
@@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is:
 2. **Single-node** — spawns its own server via
    `RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
    isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
-3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
-   no fixed protocol ports. Tools that may be absent on the runner (e.g.
-   `awscurl`) are acceptable only when the test skips gracefully with a
-   visible log line (see `bucket_policy_check_test.rs`).
+3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
+   Vault, and no fixed protocol ports. Any required CLI must be pinned and
+   installed by the workflow; a missing CLI must fail the test.
 4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
    ci-13 / backlog#1148 ilm-3), not smoke candidates.
 
@@ -278,4 +276,9 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
 moving e2e tests so acceptance numbers in the test-strategy issues
 (backlog#1147–#1155) stay auditable. When a profile membership change is
 intentional, review its JSON listing before updating the matching
-`.config/e2e-*-selection.txt` test-ID digest.
+`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
+produced the listing:
+
+```bash
+python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
+```

crates/e2e_test/src/admin_auth_test.rs

@@ -33,6 +33,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
     use aws_sdk_s3::config::{Credentials, Region};
+    use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::{Client, Config};
     use http::header::HOST;
     use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -368,10 +369,15 @@ mod tests {
             reqwest::StatusCode::FORBIDDEN,
             "stale root must be rejected on the admin API after rotation, body: {body}"
         );
-        let s3_old = s3_client_with(&env, &old_ak, &old_sk).list_buckets().send().await;
-        assert!(
-            s3_old.is_err(),
-            "stale root must be rejected on the S3 plane after rotation, got: {s3_old:?}"
+        let s3_old = s3_client_with(&env, &old_ak, &old_sk)
+            .list_buckets()
+            .send()
+            .await
+            .expect_err("stale root must be rejected on the S3 plane after rotation");
+        assert_eq!(
+            s3_old.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("InvalidAccessKeyId"),
+            "stale root must receive InvalidAccessKeyId after rotation: {s3_old:?}"
         );
 
         env.stop_server();

crates/e2e_test/src/admin_iam_crud_test.rs

@@ -30,6 +30,7 @@ use crate::common::{
     RustFSTestEnvironment, admin_ok, admin_request, admin_request_with_session_token, build_test_sts_client, init_logging,
 };
 use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
 use reqwest::StatusCode;
@@ -411,8 +412,13 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
         .key("before-attach")
         .body(ByteStream::from_static(b"x"))
         .send()
-        .await;
-    assert!(denied.is_err(), "user without a policy must not be able to write to {bucket}");
+        .await
+        .expect_err("user without a policy must not be able to write to the bucket");
+    assert_eq!(
+        denied.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "user without a policy must receive AccessDenied: {denied:?}"
+    );
 
     // --- attach policy: the credential actually gains S3 access -----------------
     admin_ok(
@@ -499,13 +505,19 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
             .body(ByteStream::from_static(b"x"))
             .send()
             .await;
-        if revoked.is_err() {
-            break;
+        match revoked {
+            Ok(_) if tokio::time::Instant::now() >= deadline => {
+                return Err("deleted service account credential still works".into());
+            }
+            Ok(_) => sleep(Duration::from_millis(500)).await,
+            Err(error) => {
+                let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+                if matches!(code, Some("AccessDenied" | "InvalidAccessKeyId")) {
+                    break;
+                }
+                return Err(format!("deleted service account must fail with an authorization error, got {error:?}").into());
+            }
         }
-        if tokio::time::Instant::now() >= deadline {
-            return Err("deleted service account credential still works".into());
-        }
-        sleep(Duration::from_millis(500)).await;
     }
 
     // Disable then remove the user; the credential must stop working.
@@ -525,13 +537,19 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
             .body(ByteStream::from_static(b"x"))
             .send()
             .await;
-        if disabled.is_err() {
-            break;
+        match disabled {
+            Ok(_) if tokio::time::Instant::now() >= deadline => {
+                return Err("disabled user credential still works".into());
+            }
+            Ok(_) => sleep(Duration::from_millis(500)).await,
+            Err(error) => {
+                let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+                if matches!(code, Some("AccessDenied" | "InvalidAccessKeyId")) {
+                    break;
+                }
+                return Err(format!("disabled user must fail with an authorization error, got {error:?}").into());
+            }
         }
-        if tokio::time::Instant::now() >= deadline {
-            return Err("disabled user credential still works".into());
-        }
-        sleep(Duration::from_millis(500)).await;
     }
 
     admin_ok(

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[tokio::test]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
-        return Ok(());
-    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/checksum_upload_test.rs

@@ -199,8 +199,18 @@ mod tests {
         );
 
         // And the object must not have been stored.
-        let head = client.head_object().bucket(bucket).key(key).send().await;
-        assert!(head.is_err(), "Object must not exist after a rejected mismatched-checksum PutObject");
+        let error = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect_err("Object must not exist after a rejected mismatched-checksum PutObject");
+        assert_eq!(
+            error.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "Rejected mismatched-checksum PutObject absence probe must return HTTP 404, got {error:?}"
+        );
         info!("PASSED: PutObject rejects mismatched SHA256 and stores nothing");
     }
 
@@ -552,8 +562,18 @@ mod tests {
                 msg.contains("BadDigest") || msg.to_lowercase().contains("digest") || msg.to_lowercase().contains("checksum"),
                 "{header}: expected a BadDigest/checksum error, got: {msg}"
             );
-            let head = client.head_object().bucket(bucket).key(&bad_key).send().await;
-            assert!(head.is_err(), "{header}: nothing must be stored after a rejected PutObject");
+            let error = client
+                .head_object()
+                .bucket(bucket)
+                .key(&bad_key)
+                .send()
+                .await
+                .expect_err("nothing must be stored after a rejected PutObject");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "{header}: rejected PutObject absence probe must return HTTP 404, got {error:?}"
+            );
 
             info!("PASSED additional-checksum verify-on-write: {header}");
         }

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -24,10 +24,9 @@ use tracing::{info, warn};
 const BUCKET: &str = "conditional-put-race-bucket";
 const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
 
-async fn cleanup_object(client: &Client, key: &str) {
-    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
-        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
-    }
+async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client.delete_object().bucket(BUCKET).key(key).send().await?;
+    Ok(())
 }
 
 async fn assert_bucket_cors_missing(client: &Client) {
@@ -83,14 +82,13 @@ async fn run_race_iteration(
     test_key: &str,
     iteration: usize,
 ) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
-    cleanup_object(&clients[0], test_key).await;
+    cleanup_object(&clients[0], test_key).await?;
     tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
 
-    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
-
-    if head_result.is_ok() {
-        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
-        return Ok(0);
+    match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
+        Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
+        Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
+        Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
     }
 
     info!("\n=== Iteration {} ===", iteration);
@@ -132,14 +130,16 @@ async fn run_race_iteration(
 
     info!("Result: {} out of {} succeeded", success_count, clients.len());
 
+    if had_error {
+        return Err("one or more conditional PUTs failed unexpectedly".into());
+    }
+
     if success_count > 1 {
         info!(">>> RACE CONDITION DETECTED!");
     } else if success_count == 1 {
         info!(">>> Correct behavior: exactly 1 writer succeeded.");
-    } else if had_error {
-        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
     } else {
-        info!(">>> Unexpected: no writers succeeded.");
+        return Err("no conditional PUT succeeded".into());
     }
 
     Ok(success_count)
@@ -179,7 +179,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
             }
         }
 
-        cleanup_object(&clients[0], &test_key).await;
+        cleanup_object(&clients[0], &test_key).await?;
         tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
     }
 
@@ -189,7 +189,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
     info!("Total iterations:   {}", iterations);
     info!("Correct (1 winner): {}", correct_count);
     info!("Race conditions:    {}", races_detected);
-    info!("Errors (skipped):   {}", error_count);
+    info!("Failed iterations:  {}", error_count);
 
     assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
     assert_eq!(
@@ -197,6 +197,10 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
         "{} iteration(s) failed due to errors (e.g. cluster not ready)",
         error_count
     );
+    assert_eq!(
+        correct_count, iterations,
+        "only {correct_count}/{iterations} iterations observed exactly one winner"
+    );
 
     Ok(())
 }
@@ -213,7 +217,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
 
     let client = cluster.create_s3_client(0)?;
     let test_key = "basic-conditional-put";
-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;
 
     let result = client
         .put_object()
@@ -245,7 +249,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
         assert_eq!(code, "PreconditionFailed");
     }
 
-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;
     Ok(())
 }
 

crates/e2e_test/src/common.rs

@@ -310,6 +310,17 @@ pub fn rustfs_binary_path() -> PathBuf {
     rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
 }
 
+fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&Path>) -> PathBuf {
+    let mut path = match configured_target_dir {
+        Some(path) if path.is_absolute() => path.to_path_buf(),
+        Some(path) => workspace.join(path),
+        None => workspace.join("target"),
+    };
+    path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
+    path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
+    path
+}
+
 /// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
 pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
     if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
@@ -317,11 +328,9 @@ pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> Pat
     }
     let requested_features = requested_features.and_then(normalize_rustfs_build_features);
 
-    let mut binary_path = workspace_root();
-    binary_path.push("target");
-    let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" };
-    binary_path.push(profile_dir);
-    binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
+    let workspace = workspace_root();
+    let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
+    let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
 
     let features_match = binary_features_match(&binary_path, requested_features.as_deref());
     let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
@@ -338,7 +347,7 @@ pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> Pat
     }
 
     info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary(requested_features.as_deref());
+    build_rustfs_binary(requested_features.as_deref(), &binary_path);
 
     info!("Using RustFS binary at {:?}", binary_path);
     binary_path
@@ -440,7 +449,7 @@ fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bo
 }
 
 /// Build the RustFS binary using cargo
-fn build_rustfs_binary(requested_features: Option<&str>) {
+fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
     let workspace = workspace_root();
     info!("Building RustFS binary from workspace: {:?}", workspace);
 
@@ -476,11 +485,7 @@ fn build_rustfs_binary(requested_features: Option<&str>) {
         panic!("Failed to build RustFS binary. Error: {stderr}");
     }
 
-    let mut binary_path = workspace;
-    binary_path.push("target");
-    binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
-    binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
-    let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
     if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
         warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
     }
@@ -494,15 +499,20 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
-pub fn awscurl_available() -> bool {
-    let path = awscurl_binary_path();
-    if path.components().count() > 1 || path.is_absolute() {
-        return path.is_file();
+fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
+    let output = Command::new(path).arg("--help").output()?;
+    if output.status.success() {
+        return Ok(());
     }
 
-    std::env::var_os("PATH")
-        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
-        .unwrap_or(false)
+    Err(std::io::Error::other(format!(
+        "awscurl prerequisite check failed: {}",
+        String::from_utf8_lossy(&output.stderr).trim()
+    )))
+}
+
+pub fn require_awscurl() -> std::io::Result<()> {
+    verify_awscurl_path(&awscurl_binary_path())
 }
 
 // Global initialization
@@ -1747,6 +1757,22 @@ mod tests {
         assert_eq!(normalize_rustfs_build_features(" , "), None);
     }
 
+    #[test]
+    fn missing_awscurl_is_a_prerequisite_failure() {
+        let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
+
+        let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
+
+        assert_eq!(error.kind(), ErrorKind::NotFound);
+    }
+
+    #[test]
+    fn available_awscurl_client_passes_prerequisite_check() {
+        let executable = std::env::current_exe().expect("the test executable should have a path");
+
+        verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
+    }
+
     #[test]
     fn capture_log_path_uses_temp_directory_basename() {
         assert_eq!(
@@ -1755,6 +1781,27 @@ mod tests {
         );
     }
 
+    #[test]
+    fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
+        let workspace = Path::new("workspace");
+        let profile = if cfg!(debug_assertions) { "debug" } else { "release" };
+        let binary = format!("rustfs{}", std::env::consts::EXE_SUFFIX);
+        assert_eq!(
+            resolve_rustfs_binary_path(workspace, None),
+            workspace.join("target").join(profile).join(&binary)
+        );
+        assert_eq!(
+            resolve_rustfs_binary_path(workspace, Some(Path::new("custom-target"))),
+            workspace.join("custom-target").join(profile).join(&binary)
+        );
+
+        let absolute = std::env::temp_dir().join("rustfs-e2e-custom-target");
+        assert_eq!(
+            resolve_rustfs_binary_path(workspace, Some(&absolute)),
+            absolute.join(profile).join(binary)
+        );
+    }
+
     #[test]
     fn full_feature_enables_any_required_feature() {
         assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));

crates/e2e_test/src/compression_test.rs

@@ -4,7 +4,8 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use std::fs;
-use std::path::PathBuf;
+use std::io;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use tracing::info;
 
@@ -31,30 +32,58 @@ fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
         .collect()
 }
 
-fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
+fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> io::Result<Vec<PathBuf>> {
     let bucket_path = PathBuf::from(temp_dir).join(bucket);
     let mut part_files = Vec::new();
 
-    fn scan_dir(dir: &PathBuf, target: &str, results: &mut Vec<PathBuf>) {
-        if let Ok(entries) = fs::read_dir(dir) {
-            for entry in entries.flatten() {
-                let path = entry.path();
-                if path.is_dir() {
-                    scan_dir(&path, target, results);
-                } else if path
-                    .file_name()
-                    .map(|n| n.to_string_lossy().starts_with("part."))
-                    .unwrap_or(false)
-                    && path.to_string_lossy().contains(target)
-                {
-                    results.push(path);
+    fn scan_dir(dir: &Path, target: &str, results: &mut Vec<PathBuf>) -> io::Result<()> {
+        let entries = fs::read_dir(dir)
+            .map_err(|error| io::Error::new(error.kind(), format!("failed to read {}: {error}", dir.display())))?;
+        for entry in entries {
+            let entry = entry
+                .map_err(|error| io::Error::new(error.kind(), format!("failed to read entry in {}: {error}", dir.display())))?;
+            let path = entry.path();
+            let file_type = entry
+                .file_type()
+                .map_err(|error| io::Error::new(error.kind(), format!("failed to inspect {}: {error}", path.display())))?;
+            if file_type.is_dir() {
+                scan_dir(&path, target, results)?;
+            } else if path
+                .file_name()
+                .map(|n| n.to_string_lossy().starts_with("part."))
+                .unwrap_or(false)
+                && path.to_string_lossy().contains(target)
+            {
+                if !file_type.is_file() {
+                    return Err(io::Error::new(
+                        io::ErrorKind::InvalidData,
+                        format!("expected regular part file at {}", path.display()),
+                    ));
                 }
+                results.push(path);
             }
         }
+        Ok(())
     }
 
-    scan_dir(&bucket_path, object_key, &mut part_files);
-    part_files
+    scan_dir(&bucket_path, object_key, &mut part_files)?;
+    Ok(part_files)
+}
+
+fn part_files_total_size(part_files: &[PathBuf]) -> io::Result<u64> {
+    part_files.iter().try_fold(0_u64, |total, path| {
+        let metadata = fs::symlink_metadata(path)
+            .map_err(|error| io::Error::new(error.kind(), format!("failed to stat {}: {error}", path.display())))?;
+        if !metadata.file_type().is_file() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("expected regular part file at {}", path.display()),
+            ));
+        }
+        total
+            .checked_add(metadata.len())
+            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "on-disk part size overflow"))
+    })
 }
 
 async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -123,8 +152,9 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
     let content_length = head_response.content_length().unwrap_or(0);
     assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
 
-    let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key);
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
+    assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
+    let total_physical_size = part_files_total_size(&part_files)?;
 
     assert!(
         total_physical_size < original_size as u64,
@@ -246,9 +276,9 @@ async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error
         "Content-Length should be the logical object size"
     );
 
-    let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
+    let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key)?;
     assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let total_physical_size = part_files_total_size(&part_files)?;
     assert!(
         total_physical_size < (total_size / 2) as u64,
         "Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -366,9 +396,9 @@ async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(),
 
     // This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
     // margin still proves the parts were stored compressed rather than raw or double-encoded.
-    let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
+    let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key)?;
     assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let total_physical_size = part_files_total_size(&part_files)?;
     assert!(
         total_physical_size < (total_size as u64) / 2,
         "Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
@@ -522,9 +552,9 @@ async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), B
         "Content-Length should be the logical object size"
     );
 
-    let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
+    let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key)?;
     assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let total_physical_size = part_files_total_size(&part_files)?;
     assert!(
         total_physical_size < (total_size / 2) as u64,
         "Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
@@ -585,9 +615,9 @@ async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(),
         "Content-Length should be the logical object size"
     );
 
-    let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
+    let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key)?;
     assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let total_physical_size = part_files_total_size(&part_files)?;
     assert!(
         total_physical_size < (total_size / 2) as u64,
         "Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -734,9 +764,9 @@ async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std
         "HEAD must report SSE-S3"
     );
 
-    let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
+    let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key)?;
     assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
-    let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
+    let total_physical_size = part_files_total_size(&part_files)?;
     assert!(
         total_physical_size < (total_size / 2) as u64,
         "Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"

crates/e2e_test/src/data_usage_test.rs

@@ -59,7 +59,6 @@ where
 /// Regression test for data usage accuracy (issue #1012).
 /// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
 async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -86,28 +85,20 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
         usage
             .buckets_usage
             .get(TEST_BUCKET)
-            .map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
+            .map(|bucket_usage| usage.objects_total_count == 1000 && bucket_usage.objects_count == 1000)
             .unwrap_or(false)
     })
     .await?;
 
-    // Assert total object count and per-bucket count are not truncated
+    // Assert total object count and per-bucket count are exact.
     let bucket_usage = usage
         .buckets_usage
         .get(TEST_BUCKET)
         .cloned()
         .expect("bucket usage should exist");
 
-    assert!(
-        usage.objects_total_count >= 1000,
-        "total object count should be at least 1000, got {}",
-        usage.objects_total_count
-    );
-    assert!(
-        bucket_usage.objects_count >= 1000,
-        "bucket object count should be at least 1000, got {}",
-        bucket_usage.objects_count
-    );
+    assert_eq!(usage.objects_total_count, 1000, "total object count should be exact");
+    assert_eq!(bucket_usage.objects_count, 1000, "bucket object count should be exact");
 
     env.stop_server();
     Ok(())
@@ -116,7 +107,6 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
 /// Regression test for issue #3898.
 /// Versioned buckets should expose versions and delete markers through admin data usage.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
 async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 

crates/e2e_test/src/delete_regression_test.rs

@@ -117,9 +117,18 @@ mod tests {
         );
 
         // Verify HEAD returns 404
-        let head = client.head_object().bucket(bucket).key("to-delete.txt").send().await;
-
-        assert!(head.is_err(), "RT-05 FAIL: HEAD on deleted object should return error, got success");
+        let error = client
+            .head_object()
+            .bucket(bucket)
+            .key("to-delete.txt")
+            .send()
+            .await
+            .expect_err("RT-05 FAIL: HEAD on deleted object should return 404, got success");
+        assert_eq!(
+            error.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "RT-05 FAIL: HEAD on deleted object must return HTTP 404, got {error:?}"
+        );
 
         info!("RT-05 PASS: delete correctly removes object from LIST and HEAD");
         Ok(())
@@ -414,9 +423,18 @@ mod tests {
 
         // All HEAD requests should return 404
         for key in &keys {
-            let head = client.head_object().bucket(bucket).key(*key).send().await;
-
-            assert!(head.is_err(), "RT-05f FAIL: HEAD on deleted key '{key}' should return error");
+            let error = client
+                .head_object()
+                .bucket(bucket)
+                .key(*key)
+                .send()
+                .await
+                .expect_err("RT-05f FAIL: HEAD on deleted key should return 404, got success");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "RT-05f FAIL: HEAD on deleted key '{key}' must return HTTP 404, got {error:?}"
+            );
         }
 
         // LIST should be empty

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -16,10 +16,9 @@
 //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
 //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
 use aws_sdk_s3::{Client, Config};
@@ -175,11 +174,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
 #[tokio::test]
 async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2eiamtag-{suffix}");
     let user_secret = "longSecretKeyForTest123!";
@@ -215,10 +209,17 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
     let _ = out.body.collect().await?;
 
     put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
-    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
-    assert!(
-        denied.is_err(),
-        "GetObject must be denied when ExistingObjectTag no longer matches IAM policy"
+    let denied = uclient
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("GetObject must be denied when ExistingObjectTag no longer matches IAM policy");
+    assert_eq!(
+        denied.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "IAM ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
     );
 
     cleanup_bucket_and_object(&admin, &bucket, key).await;
@@ -233,11 +234,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
 #[tokio::test]
 async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2ebptag-{suffix}");
     let user_secret = "longSecretKeyForTest456!";
@@ -257,8 +253,13 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
         .bucket(&bucket)
         .key(key)
         .send()
-        .await;
-    assert!(deny_before.is_err(), "without bucket policy, user must be denied");
+        .await
+        .expect_err("without bucket policy, user must be denied");
+    assert_eq!(
+        deny_before.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "missing bucket policy must return AccessDenied: {deny_before:?}"
+    );
 
     let bp = serde_json::json!({
         "Version": "2012-10-17",
@@ -280,8 +281,18 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
     let _ = ok.body.collect().await?;
 
     put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
-    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
-    assert!(denied.is_err(), "GetObject must fail when tag no longer satisfies bucket policy");
+    let denied = uclient
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("GetObject must fail when tag no longer satisfies bucket policy");
+    assert_eq!(
+        denied.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "bucket-policy ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
+    );
 
     cleanup_bucket_and_object(&admin, &bucket, key).await;
     admin_remove_user(&env, &user).await;
@@ -294,11 +305,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
 #[tokio::test]
 async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-par-{suffix}");
     let parent_secret = "longSecretKeyForParentSts99!";
@@ -352,10 +358,17 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
     let _ = ok.body.collect().await?;
 
     put_object_tag_kv(&parent_client, &bucket, key, "security", "private").await?;
-    let denied = session_client.get_object().bucket(&bucket).key(key).send().await;
-    assert!(
-        denied.is_err(),
-        "session policy must deny GetObject when ExistingObjectTag no longer matches"
+    let denied = session_client
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("session policy must deny GetObject when ExistingObjectTag no longer matches");
+    assert_eq!(
+        denied.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "STS ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
     );
 
     cleanup_bucket_and_object(&admin, &bucket, key).await;
@@ -370,11 +383,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
 #[tokio::test]
 async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-del-par-{suffix}");
     let parent_secret = "longSecretKeyForParentDelete99!";

crates/e2e_test/src/group_delete_test.rs

@@ -14,7 +14,7 @@
 
 //! E2E tests for group management (fixes #2028).
 
-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
 
 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
     env.start_rustfs_server(vec![]).await?;
 
     // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
     let user_body = serde_json::json!({
         "secretKey": "testuser1secret",
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created testuser1");
 
     // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": "testgroup",
         "members": ["testuser1"],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added testuser1 to testgroup");
 
     // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
     info!("Delete of non-empty group correctly rejected");
 
     // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
         "isRemove": true,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
     info!("Removed testuser1 from testgroup");
 
     // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
     info!("Deleted empty testgroup successfully");
 
     // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
     info!("Confirmed testgroup no longer exists");
 
     Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
         "Statement": [{
             "Effect": "Allow",
             "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
         }]
     });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
     info!("Created canned policy {}", policy_name);
 
     // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {} with no explicit policy", user_name);
 
     // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
     info!("Attached policy {} to group {}", policy_name, group_name);
 
     // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
     let group_name = "soledeletegroup";
 
     // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {}", user_name);
 
     // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
     info!("Deleted user {}", user_name);
 
     // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
     info!("Deleted group {} after user was removed", group_name);
 
     Ok(())

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -380,10 +380,24 @@ mod tests {
         cluster.start_node(1).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-        assert!(
-            !status_body.contains("MissingContentLength"),
-            "background heal status should not fail without an explicit Content-Length: {status_body}"
+        let mut recovered = serde_json::Value::Null;
+        for _ in 0..60 {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            recovered = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
+                break;
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        assert_eq!(
+            recovered["clusterStatusComplete"],
+            serde_json::Value::Bool(true),
+            "cluster heal status should recover before root heal starts: {recovered}"
         );
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;

crates/e2e_test/src/kms/bucket_default_encryption_test.rs

@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/common.rs

@@ -22,9 +22,7 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
@@ -59,15 +57,6 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
-pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
-    if awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping {} because awscurl is not available in PATH", test_name);
-    true
-}
-
 pub fn sse_customer_key_md5_base64(key: &str) -> String {
     let mut hasher = Md5::new();
     hasher.update(key.as_bytes());
@@ -189,34 +178,121 @@ pub async fn wait_for_kms_ready(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
+    wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
+}
+
+async fn wait_for_kms_ready_with_timeout(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    total_deadline: Duration,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let start = tokio::time::Instant::now();
+    let deadline = start + total_deadline;
     let mut backoff = Duration::from_millis(200);
     let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;
 
     loop {
-        if !first_attempt {
-            if start.elapsed() >= total_deadline {
-                return Err("KMS failed to become ready within 5 seconds".into());
-            }
-            sleep(backoff).await;
-            backoff = (backoff * 2).min(max_backoff);
-        }
-        first_attempt = false;
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
+        match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
+            Ok(Ok(status)) => {
+                let backend_status = serde_json::from_str::<serde_json::Value>(&status)
+                    .ok()
+                    .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
+                if backend_status.as_deref() == Some("healthy") {
+                    info!("KMS is ready (status: {})", status);
+                    return Ok(());
                 }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
+                warn!(
+                    backend_status = backend_status.as_deref().unwrap_or("missing"),
+                    elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
+                    "KMS not ready yet, retrying…"
+                );
             }
+            Ok(Err(e)) => {
+                let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+                warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
+            }
+            Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
         }
+
+        let now = tokio::time::Instant::now();
+        if now >= deadline {
+            return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
+        }
+        sleep((now + backoff).min(deadline) - now).await;
+        backoff = (backoff * 2).min(max_backoff);
+    }
+}
+
+#[cfg(test)]
+mod readiness_tests {
+    use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    #[tokio::test]
+    async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let requests = Arc::new(AtomicUsize::new(0));
+        let server_requests = Arc::clone(&requests);
+        let server = tokio::spawn(async move {
+            for backend_status in ["error", "healthy"] {
+                let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+                let mut request = Vec::new();
+                let mut chunk = [0_u8; 1024];
+                while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    let read = socket.read(&mut chunk).await.expect("read readiness request");
+                    if read == 0 {
+                        break;
+                    }
+                    request.extend_from_slice(&chunk[..read]);
+                }
+                server_requests.fetch_add(1, Ordering::SeqCst);
+
+                let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                socket.write_all(response.as_bytes()).await.expect("write readiness response");
+            }
+        });
+
+        wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
+            .await
+            .expect("KMS should become ready after the healthy response");
+
+        let observed_requests = requests.load(Ordering::SeqCst);
+        server.abort();
+        assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
+    }
+
+    #[tokio::test]
+    async fn kms_readiness_deadline_covers_a_stalled_status_request() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+            let mut request = [0_u8; 1024];
+            let _ = socket.read(&mut request).await.expect("read readiness request");
+            std::future::pending::<()>().await;
+        });
+
+        let result = tokio::time::timeout(
+            Duration::from_secs(1),
+            wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
+        )
+        .await
+        .expect("readiness helper must enforce its own deadline");
+
+        server.abort();
+        assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
     }
 }
 
@@ -403,10 +479,6 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
-        return Ok(());
-    }
-
     info!("Testing KMS key management APIs");
 
     // Test CreateKey

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -432,7 +432,6 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
 }
 
 #[tokio::test]
-#[ignore = "requires a Vault binary"]
 async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
     let mut env = VaultTestEnvironment::new().await?;
     env.start_vault().await?;

crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs

@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     let bucket = "copy-object-self-copy-bucket-default-sse-test";

crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs

@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     let bucket = "copy-object-version-restore-sse-test";

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -24,7 +24,6 @@ use super::common::{
     test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::info;
 
 /// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = Arc::new(kms_env.base_env.create_s3_client());
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_fault_recovery_test.rs

@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -393,15 +393,14 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
     Ok(())
 }
 
-/// Test KMS resilience to temporary resource constraints
+/// Test concurrent KMS encryption requests
 #[tokio::test]
-async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+async fn test_kms_concurrent_encryption_requests() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    info!("🧪 Testing KMS behavior under resource constraints");
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -431,29 +430,27 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
     }
 
     // Wait for all uploads to complete
-    let mut successful_uploads = 0;
-    let mut failed_uploads = 0;
+    let mut failures = Vec::new();
 
     for task in upload_tasks {
-        let (object_key, result) = task.await.unwrap();
+        let (object_key, result) = task.await?;
         match result {
             Ok(_) => {
-                successful_uploads += 1;
                 info!("✅ Rapid upload {} succeeded", object_key);
             }
             Err(e) => {
-                failed_uploads += 1;
                 warn!("❌ Rapid upload {} failed: {}", object_key, e);
+                failures.push(format!("{object_key}: {e}"));
             }
         }
     }
 
-    info!("📊 Rapid upload results: {} succeeded, {} failed", successful_uploads, failed_uploads);
-
-    // We expect most uploads to succeed even under load
-    assert!(successful_uploads >= 7, "Expected at least 7/10 rapid uploads to succeed");
+    assert!(
+        failures.is_empty(),
+        "all 10 concurrent KMS uploads must succeed; failures: {}",
+        failures.join("; ")
+    );
 
     kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ Resource constraints test completed successfully");
     Ok(())
 }

crates/e2e_test/src/kms/kms_local_test.rs

@@ -20,8 +20,7 @@
 //! - Complete encryption/decryption lifecycle
 
 use super::common::{
-    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    test_kms_key_management, test_sse_c_encryption,
+    LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
@@ -29,9 +28,6 @@ use tracing::{error, info};
 #[tokio::test]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment
@@ -46,7 +42,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -127,7 +123,7 @@ async fn test_local_kms_key_isolation() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -227,7 +223,7 @@ async fn test_local_kms_large_file() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -309,7 +305,7 @@ async fn test_local_kms_multipart_upload() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait for KMS initialization
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,12 +19,11 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
+    test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
     test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
@@ -45,8 +44,8 @@ impl VaultKmsTestContext {
 
         start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
-        // Allow Vault to finish initialising token auth and transit engine.
-        sleep(Duration::from_secs(2)).await;
+        // Wait for KMS to finish initialising.
+        super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
         Ok(Self { env })
     }
@@ -63,9 +62,6 @@ impl VaultKmsTestContext {
 #[tokio::test]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -118,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
-        return Ok(());
-    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -204,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[tokio::test]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
-        return Ok(());
-    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -268,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
-        return Ok(());
-    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -298,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[tokio::test]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
-        return Ok(());
-    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/kms/mod.rs

@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
 #[cfg(test)]
 mod kms_fault_recovery_test;
 
-#[cfg(test)]
-mod test_runner;
-
 #[cfg(test)]
 mod bucket_default_encryption_test;
 

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/test_runner.rs

@@ -1,499 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-#![allow(dead_code)]
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Unified KMS test suite runner
-//!
-//! This module provides a unified interface for running KMS tests with categorization,
-//! filtering, and comprehensive reporting capabilities.
-
-use crate::common::init_logging;
-use std::time::Instant;
-use tokio::time::{Duration, sleep};
-use tracing::{debug, error, info, warn};
-
-/// Test category for organization and filtering
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TestCategory {
-    CoreFunctionality,
-    MultipartEncryption,
-    EdgeCases,
-    FaultRecovery,
-    Comprehensive,
-    Performance,
-}
-
-impl TestCategory {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            TestCategory::CoreFunctionality => "core-functionality",
-            TestCategory::MultipartEncryption => "multipart-encryption",
-            TestCategory::EdgeCases => "edge-cases",
-            TestCategory::FaultRecovery => "fault-recovery",
-            TestCategory::Comprehensive => "comprehensive",
-            TestCategory::Performance => "performance",
-        }
-    }
-}
-
-/// Test definition with metadata
-#[derive(Debug, Clone)]
-pub struct TestDefinition {
-    pub name: String,
-    pub description: String,
-    pub category: TestCategory,
-    pub estimated_duration: Duration,
-    pub is_critical: bool,
-}
-
-impl TestDefinition {
-    pub fn new(
-        name: impl Into<String>,
-        description: impl Into<String>,
-        category: TestCategory,
-        estimated_duration: Duration,
-        is_critical: bool,
-    ) -> Self {
-        Self {
-            name: name.into(),
-            description: description.into(),
-            category,
-            estimated_duration,
-            is_critical,
-        }
-    }
-}
-
-/// Test execution result
-#[derive(Debug, Clone)]
-pub struct TestResult {
-    pub test_name: String,
-    pub category: TestCategory,
-    pub success: bool,
-    pub duration: Duration,
-    pub error_message: Option<String>,
-}
-
-impl TestResult {
-    pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
-        Self {
-            test_name,
-            category,
-            success: true,
-            duration,
-            error_message: None,
-        }
-    }
-
-    pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
-        Self {
-            test_name,
-            category,
-            success: false,
-            duration,
-            error_message: Some(error),
-        }
-    }
-}
-
-/// Comprehensive test suite configuration
-#[derive(Debug, Clone)]
-pub struct TestSuiteConfig {
-    pub categories: Vec<TestCategory>,
-    pub include_critical_only: bool,
-    pub max_duration: Option<Duration>,
-    pub parallel_execution: bool,
-}
-
-impl Default for TestSuiteConfig {
-    fn default() -> Self {
-        Self {
-            categories: vec![
-                TestCategory::CoreFunctionality,
-                TestCategory::MultipartEncryption,
-                TestCategory::EdgeCases,
-                TestCategory::FaultRecovery,
-                TestCategory::Comprehensive,
-            ],
-            include_critical_only: false,
-            max_duration: None,
-            parallel_execution: false,
-        }
-    }
-}
-
-/// Unified KMS test suite runner
-pub struct KMSTestSuite {
-    tests: Vec<TestDefinition>,
-    config: TestSuiteConfig,
-}
-
-impl KMSTestSuite {
-    /// Create a new test suite with default configuration
-    pub fn new() -> Self {
-        let tests = vec![
-            // Core Functionality Tests
-            TestDefinition::new(
-                "test_local_kms_end_to_end",
-                "End-to-end KMS test with all encryption types",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_local_kms_key_isolation",
-                "Test KMS key isolation and security",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(45),
-                true,
-            ),
-            // Multipart Encryption Tests
-            TestDefinition::new(
-                "test_local_kms_multipart_upload",
-                "Test large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step1_basic_single_file_encryption",
-                "Basic single file encryption test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step2_basic_multipart_upload_without_encryption",
-                "Basic multipart upload without encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step3_multipart_upload_with_sse_s3",
-                "Multipart upload with SSE-S3 encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step4_large_multipart_upload_with_encryption",
-                "Large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(90),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step5_all_encryption_types_multipart",
-                "All encryption types multipart test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            // Edge Cases Tests
-            TestDefinition::new(
-                "test_kms_zero_byte_file_encryption",
-                "Test encryption of zero-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_single_byte_file_encryption",
-                "Test encryption of single-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_boundary_conditions",
-                "Test multipart upload boundary conditions",
-                TestCategory::EdgeCases,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_invalid_key_scenarios",
-                "Test invalid key scenarios",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_concurrent_encryption",
-                "Test concurrent encryption operations",
-                TestCategory::EdgeCases,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_key_validation_security",
-                "Test key validation security",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            // Fault Recovery Tests
-            TestDefinition::new(
-                "test_kms_key_directory_unavailable",
-                "Test KMS when key directory is unavailable",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_corrupted_key_files",
-                "Test KMS with corrupted key files",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_upload_interruption",
-                "Test multipart upload interruption recovery",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_resource_constraints",
-                "Test KMS under resource constraints",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(90),
-                false,
-            ),
-            // Comprehensive Tests
-            TestDefinition::new(
-                "test_comprehensive_kms_full_workflow",
-                "Full KMS workflow comprehensive test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(300),
-                true,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_stress_test",
-                "KMS stress test with large datasets",
-                TestCategory::Comprehensive,
-                Duration::from_secs(400),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_key_isolation",
-                "Comprehensive key isolation test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(180),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_concurrent_operations",
-                "Comprehensive concurrent operations test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(240),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_performance_benchmark",
-                "KMS performance benchmark test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(360),
-                false,
-            ),
-        ];
-
-        Self {
-            tests,
-            config: TestSuiteConfig::default(),
-        }
-    }
-
-    /// Configure the test suite
-    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
-        self.config = config;
-        self
-    }
-
-    /// Filter tests based on category
-    pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| &test.category == category).collect()
-    }
-
-    /// Filter tests based on criticality
-    pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| test.is_critical).collect()
-    }
-
-    /// Get test summary by category
-    pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
-        let mut summary = std::collections::HashMap::new();
-        for test in &self.tests {
-            summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
-        }
-        summary
-    }
-
-    /// Run the complete test suite
-    pub async fn run_test_suite(&self) -> Vec<TestResult> {
-        init_logging();
-        info!("🚀 Starting unified KMS test suite");
-
-        let start_time = Instant::now();
-        let mut results = Vec::new();
-
-        // Filter tests based on configuration
-        let tests_to_run: Vec<&TestDefinition> = self
-            .tests
-            .iter()
-            .filter(|test| self.config.categories.contains(&test.category))
-            .filter(|test| !self.config.include_critical_only || test.is_critical)
-            .collect();
-
-        info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
-        for (i, test) in tests_to_run.iter().enumerate() {
-            info!("  {}. {} ({})", i + 1, test.name, test.category.as_str());
-        }
-
-        // Execute tests
-        for (i, test_def) in tests_to_run.iter().enumerate() {
-            info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
-            info!("   📝 Description: {}", test_def.description);
-            info!("   🏷️  Category: {}", test_def.category.as_str());
-            info!("   ⏱️  Estimated duration: {:?}", test_def.estimated_duration);
-
-            let test_start = Instant::now();
-            let result = self.run_single_test(test_def).await;
-            let test_duration = test_start.elapsed();
-
-            match result {
-                Ok(_) => {
-                    info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
-                    results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
-                }
-                Err(e) => {
-                    error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
-                    results.push(TestResult::failure(
-                        test_def.name.clone(),
-                        test_def.category.clone(),
-                        test_duration,
-                        e.to_string(),
-                    ));
-                }
-            }
-
-            // Add delay between tests to avoid resource conflicts
-            if i < tests_to_run.len() - 1 {
-                debug!("⏸️  Waiting two seconds before the next test...");
-                sleep(Duration::from_secs(2)).await;
-            }
-        }
-
-        let total_duration = start_time.elapsed();
-        self.print_test_summary(&results, total_duration);
-
-        results
-    }
-
-    /// Run a single test by dispatching to the appropriate test function
-    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        // This is a placeholder for test dispatch logic
-        // In a real implementation, this would dispatch to actual test functions
-        warn!("⚠️  Test '{}' is not implemented in the unified runner; skipping", test_def.name);
-        Ok(())
-    }
-
-    /// Print comprehensive test summary
-    fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
-        info!("📊 KMS test suite summary");
-        info!("⏱️  Total duration: {:.2} seconds", total_duration.as_secs_f64());
-        info!("📈 Total tests: {}", results.len());
-
-        let passed = results.iter().filter(|r| r.success).count();
-        let failed = results.iter().filter(|r| !r.success).count();
-
-        info!("✅ Passed: {}", passed);
-        info!("❌ Failed: {}", failed);
-        info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
-
-        // Summary by category
-        let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
-        for result in results {
-            let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
-            *total += 1;
-            if result.success {
-                *passed_count += 1;
-            }
-        }
-
-        info!("📊 Category summary:");
-        for (category, (total, passed_count)) in category_summary {
-            info!(
-                "  🏷️  {}: {}/{} ({:.1}%)",
-                category.as_str(),
-                passed_count,
-                total,
-                (passed_count as f64 / total as f64) * 100.0
-            );
-        }
-
-        // List failed tests
-        if failed > 0 {
-            warn!("❌ Failing tests:");
-            for result in results.iter().filter(|r| !r.success) {
-                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
-            }
-        }
-    }
-}
-
-/// Quick test suite for critical tests only
-#[tokio::test]
-async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let config = TestSuiteConfig {
-        categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
-        include_critical_only: true,
-        max_duration: Some(Duration::from_secs(600)), // 10 minutes max
-        parallel_execution: false,
-    };
-
-    let suite = KMSTestSuite::new().with_config(config);
-    let results = suite.run_test_suite().await;
-
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
-    }
-
-    info!("✅ All critical tests passed");
-    Ok(())
-}
-
-/// Full comprehensive test suite
-#[tokio::test]
-async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let suite = KMSTestSuite::new();
-    let results = suite.run_test_suite().await;
-
-    let total_tests = results.len();
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
-
-    info!("📊 Full suite success rate: {:.1}%", success_rate);
-
-    // Allow up to 10% failure rate for non-critical tests
-    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
-    }
-
-    info!("✅ Full test suite succeeded");
-    Ok(())
-}

crates/e2e_test/src/leading_slash_key_test.rs

@@ -131,8 +131,18 @@ mod tests {
 
         // DELETE through the raw key removes the normalized object.
         client.delete_object().bucket(bucket).key("//keyname").send().await?;
-        let result = client.get_object().bucket(bucket).key("keyname").send().await;
-        assert!(result.is_err(), "object must be gone after DELETE with raw key");
+        let error = client
+            .get_object()
+            .bucket(bucket)
+            .key("keyname")
+            .send()
+            .await
+            .expect_err("object must be gone after DELETE with raw key");
+        assert_eq!(
+            error.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "GET after DELETE with raw key must return HTTP 404, got {error:?}"
+        );
 
         env.stop_server();
         info!("Test completed successfully");

crates/e2e_test/src/lib.rs

@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
+// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
+// replication metadata must not overwrite a newer local category state.
+#[cfg(test)]
+mod replication_lww_receiver_test;
+
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;

crates/e2e_test/src/mc_mirror_small_bucket_test.rs

@@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
     Ok(())
 }
 
-fn mc_available() -> bool {
-    Command::new("mc")
-        .arg("--version")
-        .output()
-        .is_ok_and(|output| output.status.success())
-}
-
 fn run_mc(args: &[&str]) -> TestResult {
     let output = Command::new("mc").args(args).output()?;
     if !output.status.success() {
@@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize {
 async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
     crate::common::init_logging();
     info!("Starting issue #3107 mc mirror regression test");
-    if !mc_available() {
-        info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
-        return Ok(());
-    }
+    run_mc(&["--version"])?;
 
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;

crates/e2e_test/src/multipart_auth_test.rs

@@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
 async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        return Ok(());
-    }
-
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/e2e_test/src/object_lambda_test.rs

@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
 use rustfs_signer::pre_sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let bucket = "object-lambda-e2e-disabled-target";
     let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
         .send()
         .await?;
 
-    configure_webhook_target_with_key_values(
-        &env,
-        "transformer",
-        vec![
-            ("endpoint", "http://127.0.0.1:9/transform".to_string()),
-            ("auth_token", "secret-token".to_string()),
-            ("enable", "off".to_string()),
-        ],
+    let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
+    tokio::fs::create_dir_all(&queue_dir).await?;
+    let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
+    let directive = format!(
+        "notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
+    );
+    let disable_response = signed_request(
+        http::Method::PUT,
+        &config_url,
+        &env.access_key,
+        &env.secret_key,
+        Some(directive.into_bytes()),
+        Some("text/plain"),
     )
     .await?;
-    wait_for_target_visibility(&env, "transformer").await?;
+    let disable_status = disable_response.status();
+    let disable_body = disable_response.text().await?;
+    assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
 
     let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
     let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let bucket = "object-lambda-e2e-invalid-endpoint";
 
@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let response = send_configure_webhook_target_request(
         &env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
     init_logging();
 
     let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.set_env(ENV_NOTIFY_ENABLE, "true");
+    cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
     cluster.start().await?;
 
     let bucket = "listen-notification-cluster";

crates/e2e_test/src/policy/README.md

@@ -11,29 +11,20 @@ The tests cover the following AWS policy variable scenarios:
 3. **Variable concatenation** - Combining variables with static text like `prefix-${aws:username}-suffix`
 4. **Nested variables** - Complex nested variable patterns like `${${aws:username}-test}`
 5. **Deny scenarios** - Testing deny policies with variables
+6. **STS credentials** - Variable resolution inherited by temporary credentials
 
 ## Prerequisites
 
-- RustFS server binary
 - `awscurl` utility for admin API calls
 - AWS SDK for Rust (included in the project)
 
 ## Running Tests
 
-### Run All Policy Tests Using Unified Test Runner
-
-```bash
-# Run all policy tests with comprehensive reporting
-# Note: Requires a RustFS server running on localhost:9000
-cargo test -p e2e_test policy::test_runner::test_policy_full_suite -- --nocapture --ignored  --test-threads=1
-
-# Run only critical policy tests
-cargo test -p e2e_test policy::test_runner::test_policy_critical_suite -- --nocapture --ignored --test-threads=1
-```
-
 ### Run All Policy Tests
 
 ```bash
 # From the project root directory
-cargo test -p e2e_test policy:: -- --nocapture --ignored --test-threads=1
-```
+cargo test -p e2e_test policy:: -- --nocapture
+```
+
+Each test starts an isolated RustFS server on a dynamically allocated local port and cleans it up afterward.

crates/e2e_test/src/policy/mod.rs

@@ -18,5 +18,3 @@
 //! including single-value, multi-value, and nested variable scenarios.
 
 mod policy_variables_test;
-mod test_env;
-mod test_runner;

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -14,14 +14,17 @@
 
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
-use crate::common::{awscurl_delete, awscurl_put, init_logging};
-use crate::policy::test_env::PolicyTestEnvironment;
+use crate::common::{
+    RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use tracing::info;
 
 /// Helper function to create a regular user with given credentials
 async fn create_user(
-    env: &PolicyTestEnvironment,
+    env: &RustFSTestEnvironment,
     username: &str,
     password: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -36,20 +39,9 @@ async fn create_user(
     Ok(())
 }
 
-/// Helper function to create an STS user with given credentials
-async fn create_sts_user(
-    env: &PolicyTestEnvironment,
-    username: &str,
-    password: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    // For STS, we create a regular user first, then use it to assume roles
-    create_user(env, username, password).await?;
-    Ok(())
-}
-
 /// Helper function to create and attach a policy
 async fn create_and_attach_policy(
-    env: &PolicyTestEnvironment,
+    env: &RustFSTestEnvironment,
     policy_name: &str,
     username: &str,
     policy_document: serde_json::Value,
@@ -70,9 +62,9 @@ async fn create_and_attach_policy(
 }
 
 /// Helper function to clean up test resources
-async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, policy_name: &str) {
+async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, policy_name: &str) {
     // Create admin client for cleanup
-    let admin_client = env.create_s3_client(&env.access_key, &env.secret_key);
+    let admin_client = env.create_s3_client();
 
     // Delete buckets that might have been created by this user
     let bucket_patterns = [
@@ -84,7 +76,7 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
         format!("{username}-test"),
         format!("{username}-sts-bucket"),
         format!("{username}-service-bucket"),
-        "private-test-bucket".to_string(), // For deny test
+        format!("{username}-private-bucket"),
     ];
 
     // Try to delete objects and buckets
@@ -121,24 +113,18 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
 
 /// Test AWS policy variables with single-value scenarios
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_single_value_impl().await
-}
-
-/// Implementation function for single-value policy variables test
-pub async fn test_aws_policy_variables_single_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables single-value test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_single_value_impl_with_env(&env).await
 }
 
-/// Implementation function for single-value policy variables test with shared environment
-pub async fn test_aws_policy_variables_single_value_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_single_value_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser1";
@@ -198,9 +184,7 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
     awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
-
-    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
 
     // Test 1: User should be able to list buckets (allowed by policy)
     info!("Test 1: User listing buckets");
@@ -257,11 +241,13 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
     // Test 6: User should NOT be able to create bucket NOT matching username pattern
     info!("Test 6: User attempting to create bucket NOT matching pattern");
     let other_bucket_name = "other-user-bucket";
-    let create_other_result = test_client.create_bucket().bucket(other_bucket_name).send().await;
-    if create_other_result.is_ok() {
-        cleanup().await;
-        return Err("User should NOT be able to create bucket NOT matching username pattern".into());
-    }
+    let denied = test_client
+        .create_bucket()
+        .bucket(other_bucket_name)
+        .send()
+        .await
+        .expect_err("a bucket outside the username pattern must be denied");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
 
     // Cleanup
     info!("Cleaning up test resources");
@@ -273,24 +259,18 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
 
 /// Test AWS policy variables with multi-value scenarios
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_multi_value_impl().await
-}
-
-/// Implementation function for multi-value policy variables test
-pub async fn test_aws_policy_variables_multi_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables multi-value test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_multi_value_impl_with_env(&env).await
 }
 
-/// Implementation function for multi-value policy variables test with shared environment
-pub async fn test_aws_policy_variables_multi_value_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_multi_value_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser2";
@@ -338,7 +318,7 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
+    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
 
     // Test 1: User should be able to create buckets matching any of the multi-value patterns
     info!("Test 1: User creating first bucket matching multi-value pattern");
@@ -368,11 +348,13 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
     // Test 4: User should NOT be able to create bucket NOT matching any multi-value pattern
     info!("Test 4: User attempting to create bucket NOT matching any pattern");
     let other_bucket_name = format!("{test_user}-other-bucket");
-    let create_other_result = test_client.create_bucket().bucket(&other_bucket_name).send().await;
-    if create_other_result.is_ok() {
-        cleanup().await;
-        return Err("User should NOT be able to create bucket NOT matching any multi-value pattern".into());
-    }
+    let denied = test_client
+        .create_bucket()
+        .bucket(&other_bucket_name)
+        .send()
+        .await
+        .expect_err("a bucket outside all allowed patterns must be denied");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
 
     // Test 5: User should be able to list objects in their allowed buckets
     info!("Test 5: User listing objects in allowed buckets");
@@ -398,24 +380,18 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
 
 /// Test AWS policy variables with variable concatenation
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_concatenation_impl().await
-}
-
-/// Implementation function for concatenation policy variables test
-pub async fn test_aws_policy_variables_concatenation_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables concatenation test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_concatenation_impl_with_env(&env).await
 }
 
-/// Implementation function for concatenation policy variables test with shared environment
-pub async fn test_aws_policy_variables_concatenation_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_concatenation_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser3";
@@ -455,10 +431,7 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
-
-    // Add a small delay to allow policy to propagate
-    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
 
     // Test: User should be able to create bucket matching concatenated pattern
     info!("Test: User creating bucket matching concatenated pattern");
@@ -487,41 +460,30 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
 
 /// Test AWS policy variables with nested scenarios
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_nested_impl().await
-}
-
-/// Implementation function for nested policy variables test
-pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables nested test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_nested_impl_with_env(&env).await
 }
 
 /// Test AWS policy variables with STS temporary credentials
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_sts_impl().await
-}
-
-/// Implementation function for STS policy variables test
-pub async fn test_aws_policy_variables_sts_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables STS test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_sts_impl_with_env(&env).await
 }
 
-/// Implementation function for nested policy variables test with shared environment
-pub async fn test_aws_policy_variables_nested_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_nested_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser4";
@@ -561,10 +523,7 @@ pub async fn test_aws_policy_variables_nested_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
-
-    // Add a small delay to allow policy to propagate
-    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
 
     // Test nested variable resolution
     info!("Test: Nested variable resolution");
@@ -581,14 +540,14 @@ pub async fn test_aws_policy_variables_nested_impl_with_env(
         return Err(format!("User should be able to create bucket with nested variable: {e}").into());
     }
 
-    // Verify bucket creation fails with unresolved variable
-    let unresolved_bucket = format!("${{}}-test {test_user}");
-    let create_unresolved = test_client.create_bucket().bucket(&unresolved_bucket).send().await;
-
-    if create_unresolved.is_ok() {
-        cleanup().await;
-        return Err("User should NOT be able to create bucket with unresolved variable".into());
-    }
+    // Verify a valid bucket name outside the resolved resource is denied.
+    let denied = test_client
+        .create_bucket()
+        .bucket("other-user-test")
+        .send()
+        .await
+        .expect_err("a bucket outside the resolved nested variable must be denied");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
 
     // Cleanup
     info!("Cleaning up test resources");
@@ -598,9 +557,8 @@ pub async fn test_aws_policy_variables_nested_impl_with_env(
     Ok(())
 }
 
-/// Implementation function for STS policy variables test with shared environment
-pub async fn test_aws_policy_variables_sts_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_sts_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user for STS
     let test_user = "testuser-sts";
@@ -612,8 +570,7 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
         cleanup_user_and_policy(env, test_user, policy_name).await;
     };
 
-    // Create STS user
-    create_sts_user(env, test_user, test_password).await?;
+    create_user(env, test_user, test_password).await?;
 
     // Create policy with STS-compatible variables
     let policy_document = serde_json::json!({
@@ -624,6 +581,11 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
                 "Action": ["s3:ListAllMyBuckets"],
                 "Resource": ["arn:aws:s3:::*"]
             },
+            {
+                "Effect": "Allow",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"]
+            },
             {
                 "Effect": "Allow",
                 "Action": ["s3:CreateBucket"],
@@ -631,7 +593,12 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
             },
             {
                 "Effect": "Allow",
-                "Action": ["s3:ListBucket", "s3:PutObject", "s3:GetObject"],
+                "Action": ["s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{}-sts-bucket", "${aws:username}")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:PutObject", "s3:GetObject"],
                 "Resource": [format!("arn:aws:s3:::{}-sts-bucket/*", "${aws:username}")]
             }
         ]
@@ -639,11 +606,22 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
 
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
-    // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
-
-    // Add a small delay to allow policy to propagate
-    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    let assumed = build_test_sts_client(&env.url, test_user, test_password, None, "policy-variable-sts")
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/policy-variable")
+        .role_session_name("policy-variable-e2e")
+        .send()
+        .await?;
+    let credentials = assumed
+        .credentials()
+        .ok_or("AssumeRole response should contain temporary credentials")?;
+    let test_client = Client::from_conf(build_test_s3_config(
+        &env.url,
+        credentials.access_key_id(),
+        credentials.secret_access_key(),
+        Some(credentials.session_token()),
+        "policy-variable-sts-session",
+    ));
 
     // Test: User should be able to create bucket matching STS pattern
     info!("Test: User creating bucket matching STS pattern");
@@ -699,24 +677,18 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
 
 /// Test AWS policy variables with deny scenarios
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    test_aws_policy_variables_deny_impl().await
-}
-
-/// Implementation function for deny policy variables test
-pub async fn test_aws_policy_variables_deny_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables deny test");
 
-    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
 
     test_aws_policy_variables_deny_impl_with_env(&env).await
 }
 
-/// Implementation function for deny policy variables test with shared environment
-pub async fn test_aws_policy_variables_deny_impl_with_env(
-    env: &PolicyTestEnvironment,
+async fn test_aws_policy_variables_deny_impl_with_env(
+    env: &RustFSTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser5";
@@ -759,10 +731,7 @@ pub async fn test_aws_policy_variables_deny_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client(test_user, test_password);
-
-    // Add a small delay to allow policy to propagate
-    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
 
     // Test 1: User should be able to create bucket matching username pattern
     info!("Test 1: User creating bucket matching username pattern");
@@ -775,12 +744,14 @@ pub async fn test_aws_policy_variables_deny_impl_with_env(
 
     // Test 2: User should NOT be able to create bucket with "private" in the name (deny rule)
     info!("Test 2: User attempting to create bucket with 'private' in name (should be denied)");
-    let private_bucket_name = "private-test-bucket";
-    let create_private_result = test_client.create_bucket().bucket(private_bucket_name).send().await;
-    if create_private_result.is_ok() {
-        cleanup().await;
-        return Err("User should NOT be able to create bucket with 'private' in name due to deny rule".into());
-    }
+    let private_bucket_name = format!("{test_user}-private-bucket");
+    let denied = test_client
+        .create_bucket()
+        .bucket(&private_bucket_name)
+        .send()
+        .await
+        .expect_err("the explicit deny must reject a matching bucket name");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
 
     // Cleanup
     info!("Cleaning up test resources");

crates/e2e_test/src/policy/test_env.rs

@@ -1,100 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Custom test environment for policy variables tests
-//!
-//! This module provides a custom test environment that doesn't automatically
-//! stop servers when destroyed, addressing the server stopping issue.
-
-use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Config, Credentials, Region};
-use std::net::TcpStream;
-use std::time::Duration;
-use tokio::time::sleep;
-use tracing::{info, warn};
-
-// Default credentials
-const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
-const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
-
-/// Custom test environment that doesn't automatically stop servers
-pub struct PolicyTestEnvironment {
-    pub temp_dir: String,
-    pub address: String,
-    pub url: String,
-    pub access_key: String,
-    pub secret_key: String,
-}
-
-impl PolicyTestEnvironment {
-    /// Create a new test environment with specific address
-    /// This environment won't stop any server when dropped
-    pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
-        let temp_dir = format!("/tmp/rustfs_policy_test_{}", uuid::Uuid::new_v4());
-        tokio::fs::create_dir_all(&temp_dir).await?;
-
-        let url = format!("http://{address}");
-
-        Ok(Self {
-            temp_dir,
-            address: address.to_string(),
-            url,
-            access_key: DEFAULT_ACCESS_KEY.to_string(),
-            secret_key: DEFAULT_SECRET_KEY.to_string(),
-        })
-    }
-
-    /// Create an AWS S3 client configured for this RustFS instance
-    pub fn create_s3_client(&self, access_key: &str, secret_key: &str) -> Client {
-        let credentials = Credentials::new(access_key, secret_key, None, None, "policy-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-        Client::from_conf(config)
-    }
-
-    /// Wait for RustFS server to be ready by checking TCP connectivity
-    pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        info!("Waiting for RustFS server to be ready on {}", self.address);
-
-        for i in 0..30 {
-            if TcpStream::connect(&self.address).is_ok() {
-                info!("✅ RustFS server is ready after {} attempts", i + 1);
-                return Ok(());
-            }
-
-            if i == 29 {
-                return Err("RustFS server failed to become ready within 30 seconds".into());
-            }
-
-            sleep(Duration::from_secs(1)).await;
-        }
-
-        Ok(())
-    }
-}
-
-// Implement Drop trait that doesn't stop servers
-impl Drop for PolicyTestEnvironment {
-    fn drop(&mut self) {
-        // Clean up temp directory only, don't stop any server
-        if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
-            warn!("Failed to clean up temp directory {}: {}", self.temp_dir, e);
-        }
-    }
-}

crates/e2e_test/src/policy/test_runner.rs

@@ -1,230 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::common::init_logging;
-use crate::policy::test_env::PolicyTestEnvironment;
-use std::time::Instant;
-use tokio::time::{Duration, sleep};
-use tracing::{error, info};
-
-/// Test case definition
-#[derive(Debug, Clone)]
-pub struct TestDefinition {
-    pub name: String,
-    pub is_critical: bool,
-}
-
-impl TestDefinition {
-    pub fn new(name: impl Into<String>, is_critical: bool) -> Self {
-        Self {
-            name: name.into(),
-            is_critical,
-        }
-    }
-}
-
-/// Test result
-#[derive(Debug, Clone)]
-pub struct TestResult {
-    pub test_name: String,
-    pub success: bool,
-    pub error_message: Option<String>,
-}
-
-impl TestResult {
-    pub fn success(test_name: String) -> Self {
-        Self {
-            test_name,
-            success: true,
-            error_message: None,
-        }
-    }
-
-    pub fn failure(test_name: String, error: String) -> Self {
-        Self {
-            test_name,
-            success: false,
-            error_message: Some(error),
-        }
-    }
-}
-
-/// Test suite configuration
-#[derive(Debug, Clone, Default)]
-pub struct TestSuiteConfig {
-    pub include_critical_only: bool,
-}
-
-/// Policy test suite
-pub struct PolicyTestSuite {
-    tests: Vec<TestDefinition>,
-    config: TestSuiteConfig,
-}
-
-impl PolicyTestSuite {
-    /// Create default test suite
-    pub fn new() -> Self {
-        let tests = vec![
-            TestDefinition::new("test_aws_policy_variables_single_value", true),
-            TestDefinition::new("test_aws_policy_variables_multi_value", true),
-            TestDefinition::new("test_aws_policy_variables_concatenation", true),
-            TestDefinition::new("test_aws_policy_variables_nested", true),
-            TestDefinition::new("test_aws_policy_variables_deny", true),
-            TestDefinition::new("test_aws_policy_variables_sts", true),
-        ];
-
-        Self {
-            tests,
-            config: TestSuiteConfig::default(),
-        }
-    }
-
-    /// Configure test suite
-    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
-        self.config = config;
-        self
-    }
-
-    /// Run test suite
-    pub async fn run_test_suite(&self) -> Vec<TestResult> {
-        init_logging();
-        info!("Starting Policy Variables test suite");
-
-        let start_time = Instant::now();
-        let mut results = Vec::new();
-
-        // Create test environment
-        let env = match PolicyTestEnvironment::with_address("127.0.0.1:9000").await {
-            Ok(env) => env,
-            Err(e) => {
-                error!("Failed to create test environment: {}", e);
-                return vec![TestResult::failure("env_creation".into(), e.to_string())];
-            }
-        };
-
-        // Wait for server to be ready
-        if env.wait_for_server_ready().await.is_err() {
-            error!("Server is not ready");
-            return vec![TestResult::failure("server_check".into(), "Server not ready".into())];
-        }
-
-        // Filter tests
-        let tests_to_run: Vec<&TestDefinition> = self
-            .tests
-            .iter()
-            .filter(|test| !self.config.include_critical_only || test.is_critical)
-            .collect();
-
-        info!("Scheduled {} tests", tests_to_run.len());
-
-        // Run tests
-        for (i, test_def) in tests_to_run.iter().enumerate() {
-            info!("Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
-            let test_start = Instant::now();
-
-            let result = self.run_single_test(test_def, &env).await;
-            let test_duration = test_start.elapsed();
-
-            match result {
-                Ok(_) => {
-                    info!("Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
-                    results.push(TestResult::success(test_def.name.clone()));
-                }
-                Err(e) => {
-                    error!("Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
-                    results.push(TestResult::failure(test_def.name.clone(), e.to_string()));
-                }
-            }
-
-            // Delay between tests to avoid resource conflicts
-            if i < tests_to_run.len() - 1 {
-                sleep(Duration::from_secs(2)).await;
-            }
-        }
-
-        // Print summary
-        self.print_summary(&results, start_time.elapsed());
-
-        results
-    }
-
-    /// Run a single test
-    async fn run_single_test(
-        &self,
-        test_def: &TestDefinition,
-        env: &PolicyTestEnvironment,
-    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        match test_def.name.as_str() {
-            "test_aws_policy_variables_single_value" => {
-                super::policy_variables_test::test_aws_policy_variables_single_value_impl_with_env(env).await
-            }
-            "test_aws_policy_variables_multi_value" => {
-                super::policy_variables_test::test_aws_policy_variables_multi_value_impl_with_env(env).await
-            }
-            "test_aws_policy_variables_concatenation" => {
-                super::policy_variables_test::test_aws_policy_variables_concatenation_impl_with_env(env).await
-            }
-            "test_aws_policy_variables_nested" => {
-                super::policy_variables_test::test_aws_policy_variables_nested_impl_with_env(env).await
-            }
-            "test_aws_policy_variables_deny" => {
-                super::policy_variables_test::test_aws_policy_variables_deny_impl_with_env(env).await
-            }
-            "test_aws_policy_variables_sts" => {
-                super::policy_variables_test::test_aws_policy_variables_sts_impl_with_env(env).await
-            }
-            _ => Err(format!("Test {} not implemented", test_def.name).into()),
-        }
-    }
-
-    /// Print test summary
-    fn print_summary(&self, results: &[TestResult], total_duration: Duration) {
-        info!("=== Test Suite Summary ===");
-        info!("Total duration: {:.2}s", total_duration.as_secs_f64());
-        info!("Total tests: {}", results.len());
-
-        let passed = results.iter().filter(|r| r.success).count();
-        let failed = results.len() - passed;
-        let success_rate = (passed as f64 / results.len() as f64) * 100.0;
-
-        info!("Passed: {} | Failed: {}", passed, failed);
-        info!("Success rate: {:.1}%", success_rate);
-
-        if failed > 0 {
-            error!("Failed tests:");
-            for result in results.iter().filter(|r| !r.success) {
-                error!("  - {}: {}", result.test_name, result.error_message.as_ref().unwrap());
-            }
-        }
-    }
-}
-
-/// Test suite
-#[tokio::test]
-#[ignore = "Connects to existing rustfs server"]
-async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let config = TestSuiteConfig {
-        include_critical_only: true,
-    };
-    let suite = PolicyTestSuite::new().with_config(config);
-    let results = suite.run_test_suite().await;
-
-    let failed = results.iter().filter(|r| !r.success).count();
-    if failed > 0 {
-        return Err(format!("Critical tests failed: {failed} failures").into());
-    }
-
-    info!("All critical tests passed");
-    Ok(())
-}

crates/e2e_test/src/presigned_negative_test.rs

@@ -340,7 +340,18 @@ async fn tampered_presigned_put_returns_signature_does_not_match() -> Result<(),
     assert_error_code(&body, "SignatureDoesNotMatch");
 
     // The rejected write must not have created the object.
-    let head = env.create_s3_client().head_object().bucket(BUCKET).key(key).send().await;
-    assert!(head.is_err(), "tampered presigned PUT must not store the object");
+    let error = env
+        .create_s3_client()
+        .head_object()
+        .bucket(BUCKET)
+        .key(key)
+        .send()
+        .await
+        .expect_err("tampered presigned PUT must not store the object");
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "tampered presigned PUT absence probe must return HTTP 404, got {error:?}"
+    );
     Ok(())
 }

crates/e2e_test/src/protocols/sftp_compliance_tests.rs

@@ -453,27 +453,28 @@ fn watch_session_lifecycle_events(child: &mut Child, counters: Arc<SessionCounte
 }
 
 /// Count TCP connections in CLOSE_WAIT against the given local port
-/// by shelling out to ss -tn state CLOSE-WAIT. The check is
-/// best-effort: if ss is missing on the host the function returns
-/// Ok(None) and the caller skips the assertion. The contract is zero
+/// by shelling out to ss -tn state CLOSE-WAIT. The oracle fails closed
+/// if ss is missing or cannot inspect socket state. The contract is zero
 /// CLOSE_WAIT entries attributable to the test.
 #[cfg(target_os = "linux")]
-async fn count_close_wait_on_port(port: u16) -> Result<Option<usize>> {
-    let output = match Command::new("ss").args(["-tn", "state", "CLOSE-WAIT"]).output().await {
-        Ok(o) => o,
-        Err(_) => return Ok(None),
-    };
+async fn count_close_wait_on_port(port: u16, test_id: &str) -> Result<usize> {
+    let port_filter = format!("sport = :{port}");
+    let output = Command::new("ss")
+        .args(["-H", "-t", "-n", "state", "close-wait"])
+        .arg(&port_filter)
+        .output()
+        .await
+        .map_err(|error| anyhow!("{test_id} failed to run ss CLOSE_WAIT oracle: {error}"))?;
     if !output.status.success() {
-        return Ok(None);
+        return Err(anyhow!(
+            "{test_id} ss CLOSE_WAIT oracle exited with {}: {}",
+            output.status,
+            String::from_utf8_lossy(&output.stderr)
+        ));
     }
     let stdout = String::from_utf8_lossy(&output.stdout);
-    let needle_local = format!(":{port} ");
-    let needle_local_eol = format!(":{port}\n");
-    let count = stdout
-        .lines()
-        .filter(|l| l.contains(&needle_local) || l.contains(needle_local_eol.trim_end()))
-        .count();
-    Ok(Some(count))
+    let count = stdout.lines().filter(|line| !line.trim().is_empty()).count();
+    Ok(count)
 }
 
 // CMPTST-01: medium-binary upload then download with SHA256 compare.
@@ -1400,7 +1401,7 @@ pub(crate) mod cmptst_24 {
     //    the JoinSet flushes finished tasks before the assertion runs.
     // 5. Assert the entered/finished session counters balance and that
     //    no CLOSE_WAIT sockets remain on the bind port (Linux ss(8)
-    //    only; the assertion skips with a warn if ss is unavailable).
+    //    only; missing or failed socket inspection is an error).
     pub(crate) async fn run_concurrent_half_close_no_leak() -> Result<()> {
         let env = ProtocolTestEnvironment::new().map_err(|e| anyhow!("{}", e))?;
         let host_key_dir = PathBuf::from(&env.temp_dir).join("sftp_host_keys");
@@ -1520,15 +1521,13 @@ pub(crate) mod cmptst_24 {
                 ));
             }
 
-            match count_close_wait_on_port(HALF_CLOSE_SFTP_PORT).await? {
-                Some(0) => info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}"),
-                Some(n) => {
-                    return Err(anyhow!(
-                        "{COMPLIANCE_TEST_OUTPUT_ID} {n} CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}, expected 0"
-                    ));
-                }
-                None => info!("{COMPLIANCE_TEST_OUTPUT_ID}: ss(8) unavailable, skipping CLOSE_WAIT assertion"),
+            let close_wait = count_close_wait_on_port(HALF_CLOSE_SFTP_PORT, COMPLIANCE_TEST_OUTPUT_ID).await?;
+            if close_wait != 0 {
+                return Err(anyhow!(
+                    "{COMPLIANCE_TEST_OUTPUT_ID} {close_wait} CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}, expected 0"
+                ));
             }
+            info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}");
 
             // Drop the keepalive vector now so the test process does
             // not leave the half-closed sockets dangling past the
@@ -1947,7 +1946,7 @@ pub(crate) mod cmptst_25 {
     //    plus two 15 s ticks worst-case = 60 s) to detect CLOSE_WAIT
     //    via /proc/net/tcp and cancel the parked session.
     // 5. Assert the session task counters balance and CLOSE_WAIT count
-    //    is zero (ss(8) only; skips with a warn when ss is missing).
+    //    is zero (ss(8) only; missing or failed socket inspection is an error).
     pub(crate) async fn run_wedge_kill_after_silence_in_close_wait() -> Result<()> {
         let env = ProtocolTestEnvironment::new().map_err(|e| anyhow!("{}", e))?;
         let host_key_dir = PathBuf::from(&env.temp_dir).join("sftp_host_keys");
@@ -2056,15 +2055,13 @@ pub(crate) mod cmptst_25 {
                 ));
             }
 
-            match count_close_wait_on_port(WEDGE_SFTP_PORT).await? {
-                Some(0) => info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}"),
-                Some(n) => {
-                    return Err(anyhow!(
-                        "{COMPLIANCE_TEST_OUTPUT_ID} {n} CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}, expected 0"
-                    ));
-                }
-                None => info!("{COMPLIANCE_TEST_OUTPUT_ID}: ss(8) unavailable, skipping CLOSE_WAIT assertion"),
+            let close_wait = count_close_wait_on_port(WEDGE_SFTP_PORT, COMPLIANCE_TEST_OUTPUT_ID).await?;
+            if close_wait != 0 {
+                return Err(anyhow!(
+                    "{COMPLIANCE_TEST_OUTPUT_ID} {close_wait} CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}, expected 0"
+                ));
             }
+            info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}");
 
             drop(keepalive);
             info!("PASS {COMPLIANCE_TEST_OUTPUT_ID}: wedged sessions killed by the watchdog");

crates/e2e_test/src/protocols/sftp_helpers.rs

@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use russh::client::{self, Handle};
 use russh::keys::ssh_key::LineEnding;
-use russh::keys::{Algorithm, PrivateKey, PublicKey};
+use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
 use russh_sftp::client::SftpSession;
 use russh_sftp::protocol::OpenFlags;
 use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
 impl client::Handler for AcceptAnyServerKey {
     type Error = anyhow::Error;
 
-    async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
+    async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
         Ok(true)
     }
 }

crates/e2e_test/src/quota_test.rs

@@ -18,15 +18,6 @@ use http::{Method, StatusCode};
 use tokio::time::{Duration, sleep, timeout};
 use tracing::{debug, info};
 
-fn skip_without_awscurl() -> bool {
-    if crate::common::awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping quota test because awscurl is not available");
-    true
-}
-
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -276,9 +267,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -320,9 +308,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -371,9 +356,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -406,9 +388,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -442,9 +421,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -480,9 +456,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -513,9 +486,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -553,9 +523,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -593,9 +560,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -628,9 +592,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -689,9 +650,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -724,18 +682,26 @@ mod integration_tests {
         assert!(resp.contains("quota_limit"));
 
         // Normal user sets quota — should be denied
-        let set_resp = awscurl_put(
+        let set_error = awscurl_put(
             &get_url,
             &serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(),
             normal_ak,
             normal_sk,
         )
-        .await;
-        assert!(set_resp.is_err(), "normal user should not be able to set quota");
+        .await
+        .expect_err("normal user should not be able to set quota")
+        .to_string();
+        assert!(set_error.contains("AccessDenied"), "quota denial must return AccessDenied: {set_error}");
 
         // Normal user clears quota — should be denied
-        let del_resp = awscurl_delete(&get_url, normal_ak, normal_sk).await;
-        assert!(del_resp.is_err(), "normal user should not be able to clear quota");
+        let delete_error = awscurl_delete(&get_url, normal_ak, normal_sk)
+            .await
+            .expect_err("normal user should not be able to clear quota")
+            .to_string();
+        assert!(
+            delete_error.contains("AccessDenied"),
+            "quota deletion denial must return AccessDenied: {delete_error}"
+        );
 
         env.cleanup_bucket().await?;
         Ok(())
@@ -744,9 +710,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -789,9 +752,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -847,9 +807,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -1,52 +1,26 @@
 #![cfg(test)]
 
-use aws_config::meta::region::RegionProviderChain;
+use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use bytes::Bytes;
 use std::error::Error;
+use std::fmt::Debug;
 
-const ENDPOINT: &str = "http://localhost:9000";
-const ACCESS_KEY: &str = "rustfsadmin";
-const SECRET_KEY: &str = "rustfsadmin";
-const BUCKET: &str = "api-test";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
 
-async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
-    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
-    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
-        .region(region_provider)
-        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
-        .endpoint_url(ENDPOINT)
-        .load()
-        .await;
-
-    let client = Client::from_conf(
-        aws_sdk_s3::Config::from(&shared_config)
-            .to_builder()
-            .force_path_style(true)
-            .build(),
+fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
+where
+    T: Debug,
+    E: ProvideErrorMetadata + Debug,
+{
+    let error = result.expect_err("conditional request must fail");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some(expected),
+        "unexpected conditional request error: {error:?}"
     );
-    Ok(client)
-}
-
-/// Setup test bucket, creating it if it doesn't exist
-async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
-    match client.create_bucket().bucket(BUCKET).send().await {
-        Ok(_) => {}
-        Err(SdkError::ServiceError(e)) => {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            if !error_code.eq("BucketAlreadyExists") {
-                return Err(e.into());
-            }
-        }
-        Err(e) => {
-            return Err(e.into());
-        }
-    }
-    Ok(())
 }
 
 /// Generate test data of specified size
@@ -60,7 +34,12 @@ fn generate_test_data(size: usize) -> Vec<u8> {
 }
 
 /// Upload an object and return its ETag
-async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
+async fn upload_object_with_metadata(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+) -> Result<String, Box<dyn Error + Send + Sync>> {
     let response = client
         .put_object()
         .bucket(bucket)
@@ -69,188 +48,164 @@ async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, d
         .send()
         .await?;
 
-    let etag = response.e_tag().unwrap_or("").to_string();
-    Ok(etag)
+    response
+        .e_tag()
+        .map(str::to_owned)
+        .ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
 }
 
-/// Cleanup test objects from bucket
-async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
-    for key in keys {
-        let _ = client.delete_object().bucket(bucket).key(*key).send().await;
-    }
-}
-
-/// Generate unique test object key
-fn generate_test_key(prefix: &str) -> String {
-    use std::time::{SystemTime, UNIX_EPOCH};
-    let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
-    format!("{prefix}-{timestamp}")
+async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
+    let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
+    Ok(response.body.collect().await?.into_bytes())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_okay() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-ok");
+    let test_key = "conditional-put-ok";
     let initial_data = generate_test_data(1024); // 1KB test data
-    let updated_data = generate_test_data(2048); // 2KB updated data
+    let matching_data = generate_test_data(2048); // 2KB updated data
+    let non_matching_data = generate_test_data(3072); // 3KB updated data
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with matching If-Match condition (should succeed)
-    let response1 = client
+    client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(matching_data.clone()).into())
         .if_match(&initial_etag)
         .send()
-        .await;
-    assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
 
     // Test 2: PUT with non-matching If-None-Match condition (should succeed)
     let fake_etag = "\"fake-etag-12345\"";
-    let response2 = client
+    client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(non_matching_data.clone()).into())
         .if_none_match(fake_etag)
         .send()
-        .await;
-    assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_failed() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-failed");
+    let test_key = "conditional-put-failed";
     let initial_data = generate_test_data(1024);
     let updated_data = generate_test_data(2048);
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with non-matching If-Match condition (should fail with 412)
     let fake_etag = "\"fake-etag-should-not-match\"";
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_match(fake_etag)
         .send()
         .await;
 
-    assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
 
     // Test 2: PUT with matching If-None-Match condition (should fail with 412)
     let response2 = client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_none_match(&initial_etag)
         .send()
         .await;
 
-    assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
-    if let Err(e) = response2 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
-
-    // Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+    assert_s3_error_code(response2, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let key = "some_key";
-    cleanup_objects(&client, BUCKET, &[key]).await;
+    let key = "conditional-put-missing";
 
     // When the object does not exist, the If-Match condition should always fail
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
         .body(Bytes::from(generate_test_data(1024)).into())
         .if_match("*")
         .send()
         .await;
-    assert!(response1.is_err());
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("NoSuchKey", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "NoSuchKey");
 
     // When the object does not exist, the If-None-Match condition should be able to succeed
-    let response2 = client
+    let created_data = generate_test_data(1024);
+    client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
-        .body(Bytes::from(generate_test_data(1024)).into())
+        .body(Bytes::from(created_data.clone()).into())
         .if_none_match("*")
         .send()
-        .await;
-    assert!(response2.is_ok());
+        .await?;
+    assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
 
-    cleanup_objects(&client, BUCKET, &[key]).await;
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_multi_part_upload() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("multipart-upload-ok");
+    let test_key = "conditional-multipart-upload";
     let test_data = generate_test_data(1024);
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
 
     let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
     let num_parts = 3;
     let mut parts = Vec::new();
+    let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
 
     // Initiate multipart upload
-    let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
+    let initiate_response = client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .send()
+        .await?;
 
     let upload_id = initiate_response
         .upload_id()
@@ -258,12 +213,13 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     // Upload parts
     for part_number in 1..=num_parts {
-        let part_data = generate_test_data(part_size);
+        let part_data = vec![u8::try_from(part_number)?; part_size];
+        expected_data.extend_from_slice(&part_data);
 
         let upload_part_response = client
             .upload_part()
-            .bucket(BUCKET)
-            .key(&test_key)
+            .bucket(TEST_BUCKET)
+            .key(test_key)
             .upload_id(upload_id)
             .part_number(part_number)
             .body(Bytes::from(part_data).into())
@@ -286,57 +242,62 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
     // Test 1: Multipart upload with wildcard If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match("*")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 2: Multipart upload with matching If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match(initial_etag.clone())
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 3: Multipart upload with unmatching If-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_match("\"abcdef\"")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
+
+    let staged_parts = client
+        .list_parts()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
 
     // Test 4: Multipart upload with matching If-Match, should succeed
-    let complete_response = client
+    client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
-        .multipart_upload(completed_upload.clone())
+        .multipart_upload(completed_upload)
         .if_match(initial_etag)
         .send()
-        .await;
-
-    assert!(complete_response.is_ok());
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
 
     Ok(())
 }

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -13,207 +13,228 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::workspace_root;
+use crate::common::RustFSTestEnvironment;
 use crate::storage_api::node_interact::{
     TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
 };
-use futures::future::join_all;
+use aws_sdk_s3::primitives::ByteStream;
 use rmp_serde::{Deserializer, Serializer};
-use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
+use rustfs_filemeta::MetaCacheEntry;
 use rustfs_protos::proto_gen::node_service::WalkDirRequest;
 use rustfs_protos::{
     models::{PingBody, PingBodyBuilder},
-    proto_gen::node_service::{
-        ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
-    },
+    proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
 };
 use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::io::Cursor;
-use std::path::PathBuf;
-use tokio::spawn;
 use tonic::Request;
 use tonic::codegen::tokio_stream::StreamExt;
 
-const CLUSTER_ADDR: &str = "http://localhost:9000";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
 
 fn signature_interceptor() -> TonicInterceptor {
     TonicInterceptor::Signature(gen_tonic_signature_interceptor())
 }
 
+fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
+    std::io::Error::other(error.to_string())
+}
+
+async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
+    let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
+    let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
+    assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_without_cleanup_with_env(&[
+        ("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
+        ("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
+        ("RUST_LOG", "error"),
+    ])
+    .await?;
+    Ok(env)
+}
+
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn ping() -> Result<(), Box<dyn Error>> {
+async fn ping() -> TestResult {
+    let env = start_server().await?;
     let mut fbb = flatbuffers::FlatBufferBuilder::new();
     let payload = fbb.create_vector(b"hello world");
-
     let mut builder = PingBodyBuilder::new(&mut fbb);
     builder.add_payload(payload);
     let root = builder.finish();
     fbb.finish(root, None);
 
-    let finished_data = fbb.finished_data();
-
-    let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
-    assert!(decoded_payload.is_ok());
-
-    // Create client
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-
-    // Construct PingRequest
-    let request = Request::new(PingRequest {
-        version: 1,
-        body: bytes::Bytes::copy_from_slice(finished_data),
-    });
-
-    // Send request and get response
-    let response: PingResponse = client.ping(request).await?.into_inner();
-
-    // Print response
-    let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
-    if let Err(e) = ping_response_body {
-        eprintln!("{e}");
-    } else {
-        println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
-    }
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .ping(Request::new(PingRequest {
+            version: 1,
+            body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
+        }))
+        .await?
+        .into_inner();
 
+    assert_eq!(response.version, 1);
+    let body = flatbuffers::root::<PingBody>(&response.body)?;
+    assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn make_volume() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(MakeVolumeRequest {
-        disk: "data".to_string(),
-        volume: "dandan".to_string(),
-    });
+async fn make_volume() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-volume".to_string(),
+        }))
+        .await?
+        .into_inner();
 
-    let response = client.make_volume(request).await?.into_inner();
-    if response.success {
-        println!("success");
-    } else {
-        println!("failed: {:?}", response.error);
-    }
+    assert!(response.success, "make_volume failed: {:?}", response.error);
+    assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn list_volumes() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ListVolumesRequest {
-        disk: "data".to_string(),
-    });
+async fn list_volumes() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-listed-volume".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);
 
-    let response = client.list_volumes(request).await?.into_inner();
-    let volume_infos: Vec<VolumeInfo> = response
+    let response = client
+        .list_volumes(Request::new(ListVolumesRequest {
+            disk: env.temp_dir.clone(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "list_volumes failed: {:?}", response.error);
+    let volumes = response
         .volume_infos
-        .into_iter()
-        .filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
-        .collect();
-
-    println!("{volume_infos:?}");
+        .iter()
+        .map(|json| serde_json::from_str::<VolumeInfo>(json))
+        .collect::<Result<Vec<_>, _>>()?;
+    assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn walk_dir() -> Result<(), Box<dyn Error>> {
-    println!("walk_dir");
-    // TODO: use writer
+async fn walk_dir() -> TestResult {
+    let env = start_server().await?;
+    let s3 = env.create_s3_client();
+    let bucket = "node-rpc-walk-bucket";
+    let key = "prefix/object.txt";
+    env.create_test_bucket(bucket).await?;
+    s3.put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"walk payload"))
+        .send()
+        .await?;
+
     let opts = WalkDirOptions {
-        bucket: "dandan".to_owned(),
-        base_dir: "".to_owned(),
+        bucket: bucket.to_string(),
         recursive: true,
         ..Default::default()
     };
-    let (rd, mut wr) = tokio::io::duplex(1024);
-    let mut buf = Vec::new();
-    opts.serialize(&mut Serializer::new(&mut buf))?;
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
-        let mut path = workspace_root();
-        path.push("target");
-        path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
-        path.push("data");
-        path
-    });
-    let request = Request::new(WalkDirRequest {
-        disk: disk_path.to_string_lossy().into_owned(),
-        walk_dir_options: buf.into(),
-    });
-    let mut response = client.walk_dir(request).await?.into_inner();
+    let mut encoded = Vec::new();
+    opts.serialize(&mut Serializer::new(&mut encoded))?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let mut stream = client
+        .walk_dir(Request::new(WalkDirRequest {
+            disk: env.temp_dir.clone(),
+            walk_dir_options: encoded.into(),
+        }))
+        .await?
+        .into_inner();
 
-    let job1 = spawn(async move {
-        let mut out = MetacacheWriter::new(&mut wr);
-        loop {
-            match response.next().await {
-                Some(Ok(resp)) => {
-                    if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
-                    }
-                    let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
-                        .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
-                        .unwrap();
-                    out.write_obj(&entry).await.unwrap();
-                }
-                None => {
-                    let _ = out.close().await;
-                    break;
-                }
-                _ => {
-                    println!("Unexpected response: {response:?}");
-                    let _ = out.close().await;
-                    break;
-                }
-            }
-        }
-    });
-    let job2 = spawn(async move {
-        let mut reader = MetacacheReader::new(rd);
-        while let Ok(Some(entry)) = reader.peek().await {
-            println!("{entry:?}");
-        }
-    });
-
-    join_all(vec![job1, job2]).await;
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn read_all() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ReadAllRequest {
-        disk: "data".to_string(),
-        volume: "ff".to_string(),
-        path: "format.json".to_string(),
-    });
-
-    let response = client.read_all(request).await?.into_inner();
-    let volume_infos = response.data;
-
-    println!("{}", response.success);
-    println!("{volume_infos:?}");
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn storage_info() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(LocalStorageInfoRequest { metrics: true });
-
-    let response = client.local_storage_info(request).await?.into_inner();
-    if !response.success {
-        println!("{:?}", response.error_info);
-        return Ok(());
+    let mut entries = Vec::new();
+    while let Some(response) = stream.next().await {
+        let response = response?;
+        assert!(response.success, "walk_dir failed: {:?}", response.error_info);
+        entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
     }
-    let info = response.storage_info;
-
-    let mut buf = Deserializer::new(Cursor::new(info));
-    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
-    println!("{storage_info:?}");
+    assert!(
+        entries.iter().any(|entry| entry.name == key),
+        "walk_dir did not return {key}: {entries:?}"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn read_all() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let volume = "node-rpc-read-volume";
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);
+    tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
+
+    let response = client
+        .read_all(Request::new(ReadAllRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+            path: "payload.bin".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "read_all failed: {:?}", response.error);
+    assert_eq!(response.data.as_ref(), b"read payload");
+    Ok(())
+}
+
+#[tokio::test]
+async fn storage_info() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
+        .await?
+        .into_inner();
+    assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
+
+    let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
+    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
+    let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
+    assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
+    assert!(
+        storage_info
+            .disks
+            .iter()
+            .any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
+        "local_storage_info did not include the configured disk: {:?}",
+        storage_info.disks
+    );
     Ok(())
 }

crates/e2e_test/src/reliant/sql.rs

@@ -13,55 +13,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use aws_config::meta::region::RegionProviderChain;
+use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{
     CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
 };
 use bytes::Bytes;
 use std::error::Error;
+use std::time::Duration;
 
-const ENDPOINT: &str = "http://localhost:9000";
-const ACCESS_KEY: &str = "rustfsadmin";
-const SECRET_KEY: &str = "rustfsadmin";
 const BUCKET: &str = "test-sql-bucket";
 const CSV_OBJECT: &str = "test-data.csv";
 const JSON_OBJECT: &str = "test-data.json";
+const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
 
-async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
-    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
-    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
-        .region(region_provider)
-        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
-        .endpoint_url(ENDPOINT)
-        .load()
-        .await;
+type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
 
-    let client = Client::from_conf(
-        aws_sdk_s3::Config::from(&shared_config)
-            .to_builder()
-            .force_path_style(true) // Important for S3-compatible services
-            .build(),
-    );
-
-    Ok(client)
+async fn create_test_environment() -> TestResult<(RustFSTestEnvironment, Client)> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let client = env.create_s3_client();
+    Ok((env, client))
 }
 
-async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
-    match client.create_bucket().bucket(BUCKET).send().await {
-        Ok(_) => {}
-        Err(e) => {
-            let error_str = e.to_string();
-            if !error_str.contains("BucketAlreadyOwnedByYou") && !error_str.contains("BucketAlreadyExists") {
-                return Err(e.into());
-            }
-        }
-    }
+async fn setup_test_bucket(client: &Client) -> TestResult<()> {
+    client.create_bucket().bucket(BUCKET).send().await?;
     Ok(())
 }
 
-async fn upload_test_csv(client: &Client) -> Result<(), Box<dyn Error>> {
+async fn upload_test_csv(client: &Client) -> TestResult<()> {
     let csv_data = "name,age,city\nAlice,30,New York\nBob,25,Los Angeles\nCharlie,35,Chicago\nDiana,28,Boston";
 
     client
@@ -75,7 +57,7 @@ async fn upload_test_csv(client: &Client) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-async fn upload_test_json(client: &Client) -> Result<(), Box<dyn Error>> {
+async fn upload_test_json(client: &Client) -> TestResult<()> {
     let json_data = r#"{"name":"Alice","age":30,"city":"New York"}
 {"name":"Bob","age":25,"city":"Los Angeles"}
 {"name":"Charlie","age":35,"city":"Chicago"}
@@ -93,33 +75,38 @@ async fn upload_test_json(client: &Client) -> Result<(), Box<dyn Error>> {
 
 async fn process_select_response(
     mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
-) -> Result<String, Box<dyn Error>> {
-    let mut total_data = Vec::new();
+) -> TestResult<String> {
+    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
+        let mut total_data = Vec::new();
+        let mut saw_end = false;
 
-    while let Ok(Some(event)) = event_stream.payload.recv().await {
-        match event {
-            aws_sdk_s3::types::SelectObjectContentEventStream::Records(records_event) => {
-                if let Some(payload) = records_event.payload {
-                    let data = payload.into_inner();
-                    total_data.extend_from_slice(&data);
+        while let Some(event) = event_stream.payload.recv().await? {
+            match event {
+                aws_sdk_s3::types::SelectObjectContentEventStream::Records(records_event) => {
+                    if let Some(payload) = records_event.payload {
+                        total_data.extend_from_slice(payload.as_ref());
+                    }
                 }
-            }
-            aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
-                break;
-            }
-            _ => {
-                // Handle other event types (Stats, Progress, Cont, etc.)
+                aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
+                    saw_end = true;
+                    break;
+                }
+                _ => {}
             }
         }
-    }
 
-    Ok(String::from_utf8(total_data)?)
+        if !saw_end {
+            return Err("Select response ended without an End event".into());
+        }
+        Ok(String::from_utf8(total_data)?)
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_csv_basic() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_csv_basic() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_csv(&client).await?;
 
@@ -158,9 +145,8 @@ async fn test_select_object_content_csv_basic() -> Result<(), Box<dyn Error>> {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_csv_aggregation() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_csv_aggregation() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_csv(&client).await?;
 
@@ -203,16 +189,15 @@ async fn test_select_object_content_csv_aggregation() -> Result<(), Box<dyn Erro
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_json_basic() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_json_basic() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_json(&client).await?;
 
     // Construct JSON query
     let sql = "SELECT s.name, s.age FROM S3Object s WHERE s.age > 28";
 
-    let json_input = JsonInput::builder().set_type(Some(JsonType::Document)).build();
+    let json_input = JsonInput::builder().set_type(Some(JsonType::Lines)).build();
 
     let input_serialization = InputSerialization::builder().json(json_input).build();
 
@@ -244,9 +229,8 @@ async fn test_select_object_content_json_basic() -> Result<(), Box<dyn Error>> {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_csv_limit() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_csv(&client).await?;
 
@@ -286,9 +270,8 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_csv_order_by() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_csv(&client).await?;
 
@@ -318,9 +301,10 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>>
     println!("CSV Order By result: {result_str}");
 
     // Verify ordered by age descending
-    assert!(
-        result_str.lines().filter(|line| !line.trim().is_empty()).count() >= 2,
-        "Should return at least 2 records"
+    assert_eq!(
+        result_str.lines().filter(|line| !line.trim().is_empty()).count(),
+        2,
+        "Should return exactly 2 records"
     );
 
     // Check if contains highest age records
@@ -331,9 +315,8 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>>
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_error_handling() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_error_handling() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
     upload_test_csv(&client).await?;
 
@@ -348,7 +331,7 @@ async fn test_select_object_content_error_handling() -> Result<(), Box<dyn Error
     let output_serialization = OutputSerialization::builder().csv(csv_output).build();
 
     // This query should fail because invalid_column doesn't exist
-    let result = client
+    let error = client
         .select_object_content()
         .bucket(BUCKET)
         .key(CSV_OBJECT)
@@ -357,18 +340,20 @@ async fn test_select_object_content_error_handling() -> Result<(), Box<dyn Error
         .input_serialization(input_serialization)
         .output_serialization(output_serialization)
         .send()
-        .await;
+        .await
+        .expect_err("a query referencing an unknown column must fail");
 
-    // Verify query fails (expected behavior)
-    assert!(result.is_err(), "Query with invalid column should fail");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("EvaluatorBindingDoesNotExist")
+    );
 
     Ok(())
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_select_object_content_nonexistent_object() -> Result<(), Box<dyn Error>> {
-    let client = create_aws_s3_client().await?;
+async fn test_select_object_content_nonexistent_object() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
     setup_test_bucket(&client).await?;
 
     // Test query on nonexistent object
@@ -381,7 +366,7 @@ async fn test_select_object_content_nonexistent_object() -> Result<(), Box<dyn E
     let csv_output = CsvOutput::builder().build();
     let output_serialization = OutputSerialization::builder().csv(csv_output).build();
 
-    let result = client
+    let error = client
         .select_object_content()
         .bucket(BUCKET)
         .key("nonexistent.csv")
@@ -390,10 +375,10 @@ async fn test_select_object_content_nonexistent_object() -> Result<(), Box<dyn E
         .input_serialization(input_serialization)
         .output_serialization(output_serialization)
         .send()
-        .await;
+        .await
+        .expect_err("selecting a missing object must fail");
 
-    // Verify query fails (expected behavior)
-    assert!(result.is_err(), "Query on nonexistent object should fail");
+    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
 
     Ok(())
 }

crates/e2e_test/src/replication_extension_test.rs

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -57,7 +56,7 @@ use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
-use s3s::header::X_AMZ_REPLICATION_STATUS;
+use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::convert::Infallible;
@@ -2023,6 +2022,7 @@ async fn forward_replication_proxy_request(
     client: &reqwest::Client,
     request_count: &AtomicU64,
     mut replication_enabled: watch::Receiver<bool>,
+    mut held_tagging: watch::Receiver<Option<String>>,
 ) -> Response<Full<bytes::Bytes>> {
     let (parts, body) = request.into_parts();
     let is_replication = parts
@@ -2036,6 +2036,17 @@ async fn forward_replication_proxy_request(
                 return proxy_error_response("replication gate closed");
             }
         }
+        // Content-keyed hold: park only the replication request whose
+        // `x-amz-tagging` matches the held value, letting every other delivery
+        // through, so a test can make one specific (stale) delivery the last
+        // write the backend sees.
+        if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
+            while held_tagging.borrow().as_deref() == Some(tagging) {
+                if held_tagging.changed().await.is_err() {
+                    return proxy_error_response("replication tag hold closed");
+                }
+            }
+        }
     }
 
     let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2070,12 +2081,26 @@ async fn start_replication_counting_proxy(
     backend_url: &str,
     tasks: &mut JoinSet<()>,
 ) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
+    let (proxy_url, request_count, replication_enabled, _held_tagging) =
+        start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
+    Ok((proxy_url, request_count, replication_enabled))
+}
+
+/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
+/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
+/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
+/// counted); all other traffic flows. Send `None` to release them.
+async fn start_replication_counting_proxy_with_tag_hold(
+    backend_url: &str,
+    tasks: &mut JoinSet<()>,
+) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
     let listener = TcpListener::bind("127.0.0.1:0").await?;
     let proxy_url = format!("http://{}", listener.local_addr()?);
     let backend_url = backend_url.to_string();
     let request_count = Arc::new(AtomicU64::new(0));
     let task_request_count = request_count.clone();
     let (replication_enabled, task_replication_enabled) = watch::channel(true);
+    let (held_tagging, task_held_tagging) = watch::channel(None);
     tasks.spawn(async move {
         let client = local_http_client();
         let mut connections = JoinSet::new();
@@ -2087,12 +2112,14 @@ async fn start_replication_counting_proxy(
                     let client = client.clone();
                     let request_count = task_request_count.clone();
                     let replication_enabled = task_replication_enabled.clone();
+                    let held_tagging = task_held_tagging.clone();
                     connections.spawn(async move {
                         let service = service_fn(move |request| {
                             let backend_url = backend_url.clone();
                             let client = client.clone();
                             let request_count = request_count.clone();
                             let replication_enabled = replication_enabled.clone();
+                            let held_tagging = held_tagging.clone();
                             async move {
                                 Ok::<_, Infallible>(
                                     forward_replication_proxy_request(
@@ -2101,6 +2128,7 @@ async fn start_replication_counting_proxy(
                                         &client,
                                         &request_count,
                                         replication_enabled,
+                                        held_tagging,
                                     )
                                     .await,
                                 )
@@ -2113,7 +2141,7 @@ async fn start_replication_counting_proxy(
             }
         }
     });
-    Ok((proxy_url, request_count, replication_enabled))
+    Ok((proxy_url, request_count, replication_enabled, held_tagging))
 }
 
 async fn site_replication_remove(
@@ -6949,6 +6977,395 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
     }
 }
 
+/// Replication status a site reports for one object version via HEAD
+/// (`x-amz-replication-status`), or `None` when the header is absent.
+async fn head_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let head = client
+        .head_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(head.replication_status().map(|status| status.as_str().to_string()))
+}
+
+/// Poll one site until the version's replication status is one of `expected`.
+async fn wait_for_version_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &[&str],
+    site: &str,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let last = head_replication_status(client, bucket, key, version_id).await?;
+        if let Some(status) = last.as_deref()
+            && expected.contains(&status)
+        {
+            return Ok(status.to_string());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+async fn put_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .tagging(
+            aws_sdk_s3::types::Tagging::builder()
+                .tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn get_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string()))
+}
+
+/// Poll one site until the version's `tag_key` equals `expected`.
+async fn wait_for_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    expected: &str,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
+        if observed.as_deref() == Some(expected) {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+/// Tag key the dual-node LWW scenario edits on both sites.
+const LWW_TAG_KEY: &str = "owner";
+
+/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
+/// full quiet window (no late stale delivery flips it back).
+async fn assert_tag_stable_on_both_sites(
+    site_a_client: &Client,
+    site_b_client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &str,
+    quiet: Duration,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + quiet;
+    loop {
+        let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
+        assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(());
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+}
+
+/// Wait until the counting proxy in front of a site has admitted `expected`
+/// replication requests in total (requests held by a closed gate still count).
+async fn wait_for_proxy_replication_requests(
+    counter: &AtomicU64,
+    expected: u64,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = counter.load(Ordering::Relaxed);
+        if observed >= expected {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
+        }
+        sleep(Duration::from_millis(25)).await;
+    }
+}
+
+/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
+/// metadata categories, exercised end to end over the real dual-node
+/// active-active site-replication control plane — sender, worker, status
+/// bookkeeping and persisted failure recovery all participate (the single-server
+/// `replication_lww_receiver_test` only injects authorized replication PUTs).
+///
+/// Scenario on one versioned object:
+/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
+///    newer tag and leave the author COMPLETED / the receiver REPLICA;
+/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
+///    authors a newer edit that reaches A first; releasing the stale delivery
+///    must NOT roll B back — both sites settle on B's value and stay there
+///    through a quiet window, with no FAILED/PENDING status left behind;
+/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
+///    then B returns and the scanner-replayed edit converges both sites forward.
+/// Durable metadata-MRF serialization/reconstruction is covered separately by
+/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
+#[tokio::test]
+async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
+    init_logging();
+
+    match tokio::time::timeout(Duration::from_secs(420), async {
+        // The scanner is fast for the final persisted-failure recovery phase.
+        // Step 2 finishes and proves a quiet stable winner before that phase,
+        // so a later scanner pass cannot mask its stale-delivery assertion.
+        let mut site_env = replication_fast_env();
+        site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+        site_env.extend_from_slice(FAST_SCANNER_ENV);
+
+        let mut site_a_env = RustFSTestEnvironment::new().await?;
+        site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut site_b_env = RustFSTestEnvironment::new().await?;
+        site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut proxy_tasks = JoinSet::new();
+        let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
+        let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
+
+        let site_a_client = site_a_env.create_s3_client();
+        let site_b_client = site_b_env.create_s3_client();
+        let bucket = "site-repl-tag-lww";
+        let key = "lww.txt";
+
+        let add_status = site_replication_add(
+            &site_a_env,
+            &[
+                PeerSite {
+                    name: "lww-site-a".to_string(),
+                    endpoint: site_a_env.url.clone(),
+                    access_key: site_a_env.access_key.clone(),
+                    secret_key: site_a_env.secret_key.clone(),
+                    ..Default::default()
+                },
+                PeerSite {
+                    name: "lww-site-b".to_string(),
+                    endpoint: site_b_env.url.clone(),
+                    access_key: site_b_env.access_key.clone(),
+                    secret_key: site_b_env.secret_key.clone(),
+                    ..Default::default()
+                },
+            ],
+        )
+        .await?;
+        assert!(add_status.success, "unexpected site add result: {add_status:?}");
+
+        let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        // Route both directions through the counting proxies so inbound
+        // replication to B can be held (out-of-order delivery) and observed.
+        for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
+            let mut peer = site_info
+                .sites
+                .iter()
+                .find(|peer| peer.endpoint == *env_url)
+                .ok_or_else(|| format!("site {label} peer missing from replication info"))?
+                .clone();
+            peer.endpoint = proxy_url.clone();
+            peer.sync_state = SyncStatus::Enable;
+            let edit = site_replication_edit(&site_a_env, "", &peer).await?;
+            assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
+        }
+        for env in [&site_a_env, &site_b_env] {
+            wait_for_site_replication_info(env, |info| {
+                info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
+                    && info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
+            })
+            .await?;
+        }
+
+        site_a_client.create_bucket().bucket(bucket).send().await?;
+        wait_for_bucket_on_target(&site_b_client, bucket).await?;
+
+        let version_id = site_a_client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(b"tag lww payload"))
+            .send()
+            .await?
+            .version_id()
+            .ok_or("site A PUT omitted version ID")?
+            .to_string();
+        wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
+
+        // --- 1. reciprocal edits in real order: A then B ----------------------
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
+
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
+            .await?;
+
+        // --- 2. concurrent edits, stale delivery last ------------------------
+        // Both sites edit the same version while each other's delivery is
+        // parked at the peer's inbound proxy (content-keyed: only the
+        // `owner=a2` / `owner=b2` replication PUTs wait, everything else
+        // flows). B's edit is the newer one. Releasing A's stale `a2` first
+        // makes it the last write B sees while A itself still holds `a2`, so
+        // nothing A could re-deliver carries the winner: only receiver-side
+        // LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
+        site_b_held_tagging.send(Some("owner=a2".to_string()))?;
+        site_a_held_tagging.send(Some("owner=b2".to_string()))?;
+        let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
+        let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
+        sleep(Duration::from_millis(50)).await;
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
+        wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
+        assert_eq!(
+            get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("a2")
+        );
+        assert_eq!(
+            get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("b2")
+        );
+
+        // Release the stale a2 delivery onto B: the newer local b2 must
+        // survive, and the delivery itself must still succeed (A reaches
+        // COMPLETED instead of looping through MRF with the stale value).
+        site_b_held_tagging.send(None)?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
+        loop {
+            assert_eq!(
+                get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                    .await?
+                    .as_deref(),
+                Some("b2"),
+                "a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
+            );
+            if tokio::time::Instant::now() >= stale_deadline {
+                break;
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        // Release b2 onto A: the newer edit wins there and both sites settle.
+        // B's own version may legitimately read REPLICA here: the stale inbound
+        // a2 write re-labelled it as a replica write (keeping B's tags); what
+        // must not remain is PENDING/FAILED.
+        site_a_held_tagging.send(None)?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
+            .await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
+            .await?;
+        for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
+            let status = head_replication_status(client, bucket, key, &version_id).await?;
+            assert!(
+                matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
+                "{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
+            );
+        }
+
+        // --- 3. persisted FAILED state survives a source restart ------------
+        site_b_env.stop_server();
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
+        site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
+            .await?;
+
+        // The object itself never forked: one version on each side.
+        tokio::time::timeout(
+            Duration::from_secs(70),
+            assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
+        )
+        .await??;
+        let state = list_replication_state(&site_a_client, bucket).await?;
+        assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
+        assert_eq!(state[0].version_id, version_id);
+
+        proxy_tasks.abort_all();
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err("site replication tagging LWW test timed out".into()),
+    }
+}
 #[tokio::test]
 async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
@@ -7281,11 +7698,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
 async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
     init_logging();
 
-    if !awscurl_available() {
-        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
-        return Ok(());
-    }
-
     let mut source_env = RustFSTestEnvironment::new().await?;
     source_env
         .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)

crates/e2e_test/src/replication_lww_receiver_test.rs

@@ -0,0 +1,155 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
+//! A4/P1-6).
+//!
+//! In an active-active topology both sites' metadata states arrive at the
+//! peer as authorized replication PUTs carrying per-category source
+//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
+//! family). Before the fix the receiver applied them unconditionally, so a
+//! stale delivery overwrote a newer local state and the two sites diverged
+//! permanently while both reported COMPLETED. This test drives one live
+//! `rustfs` server with simulated inbound replication PUTs for the same
+//! object version and asserts the newer tagging state wins regardless of
+//! delivery order, while a stale delivery still succeeds at the object level
+//! (a failure would loop through MRF re-delivering the stale value).
+//!
+//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
+//! is covered by
+//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
+//! this file stays as the fast, single-process receiver check.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
+const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
+const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
+const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
+
+const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
+const T_STALE: &str = "2026-01-01T00:00:01Z";
+const T_LOCAL: &str = "2026-02-01T00:00:00Z";
+const T_NEWER: &str = "2026-03-01T00:00:00Z";
+
+/// Simulated inbound authorized replication PUT: same object version, tags and
+/// the source-authored tagging timestamp carried in transport headers.
+async fn inbound_replication_put(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tags: &str,
+    tagging_timestamp: &str,
+) -> TestResult {
+    let version_id = version_id.to_string();
+    let tagging_timestamp = tagging_timestamp.to_string();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"lww-e2e-body"))
+        .tagging(tags)
+        .customize()
+        .mutate_request(move |req| {
+            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
+            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
+            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
+            req.headers_mut()
+                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
+        })
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await
+        .expect("object tagging should be readable");
+    tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let client = env.create_s3_client();
+
+    let bucket = "replication-lww-receiver";
+    let key = "object";
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // First delivery establishes version V with tags stamped T_LOCAL.
+    let version_id = uuid::Uuid::new_v4().to_string();
+    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "the first delivery must establish the tagged version"
+    );
+
+    // A stale delivery (older source timestamp) must succeed at the object
+    // level but must NOT overwrite the newer tags.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
+    );
+
+    // A newer delivery still converges the version onto the newest state.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("newer"),
+        "a newer inbound delivery must overwrite older tags"
+    );
+
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await?;
+    env.delete_test_bucket(bucket).await.ok();
+    Ok(())
+}

crates/e2e_test/src/security_boundary_test.rs

@@ -21,12 +21,11 @@
 //! - SSRF prevention (internal/private endpoints rejected for tiering)
 //! - Race condition handling (concurrent writes converge without corruption)
 
-use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
 use std::error::Error;
-use tracing::info;
 
 /// Oversized tagging payloads must be rejected by the per-object tag limit.
 ///
@@ -225,16 +224,12 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
 /// outcome — the internal endpoint is not accepted — is asserted here.
 ///
 /// The admin API is exercised via signed `awscurl` requests, matching the
-/// pattern used by the other admin-API E2E tests in this crate; the test is
-/// skipped when `awscurl` is not installed.
+/// pattern used by the other admin-API E2E tests in this crate. The full E2E
+/// lane installs and verifies the pinned `awscurl` prerequisite.
 #[tokio::test]
 async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping tiering URL validation test because awscurl is not available");
-        return Ok(());
-    }
-
+    require_awscurl()?;
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/e2e_test/src/special_chars_test.rs

@@ -404,8 +404,18 @@ mod tests {
         info!("✅ DELETE object succeeded");
 
         // Verify it's deleted
-        let result = client.get_object().bucket(bucket).key(key).send().await;
-        assert!(result.is_err(), "Object should not exist after DELETE");
+        let error = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect_err("Object should not exist after DELETE");
+        assert_eq!(
+            error.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "GET after DELETE must return HTTP 404, got {error:?}"
+        );
 
         // Cleanup
         env.stop_server();

crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs

@@ -15,7 +15,6 @@
 use crate::common::{RustFSTestClusterEnvironment, init_logging};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::CompletedMultipartUpload;
 use tokio::time::{Duration, sleep};
 use tracing::info;
 use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
     }
 }
 
-async fn complete_reports_missing_upload(
+async fn multipart_listing_reports_missing_upload(
     client: &aws_sdk_s3::Client,
     bucket: &str,
     key: &str,
     upload_id: &str,
 ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
-    let result = client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload_id)
-        .multipart_upload(CompletedMultipartUpload::builder().build())
-        .send()
-        .await;
-    match result {
-        Ok(_) => Ok(false),
-        Err(SdkError::ServiceError(err)) => {
-            let code = err.err().meta().code().unwrap_or("");
-            if code == "NoSuchUpload" {
-                Ok(true)
-            } else {
-                Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
-            }
-        }
-        Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
-    }
+    let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
+
+    Ok(!result
+        .uploads()
+        .iter()
+        .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
 }
 
 async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
         let mut all_cleaned = true;
         for (idx, client) in clients.iter().enumerate() {
             let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
-            let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
-            if !(list_parts_missing && complete_missing) {
+            let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
+            if !(list_parts_missing && listing_missing) {
                 info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
                 all_cleaned = false;
                 break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
         1,
         "multipart upload should be visible before background cleanup"
     );
+    assert!(
+        !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
+        "multipart upload listing should contain the upload before background cleanup"
+    );
 
     wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
 

crates/ecstore/src/api/mod.rs

@@ -161,7 +161,7 @@ pub mod bucket {
         pub mod objectlock_sys {
             pub use crate::bucket::object_lock::objectlock_sys::{
                 BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
-                check_retention_for_modification, is_retention_active,
+                check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
             };
         }
     }
@@ -187,22 +187,24 @@ pub mod bucket {
         pub use crate::bucket::replication::{
             BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
             DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
-            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
-            REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
-            REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-            ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
+            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
+            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
+            REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+            REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
             ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
             ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
-            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
-            get_global_replication_stats, get_proxy_targets, init_background_replication,
-            invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
-            replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
-            resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
-            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
+            VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
+            complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
+            get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
+            invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
+            merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
+            read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
+            replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
+            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
+            should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
             unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
             version_purge_status_to_filemeta,
         };
@@ -241,8 +243,8 @@ pub mod cache {
 
 pub mod capacity {
     pub use crate::core::pools::{
-        PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, path2_bucket_object,
-        path2_bucket_object_with_base_path,
+        DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
+        path2_bucket_object, path2_bucket_object_with_base_path,
     };
     pub use crate::store::utils::is_reserved_or_invalid_bucket;
 }
@@ -317,8 +319,6 @@ pub mod config {
 }
 
 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +330,8 @@ pub mod data_usage {
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,
     };
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }
 
 pub mod disk {
@@ -413,9 +415,9 @@ pub mod notification {
     #[cfg(any(test, feature = "test-util"))]
     pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
     pub use crate::services::notification_sys::{
-        CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
-        cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
-        start_remote_version_state_fleet_probe,
+        CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
+        acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
+        new_global_notification_sys, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -424,9 +426,10 @@ pub mod object {
         BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
         GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
-        RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
-        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
-        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
+        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
+        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
+        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -440,6 +443,12 @@ pub mod rebalance {
         RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
         encode_rebalance_stop_propagation_record,
     };
+
+    #[cfg(feature = "test-util")]
+    pub mod test_util {
+        pub use crate::services::rebalance::PausedRebalanceEntryTestFixture;
+        pub use crate::services::rebalance::test_store_with_persisted_rebalance_meta;
+    }
 }
 
 pub mod rio {
@@ -453,14 +462,17 @@ pub mod rpc {
     pub use crate::cluster::rpc::{
         AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
         PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
-        check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+        ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
+        build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
+        encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
         gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
-        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
-        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
-        verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
-        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
+        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+        verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
+        verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -487,7 +499,7 @@ pub mod storage {
     pub use crate::core::pools::HealLifecycleExpiryContext;
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
-        ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
+        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
         init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
         prewarm_local_disk_id_map_with_instance_ctx,
     };

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
 use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
-    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
-    is_rustfs_header, is_standard_header, is_storageclass_header,
+    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
+    is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
 };
 use rustfs_utils::http::{
     SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,6 +1774,22 @@ impl PutObjectOptions {
             Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
         }
 
+        // MinIO PutObjectOptions.Header parity: object tags travel on the
+        // `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
+        // fills `user_tags` from the source version; without this header the
+        // whole-object transport delivered a tagless replica, so tag edits
+        // never reached the peer and the receiver-side LWW comparison
+        // (rustfs/backlog#1953) had nothing to judge.
+        if !self.user_tags.is_empty() {
+            let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
+            tags.sort();
+            let mut encoded = url::form_urlencoded::Serializer::new(String::new());
+            for (key, value) in tags {
+                encoded.append_pair(key, value);
+            }
+            Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
+        }
+
         for (k, v) in &self.user_metadata {
             let Ok(header_value) = HeaderValue::from_str(v) else {
                 warn!("skipping user metadata header with invalid value: {}", k);
@@ -3195,6 +3211,29 @@ mod tests {
         }
     }
 
+    #[test]
+    fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
+        // rustfs/backlog#1953: tag edits replicate through the whole-object
+        // transport, so the source tags must travel on x-amz-tagging.
+        let mut opts = PutObjectOptions::default();
+        opts.user_tags.insert("owner".to_string(), "site a".to_string());
+        opts.user_tags.insert("env".to_string(), "prod".to_string());
+
+        let header = opts.header();
+        let tagging = header
+            .get(AMZ_OBJECT_TAGGING_LOWER)
+            .expect("user tags must be transported on x-amz-tagging")
+            .to_str()
+            .expect("tag header must be ASCII");
+        // Deterministic key order; values are form-urlencoded.
+        assert_eq!(tagging, "env=prod&owner=site+a");
+
+        assert!(
+            PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
+            "a tagless source must not send an empty x-amz-tagging header"
+        );
+    }
+
     #[test]
     fn put_object_headers_omit_unset_replication_timestamps() {
         // UNIX_EPOCH means "never modified on the source"; sending it would
@@ -3425,6 +3464,44 @@ mod tests {
         assert!(mutexes.contains_key("second"));
     }
 
+    #[tokio::test]
+    async fn update_all_targets_publishes_disable_proxy_on_target_client() {
+        // The read-proxy selector (replication_proxy::get_proxy_targets) skips
+        // targets whose TargetClient carries disable_proxy — the persisted
+        // per-target opt-out must survive client publication.
+        let sys = BucketTargetSys::default();
+        let target = |arn: &str, disable_proxy: bool| BucketTarget {
+            arn: arn.to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            disable_proxy,
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets {
+            targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
+        };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let proxied = sys
+            .get_remote_target_client("bucket", "arn:proxied")
+            .await
+            .expect("client should be published");
+        assert!(!proxied.disable_proxy);
+        let opted_out = sys
+            .get_remote_target_client("bucket", "arn:opted-out")
+            .await
+            .expect("client should be published");
+        assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn target_updates_serialize_client_build_through_publication_per_bucket() {
         let sys = Arc::new(BucketTargetSys::default());

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport {
     }
 
     pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
+        let previous = self.clone();
+        let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
         let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
         for (reason, count) in &scan_report.tier_failure_by_reason {
             let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
-            tier_failure_by_reason.insert(*reason, current.max(*count));
+            let merged = if resumed_after_checkpoint {
+                current.saturating_add(*count)
+            } else {
+                current.max(*count)
+            };
+            tier_failure_by_reason.insert(*reason, merged);
         }
         let transition_completed = self.transition_completed;
         let transition_failed = self.transition_failed;
         *self = scan_report.clone();
+        if resumed_after_checkpoint {
+            self.scanned = previous.scanned.saturating_add(scan_report.scanned);
+            self.eligible = previous.eligible.saturating_add(scan_report.eligible);
+            self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
+            self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
+            self.skipped_not_transition = previous
+                .skipped_not_transition
+                .saturating_add(scan_report.skipped_not_transition);
+            self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
+            self.skipped_delete_marker = previous
+                .skipped_delete_marker
+                .saturating_add(scan_report.skipped_delete_marker);
+            self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
+            self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
+            self.skipped_already_transitioned = previous
+                .skipped_already_transitioned
+                .saturating_add(scan_report.skipped_already_transitioned);
+            self.skipped_already_in_flight = previous
+                .skipped_already_in_flight
+                .saturating_add(scan_report.skipped_already_in_flight);
+            self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
+            self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
+            self.skipped_queue_timeout = previous
+                .skipped_queue_timeout
+                .saturating_add(scan_report.skipped_queue_timeout);
+            self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
+        }
+        self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
+        self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
+        self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
+        self.cancelled = previous.cancelled || scan_report.cancelled;
         self.transition_completed = transition_completed;
         self.transition_failed = transition_failed;
-        self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
+        self.tier_failure = self.tier_failure.saturating_add(transition_failed);
         self.tier_failure_by_reason = tier_failure_by_reason;
     }
 
@@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken {
     version_marker: Option<String>,
 }
 
-fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
+pub(super) fn encode_manual_transition_continuation_token(
+    marker: Option<String>,
+    version_marker: Option<String>,
+) -> Option<String> {
     if marker.is_none() && version_marker.is_none() {
         return None;
     }
@@ -4324,6 +4365,16 @@ pub async fn expire_transitioned_object(
     lc_event: &lifecycle::Event,
     _src: &LcEventSrc,
     bucket_incarnation_id: Uuid,
+) -> Result<ObjectInfo, std::io::Error> {
+    expire_transitioned_object_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, None).await
+}
+
+async fn expire_transitioned_object_with_lock_lost_signal(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    bucket_incarnation_id: Uuid,
+    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 ) -> Result<ObjectInfo, std::io::Error> {
     let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
         .await
@@ -4335,6 +4386,9 @@ pub async fn expire_transitioned_object(
     let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id)
         .map_err(std::io::Error::other)?;
     opts.add_namespace_lock_guard(&publication_guard);
+    if let Some(signal) = lock_lost_signal {
+        opts.add_namespace_lock_lost_signal(signal);
+    }
     opts.delete_replication_config_snapshot = Some(Arc::new(snapshot));
     //let tags = LcAuditEvent::new(src, lcEvent).Tags();
     if lc_event.action.delete_restored() {
@@ -4993,12 +5047,32 @@ pub async fn apply_expiry_on_transitioned_object(
     lc_event: &lifecycle::Event,
     src: &LcEventSrc,
     bucket_incarnation_id: Uuid,
+) -> bool {
+    apply_expiry_on_transitioned_object_with_lock_lost_signal(api, oi, lc_event, src, bucket_incarnation_id, None).await
+}
+
+async fn apply_expiry_on_transitioned_object_with_lock_lost_signal(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    _src: &LcEventSrc,
+    bucket_incarnation_id: Uuid,
+    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 ) -> bool {
     if lc_event.action.delete_all() {
-        return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
+        return apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
+            api,
+            oi,
+            lc_event,
+            bucket_incarnation_id,
+            lock_lost_signal,
+        )
+        .await;
     }
     let time_ilm = Metrics::time_ilm(lc_event.action);
-    if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
+    if let Err(_err) =
+        expire_transitioned_object_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, lock_lost_signal).await
+    {
         return false;
     }
     time_ilm(1)();
@@ -5012,6 +5086,16 @@ pub async fn apply_expiry_on_non_transitioned_objects(
     lc_event: &lifecycle::Event,
     _src: &LcEventSrc,
     bucket_incarnation_id: Uuid,
+) -> bool {
+    apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, None).await
+}
+
+async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    bucket_incarnation_id: Uuid,
+    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 ) -> bool {
     let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
         return false;
@@ -5042,6 +5126,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(
         ..Default::default()
     };
     opts.add_namespace_lock_guard(&publication_guard);
+    if let Some(signal) = lock_lost_signal {
+        opts.add_namespace_lock_lost_signal(signal);
+    }
 
     if lc_event.action.delete_versioned() {
         opts.version_id = oi.version_id.map(|v| v.to_string());
@@ -5123,6 +5210,61 @@ async fn enqueue_expiry_rule_with_incarnation(
     expiry_state.enqueue_by_days(oi, event, src, bucket_incarnation_id)
 }
 
+fn lifecycle_expiry_object_matches(current: &ObjectInfo, expected: &ObjectInfo) -> bool {
+    current.version_id == expected.version_id
+        && current.data_dir == expected.data_dir
+        && current.mod_time == expected.mod_time
+        && current.etag == expected.etag
+        && current.delete_marker == expected.delete_marker
+        && current.transitioned_object.name == expected.transitioned_object.name
+        && current.transitioned_object.version_id == expected.transitioned_object.version_id
+        && current.transitioned_object.tier == expected.transitioned_object.tier
+        && current.transitioned_object.status == expected.transitioned_object.status
+        && current.restore_expires == expected.restore_expires
+}
+
+pub(crate) async fn apply_expiry_rule_for_data_movement(
+    api: Arc<ECStore>,
+    event: &lifecycle::Event,
+    src: &LcEventSrc,
+    oi: &ObjectInfo,
+    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
+) -> bool {
+    let Ok(_lifecycle_guard) = api.acquire_bucket_lifecycle_read_lock(&oi.bucket).await else {
+        return false;
+    };
+    let Ok(bucket_incarnation_id) = api.bucket_incarnation_id_from_disk(&oi.bucket).await else {
+        return false;
+    };
+    let current = match api
+        .get_object_info(
+            &oi.bucket,
+            &oi.name,
+            &ObjectOptions {
+                version_id: oi.version_id.map(|version_id| version_id.to_string()),
+                versioned: oi.version_id.is_some(),
+                expected_bucket_incarnation_id: Some(bucket_incarnation_id),
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(current) => current,
+        Err(_) => return false,
+    };
+    if !lifecycle_expiry_object_matches(&current, oi) {
+        return false;
+    }
+
+    if oi.transitioned_object.status.is_empty() {
+        apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(api, oi, event, bucket_incarnation_id, lock_lost_signal)
+            .await
+    } else {
+        apply_expiry_on_transitioned_object_with_lock_lost_signal(api, oi, event, src, bucket_incarnation_id, lock_lost_signal)
+            .await
+    }
+}
+
 pub(crate) async fn apply_expiry_rule_in(api: Arc<ECStore>, event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
     let Ok(_lifecycle_guard) = api.acquire_bucket_lifecycle_read_lock(&oi.bucket).await else {
         return false;
@@ -5146,17 +5288,7 @@ pub(crate) async fn apply_expiry_rule_in(api: Arc<ECStore>, event: &lifecycle::E
         Ok(current) => current,
         Err(_) => return false,
     };
-    if current.version_id != oi.version_id
-        || current.data_dir != oi.data_dir
-        || current.mod_time != oi.mod_time
-        || current.etag != oi.etag
-        || current.delete_marker != oi.delete_marker
-        || current.transitioned_object.name != oi.transitioned_object.name
-        || current.transitioned_object.version_id != oi.transitioned_object.version_id
-        || current.transitioned_object.tier != oi.transitioned_object.tier
-        || current.transitioned_object.status != oi.transitioned_object.status
-        || current.restore_expires != oi.restore_expires
-    {
+    if !lifecycle_expiry_object_matches(&current, oi) {
         return false;
     }
     enqueue_expiry_rule_with_incarnation(event, src, oi, bucket_incarnation_id).await
@@ -9136,6 +9268,7 @@ mod tests {
         assert_eq!(loaded.report.scanned, 37);
         assert_eq!(loaded.report.eligible, 11);
         assert_eq!(loaded.report.enqueued, 5);
+        assert_eq!(loaded.cursor_revision, Some(37));
         assert!(loaded.lease_expires_at_unix_nanos > 0);
         let token = loaded
             .report
@@ -9154,6 +9287,30 @@ mod tests {
         assert_eq!(admission.lease_id, loaded.lease_id);
         assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
 
+        let mut same_marker_report = report.clone();
+        same_marker_report.scanned += 1;
+        persist_manual_transition_page_checkpoint(
+            &checkpoint_options,
+            &same_marker_report,
+            Some("logs/page-end".to_string()),
+            Some("opaque-next-version".to_string()),
+        )
+        .await
+        .expect("same-marker version checkpoint should persist through the durable progress sink");
+        let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
+            .await
+            .expect("same-marker version checkpoint should reload");
+        assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
+        let (_, version_marker) = decode_manual_transition_continuation_token(
+            same_marker_checkpointed
+                .report
+                .continuation_token
+                .as_deref()
+                .expect("same-marker version checkpoint should persist a cursor"),
+        )
+        .expect("same-marker version cursor should decode");
+        assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
+
         create_test_bucket(&ecstore, &bucket).await;
         let lifecycle_xml = format!(
             r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9367,7 @@ mod tests {
         assert_eq!(checkpointed.report.scanned, 1000);
         assert_eq!(checkpointed.report.eligible, 1000);
         assert_eq!(checkpointed.report.dry_run_eligible, 1000);
+        assert_eq!(checkpointed.cursor_revision, Some(1000));
         let token = checkpointed
             .report
             .continuation_token

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use uuid::Uuid;
 
+#[cfg(test)]
+use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{
     ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
 };
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::{
+    MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
+    MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
@@ -34,10 +40,10 @@ use crate::store::ECStore;
 pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
 pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
 pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
-pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
-pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
-pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
-pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
+pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
 pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
 pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
 pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord {
     pub updated_at_unix_nanos: i128,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub completed_at_unix_nanos: Option<i128>,
+    #[serde(default, skip_serializing)]
+    pub cursor_revision: Option<u64>,
     pub report: ManualTransitionRunReport,
     pub queue_snapshot: ManualTransitionQueueSnapshot,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +232,7 @@ impl ManualTransitionJobRecord {
             created_at_unix_nanos: now,
             updated_at_unix_nanos: now,
             completed_at_unix_nanos: None,
+            cursor_revision: None,
             report: ManualTransitionRunReport {
                 bucket: bucket.to_string(),
                 prefix: options.prefix.clone(),
@@ -238,7 +247,7 @@ impl ManualTransitionJobRecord {
 
     pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         self.scan_completed = true;
-        self.report.merge_scan_report_preserving_worker(&report);
+        self.merge_scan_report(&report);
         self.queue_snapshot = queue_snapshot;
         self.error = None;
         self.mark_terminal_if_worker_drained();
@@ -316,7 +325,7 @@ impl ManualTransitionJobRecord {
             }
         }
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
     }
 
@@ -359,14 +368,14 @@ impl ManualTransitionJobRecord {
         self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
         self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
         true
     }
 
     pub fn mark_cancel_requested(&mut self) {
         self.cancel_requested = true;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
     }
 
     pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +389,7 @@ impl ManualTransitionJobRecord {
     pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
         if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
             self.lease_expires_at_unix_nanos = 0;
-            self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+            self.advance_updated_at();
         }
     }
 
@@ -398,7 +407,7 @@ impl ManualTransitionJobRecord {
 
     pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
         self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
         self.queue_snapshot = queue_snapshot;
     }
@@ -438,11 +447,16 @@ impl ManualTransitionJobRecord {
 
     pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         if self.state == ManualTransitionJobState::Running {
-            self.report.merge_scan_report_preserving_worker(&report);
+            self.merge_scan_report(&report);
             self.renew_lease(queue_snapshot);
         }
     }
 
+    fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
+        self.report.merge_scan_report_preserving_worker(report);
+        self.cursor_revision = manual_transition_cursor_revision(&self.report);
+    }
+
     pub fn mark_unknown_if_unowned(&mut self) {
         if self.state == ManualTransitionJobState::Running {
             self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
     }
 
     fn mark_updated_terminal(&mut self) {
+        self.advance_updated_at();
+        self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
+    }
+
+    fn advance_updated_at(&mut self) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
-        self.completed_at_unix_nanos = Some(now);
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
     }
 
     fn mark_terminal_if_worker_drained(&mut self) {
@@ -543,6 +561,7 @@ impl ManualTransitionJobRecord {
         if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
             job.report.cancelled = true;
         }
+        job.cursor_revision = manual_transition_cursor_revision(&job.report);
         job.validate()?;
         Ok(job)
     }
@@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
 pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current(
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
     config_boundary::save_config_with_opts_quiet(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 /// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     match config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
             Err(Error::PreconditionFailed)
         }
         result => result,
-    }
+    }?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
     job_id: Uuid,
     lease_id: Uuid,
 ) -> EcstoreResult<bool> {
-    let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
-        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
+    let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
+        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
         Ok(_) => return Ok(false),
         Err(Error::ConfigNotFound) => return Ok(true),
         Err(err) => return Err(err),
     };
     let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
+    let data = serde_json::to_vec(&admission).map_err(Error::other)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config_if_match(api, &object, &etag).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
         Err(Error::PreconditionFailed) => Ok(false),
@@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
     Error::other(err)
 }
 
+fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
+    report.continuation_token.as_ref()?;
+    (report.scanned > 0).then_some(report.scanned)
+}
+
 pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
     OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
 }
@@ -2210,6 +2240,76 @@ mod tests {
         );
     }
 
+    #[test]
+    fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let first_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                lifecycle_config_found: true,
+                scanned: 1000,
+                eligible: 800,
+                enqueued: 50,
+                dry_run_eligible: 10,
+                skipped_not_transition: 2,
+                skipped_tier: 3,
+                skipped_delete_marker: 4,
+                skipped_directory: 5,
+                skipped_replication: 6,
+                skipped_already_transitioned: 7,
+                skipped_already_in_flight: 8,
+                skipped_queue_full: 9,
+                skipped_queue_closed: 10,
+                skipped_queue_timeout: 11,
+                tier_failure: 12,
+                truncated_by_duration: true,
+                continuation_token: first_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        let next_token =
+            encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 3,
+                eligible: 2,
+                enqueued: 1,
+                dry_run_eligible: 1,
+                skipped_not_transition: 1,
+                tier_failure: 1,
+                continuation_token: next_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        assert_eq!(record.report.scanned, 1003);
+        assert_eq!(record.report.eligible, 802);
+        assert_eq!(record.report.enqueued, 51);
+        assert_eq!(record.report.dry_run_eligible, 11);
+        assert_eq!(record.report.skipped_not_transition, 3);
+        assert_eq!(record.report.skipped_tier, 3);
+        assert_eq!(record.report.skipped_delete_marker, 4);
+        assert_eq!(record.report.skipped_directory, 5);
+        assert_eq!(record.report.skipped_replication, 6);
+        assert_eq!(record.report.skipped_already_transitioned, 7);
+        assert_eq!(record.report.skipped_already_in_flight, 8);
+        assert_eq!(record.report.skipped_queue_full, 9);
+        assert_eq!(record.report.skipped_queue_closed, 10);
+        assert_eq!(record.report.skipped_queue_timeout, 11);
+        assert_eq!(record.report.tier_failure, 13);
+        assert!(record.report.lifecycle_config_found);
+        assert!(record.report.truncated_by_duration);
+        assert_eq!(record.report.continuation_token, next_token);
+        assert_eq!(record.cursor_revision, Some(1003));
+    }
+
     #[test]
     fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
         let options = ManualTransitionRunOptions::default();
@@ -2577,6 +2677,98 @@ mod tests {
         assert!(decoded.report.tier_failure_by_reason.is_empty());
     }
 
+    #[test]
+    fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 9,
+                continuation_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        let encoded = record.encode().expect("job record should encode");
+        let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        value["job"]
+            .as_object_mut()
+            .expect("job should be object")
+            .remove("cursor_revision");
+        let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
+        value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
+        let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
+
+        let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
+
+        assert_eq!(decoded.cursor_revision, Some(9));
+    }
+
+    #[test]
+    fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyPersistedManualTransitionJobRecord {
+            schema: String,
+            content_sha256: String,
+            job: LegacyManualTransitionJobRecord,
+        }
+
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyManualTransitionJobRecord {
+            job_id: Uuid,
+            scope_key: String,
+            bucket: String,
+            prefix: String,
+            tier: Option<String>,
+            dry_run: bool,
+            max_objects: Option<u64>,
+            max_duration: Option<std::time::Duration>,
+            owner_id: String,
+            lease_id: Uuid,
+            lease_expires_at_unix_nanos: i128,
+            state: ManualTransitionJobState,
+            scan_completed: bool,
+            cancel_requested: bool,
+            created_at_unix_nanos: i128,
+            updated_at_unix_nanos: i128,
+            completed_at_unix_nanos: Option<i128>,
+            report: ManualTransitionRunReport,
+            queue_snapshot: ManualTransitionQueueSnapshot,
+            error: Option<String>,
+        }
+
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 7,
+                continuation_token: continuation_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        assert_eq!(record.cursor_revision, Some(7));
+
+        let encoded = record.encode().expect("job record should encode");
+        let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        assert!(value["job"].get("cursor_revision").is_none());
+        let legacy: LegacyPersistedManualTransitionJobRecord =
+            serde_json::from_slice(&encoded).expect("old reader should accept new job record");
+
+        assert_eq!(legacy.job.job_id, record.job_id);
+        assert_eq!(legacy.job.report.continuation_token, continuation_token);
+    }
+
     #[test]
     fn manual_transition_job_record_rejects_unknown_report_fields() {
         let options = ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
 mod config_boundary;
 pub mod core;
+mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;
 pub mod transition_transaction;
+
+pub(crate) use durable_namespace::{
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
+};

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, warn};
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
     Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
-pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
+pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
         )
         .await?;
     }
+    let path = tier_delete_journal_object_name(je);
+    let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
+    let target_pool_indices = api
+        .record_durable_ilm_decommission_terminal_target_pools(&path, &data)
+        .await
+        .map_err(std::io::Error::other)?;
+    if let Some(target_pool_indices) = target_pool_indices {
+        for target_pool_idx in target_pool_indices {
+            match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
+                Ok(()) | Err(Error::ConfigNotFound) => {}
+                Err(err) => return Err(std::io::Error::other(err)),
+            }
+        }
+        return Ok(());
+    }
     remove_tier_delete_journal_entry(api, je).await
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -21,6 +21,7 @@ use tracing::{debug, warn};
 use uuid::Uuid;
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
-pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
+pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
 pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
 
 pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
     let object =
         transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
     let data = transaction.encode().map_err(transition_transaction_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
     TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
 }
 
-pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
-    let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
+pub(crate) async fn delete_transition_transaction_record(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<()> {
+    let object =
+        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
+    let data = transaction.encode().map_err(transition_transaction_store_error)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config(api, &object).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
         Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
     if probe != TransitionOperatorProbe::Missing {
         return Err(TransitionOperatorError::CandidateNotMissing(probe));
     }
-    delete_transition_transaction_record(api, transaction_id)
+    delete_transition_transaction_record(api, &transaction)
         .await
         .map_err(TransitionOperatorError::Store)
 }
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
     match transaction.state {
         TransitionTransactionState::Uploaded => {
             delete_transition_remote_candidate(api.clone(), transaction).await?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
             Ok(true) => {
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
             }
             Ok(false) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) if transition_source_is_missing(&err) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
         TransitionTransactionState::LocalCommitStarted => {
             match local_commit_matches_transaction(api.clone(), transaction).await {
                 Ok(true) => {
-                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    delete_transition_transaction_record(api, transaction).await?;
                     Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
                 }
                 Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
             }
         }
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
         .map_err(Error::other)?
     {
         TransitionCandidateProbe::Missing => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
             )
             .await
             .map_err(Error::other)?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
         .map_err(transition_transaction_store_error)?;
     save_transition_transaction_record(api.clone(), &cleanup).await?;
     delete_transition_remote_candidate(api.clone(), &cleanup).await?;
-    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    delete_transition_transaction_record(api, &cleanup).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1076,6 +1076,13 @@ pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfigurat
     bucket_meta_sys.get_versioning_config(bucket).await
 }
 
+pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Result<bool> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
+
+    bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
+}
+
 pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1914,6 +1921,18 @@ impl BucketMetadataSys {
         }
     }
 
+    async fn has_authoritative_never_versioned_state(&self, bucket: &str) -> Result<bool> {
+        let BucketMetadataAuthority::Authoritative(metadata) = self.get_metadata_authority(bucket).await? else {
+            return Ok(false);
+        };
+
+        if metadata.versioning_config.is_none() && !metadata.versioning_config_xml.is_empty() {
+            return Err(Error::other("persisted bucket versioning configuration is invalid"));
+        }
+
+        Ok(metadata.versioning_config.is_none() && metadata.versioning_config_xml.is_empty())
+    }
+
     pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
         let bm = match self.get_metadata_authority(bucket).await? {
             BucketMetadataAuthority::Authoritative(bm) => bm,
@@ -2469,6 +2488,10 @@ mod tests {
             sys.get_versioning_config(bucket).await.is_err(),
             "malformed versioning metadata must block destructive requests"
         );
+        assert!(
+            sys.has_authoritative_never_versioned_state(bucket).await.is_err(),
+            "malformed versioning metadata must not enable listing shortcuts"
+        );
         assert!(
             sys.get_replication_config(bucket).await.is_err(),
             "malformed replication metadata must not be reported as ConfigNotFound"
@@ -2492,8 +2515,50 @@ mod tests {
             sys.get_object_lock_config_state("authoritative-empty").await.unwrap(),
             ObjectLockConfigState::ConfirmedAbsent
         ));
+        assert!(
+            sys.has_authoritative_never_versioned_state("authoritative-empty")
+                .await
+                .unwrap(),
+            "authoritative config absence should identify a never-versioned bucket"
+        );
         assert!(matches!(sys.get_bucket_policy("authoritative-empty").await, Err(Error::ConfigNotFound)));
 
+        let mut versioned = BucketMetadata::new("authoritative-versioned");
+        versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
+        versioned.versioning_config = Some(VersioningConfiguration {
+            status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
+            ..Default::default()
+        });
+        sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
+        assert!(
+            !sys.has_authoritative_never_versioned_state("authoritative-versioned")
+                .await
+                .unwrap(),
+            "versioned buckets must retain delete-marker visibility probes"
+        );
+
+        let mut ambiguous = BucketMetadata::new("authoritative-ambiguous-versioning");
+        ambiguous.versioning_config = Some(VersioningConfiguration::default());
+        sys.set("authoritative-ambiguous-versioning".to_string(), Arc::new(ambiguous))
+            .await;
+        assert!(
+            !sys.has_authoritative_never_versioned_state("authoritative-ambiguous-versioning")
+                .await
+                .unwrap(),
+            "ambiguous versioning metadata must retain delete-marker visibility probes"
+        );
+
+        sys.fabricated_metadata
+            .write()
+            .await
+            .insert("fabricated-versioning".to_string());
+        assert!(
+            !sys.has_authoritative_never_versioned_state("fabricated-versioning")
+                .await
+                .unwrap(),
+            "fabricated metadata must retain delete-marker visibility probes"
+        );
+
         for dir in &dirs {
             std::fs::create_dir_all(dir.path().join("policy-only-legacy")).unwrap();
         }

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -15,7 +15,7 @@
 use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
 use crate::bucket::object_lock::objectlock;
 use crate::error::{Error, Result, StorageError};
-use crate::object_api::ObjectInfo;
+use crate::object_api::{ObjectInfo, ObjectOptions};
 use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use std::sync::Arc;
@@ -136,12 +136,50 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
 
 /// Check if an object has legal hold enabled.
 /// Returns true if legal hold is ON.
-#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
     let lhold = objectlock::get_object_legalhold_meta(user_defined);
     matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
 }
 
+/// Whether an authorized replication write (`ObjectOptions::replication_request`)
+/// may overwrite a locked destination version.
+///
+/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
+/// skips the existing-version check for replicas), and a source-side hold
+/// release or retention change reaches this site only through this write. The
+/// overwrite is allowed only when the write carries the source timestamp of
+/// every category that currently locks the version, so receiver-side LWW
+/// (`merge_replication_metadata_lww`) judges each of them: a category locked
+/// more recently here is kept, otherwise the source's newer state wins. A write
+/// without that timestamp carries no source decision for the category — the
+/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
+///
+/// The locking categories come from the same authoritative evaluation as the
+/// commit-time WORM gate (`check_object_lock_for_deletion_with_state`): the
+/// bucket default retention locks a version that carries no explicit
+/// retention keys, so it is judged here too rather than read off the keys.
+/// Malformed persisted lock metadata or a non-authoritative bucket
+/// configuration is an error, never a pass.
+pub fn replication_write_may_pass_worm_gate(
+    state: &ObjectLockConfigState,
+    obj_info: &ObjectInfo,
+    opts: &ObjectOptions,
+) -> Result<bool> {
+    if !opts.replication_request {
+        return Ok(false);
+    }
+    if obj_info.delete_marker {
+        // Delete markers are never locked (same as the WORM gate).
+        return Ok(true);
+    }
+    let config = object_lock_config_from_state(state)?;
+    if legal_hold_locks(obj_info)? && opts.replication_legalhold_timestamp.is_none() {
+        return Ok(false);
+    }
+    let retention_locked = active_retention(config, obj_info)?.is_some();
+    Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
+}
+
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -239,69 +277,101 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
         return Ok(None);
     }
 
-    if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
-        if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
-            return Ok(Some(ObjectLockBlockReason::LegalHold));
-        }
-        if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
-            return Err(Error::other("persisted object legal-hold metadata is invalid"));
-        }
+    if legal_hold_locks(obj_info)? {
+        return Ok(Some(ObjectLockBlockReason::LegalHold));
     }
 
-    let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
-    let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
-    let explicit_ret = match (mode, retain_until) {
-        (None, None) => None,
+    if let Some((mode_str, retain_until)) = active_retention(config, obj_info)?
+        && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
+    {
+        return Ok(Some(reason));
+    }
+
+    Ok(None)
+}
+
+/// A cleared retention / legal hold is persisted as empty strings (the MinIO
+/// on-disk shape, `parse_object_lock_retention`); read it as "no lock" rather
+/// than as corrupt metadata.
+fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a String> {
+    obj_info.user_defined.get(key).filter(|value| !value.is_empty())
+}
+
+/// Whether the version's persisted legal hold is ON. Any other non-empty
+/// value than ON/OFF is malformed metadata and fails closed.
+fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
+    let Some(status) = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) else {
+        return Ok(false);
+    };
+    if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
+        return Ok(true);
+    }
+    if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
+        return Err(Error::other("persisted object legal-hold metadata is invalid"));
+    }
+    Ok(false)
+}
+
+/// The retention that currently locks the version, if any: the explicit
+/// persisted retention when the keys are present, otherwise the bucket
+/// default retention computed from the version's modification time. Returns
+/// `(mode, retain_until)` only while the retention is still active.
+fn active_retention<'a>(
+    config: Option<&'a ObjectLockConfiguration>,
+    obj_info: &ObjectInfo,
+) -> Result<Option<(&'a str, OffsetDateTime)>> {
+    let mode = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_MODE.as_str());
+    let retain_until = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
+    match (mode, retain_until) {
+        (None, None) => {}
         (Some(mode), Some(retain_until)) => {
             let mode =
                 objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
             let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
                 .map(Date::from)
                 .map_err(|_| Error::other("persisted object retention date is invalid"))?;
-            Some((mode, retain_until))
+            let mode_str = match mode.as_str() {
+                ObjectLockRetentionMode::COMPLIANCE => ObjectLockRetentionMode::COMPLIANCE,
+                ObjectLockRetentionMode::GOVERNANCE => ObjectLockRetentionMode::GOVERNANCE,
+                _ => return Err(Error::other("persisted object retention mode is invalid")),
+            };
+            return Ok(is_retention_active(mode_str, Some(&retain_until)).then(|| (mode_str, OffsetDateTime::from(retain_until))));
         }
         _ => return Err(Error::other("persisted object retention metadata is incomplete")),
+    }
+
+    let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
+        return Ok(None);
     };
-
-    if let Some((mode, retain_until)) = &explicit_ret {
-        let mode_str = mode.as_str();
-        if is_retention_active(mode_str, Some(retain_until))
-            && let Some(reason) =
-                check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
-        {
-            return Ok(Some(reason));
-        }
+    let Some(mode) = &default_retention.mode else {
+        return Ok(None);
+    };
+    let mode_str = mode.as_str();
+    if mode_str != ObjectLockRetentionMode::COMPLIANCE && mode_str != ObjectLockRetentionMode::GOVERNANCE {
+        return Ok(None);
     }
+    // Calculate retention expiration date from object modification time
+    let mod_time = obj_info
+        .mod_time
+        .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
+    let now = objectlock::utc_now_ntp();
+    let retain_until = if let Some(days) = default_retention.days {
+        mod_time.saturating_add(time::Duration::days(i64::from(days)))
+    } else {
+        let years = default_retention
+            .years
+            .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
+        add_years(mod_time, years)
+    };
+    Ok((retain_until.unix_timestamp() > now.unix_timestamp()).then_some((mode_str, retain_until)))
+}
 
-    if explicit_ret.is_none()
-        && let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
-        && let Some(mode) = &default_retention.mode
-    {
-        let mode_str = mode.as_str();
-        if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
-            // Calculate retention expiration date from object modification time
-            let mod_time = obj_info
-                .mod_time
-                .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
-            let now = objectlock::utc_now_ntp();
-            let retain_until = if let Some(days) = default_retention.days {
-                mod_time.saturating_add(time::Duration::days(i64::from(days)))
-            } else {
-                let years = default_retention
-                    .years
-                    .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
-                add_years(mod_time, years)
-            };
-
-            if retain_until.unix_timestamp() > now.unix_timestamp()
-                && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
-            {
-                return Ok(Some(reason));
-            }
-        }
+fn object_lock_config_from_state(state: &ObjectLockConfigState) -> Result<Option<&ObjectLockConfiguration>> {
+    match state {
+        ObjectLockConfigState::Configured { config, .. } => Ok(Some(config)),
+        ObjectLockConfigState::ConfirmedAbsent => Ok(None),
+        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
     }
-
-    Ok(None)
 }
 
 pub(crate) fn check_object_lock_for_deletion_with_state(
@@ -309,13 +379,7 @@ pub(crate) fn check_object_lock_for_deletion_with_state(
     obj_info: &ObjectInfo,
     bypass_governance: bool,
 ) -> Result<Option<ObjectLockBlockReason>> {
-    match state {
-        ObjectLockConfigState::Configured { config, .. } => {
-            check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
-        }
-        ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
-        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
-    }
+    check_object_lock_for_deletion_with_config(object_lock_config_from_state(state)?, obj_info, bypass_governance)
 }
 
 /// Compatibility wrapper for callers that predate fallible metadata lookup.
@@ -486,6 +550,210 @@ mod tests {
         }
     }
 
+    fn replication_opts(hold_ts: bool, retention_ts: bool) -> ObjectOptions {
+        ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        }
+    }
+
+    fn lock_metadata(entries: &[&[(&str, &str)]]) -> std::collections::HashMap<String, String> {
+        entries
+            .iter()
+            .flat_map(|entries| entries.iter())
+            .map(|(key, value)| (key.to_string(), value.to_string()))
+            .collect()
+    }
+
+    fn lock_object_info(user_defined: std::collections::HashMap<String, String>) -> ObjectInfo {
+        ObjectInfo {
+            user_defined: Arc::new(user_defined),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        }
+    }
+
+    /// A replication write passes the WORM gate only when it carries the
+    /// source timestamp of every category that currently locks the version.
+    #[test]
+    fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
+        let retention = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
+        ];
+        let expired = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
+        ];
+        let absent = ObjectLockConfigState::ConfirmedAbsent;
+        let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
+            replication_write_may_pass_worm_gate(state, &lock_object_info(lock_metadata(entries)), opts)
+                .expect("well-formed lock metadata must be judged")
+        };
+
+        assert!(passes(&absent, &[&hold, &retention], &replication_opts(true, true)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(false, true)));
+
+        assert!(passes(&absent, &[&hold], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold], &replication_opts(false, true)));
+        assert!(passes(&absent, &[&retention], &replication_opts(false, true)));
+        assert!(!passes(&absent, &[&retention], &replication_opts(true, false)));
+
+        // Expired retention and a released hold no longer lock anything.
+        assert!(passes(
+            &absent,
+            &[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
+            &replication_opts(false, false)
+        ));
+
+        // Never for a non-replication write, whatever it carries.
+        let local = ObjectOptions {
+            replication_request: false,
+            ..replication_opts(true, true)
+        };
+        assert!(!passes(&absent, &[&hold], &local));
+    }
+
+    /// The bucket default retention locks a version that carries no explicit
+    /// retention keys (`check_object_lock_for_deletion_with_config` judges it
+    /// from the modification time), so the replication bypass must demand the
+    /// retention source timestamp for it too — a tagging-only replication
+    /// write must not overwrite the default-protected version unjudged.
+    #[test]
+    fn replication_write_under_bucket_default_retention_requires_retention_timestamp() {
+        use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER};
+
+        for mode in [ObjectLockRetentionMode::COMPLIANCE, ObjectLockRetentionMode::GOVERNANCE] {
+            let state = ObjectLockConfigState::Configured {
+                config: default_retention_config(mode),
+                updated_at: OffsetDateTime::now_utc(),
+            };
+            let no_keys = lock_object_info(std::collections::HashMap::new());
+            assert!(
+                check_object_lock_for_deletion_with_state(&state, &no_keys, false)
+                    .expect("default retention must be judged")
+                    .is_some(),
+                "{mode}: the gate must report the default retention lock"
+            );
+
+            let tagging_only = ObjectOptions {
+                replication_request: true,
+                replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            assert!(
+                !replication_write_may_pass_worm_gate(&state, &no_keys, &tagging_only).expect("judged"),
+                "{mode}: a tagging-only replication write must not pass the default retention lock"
+            );
+            assert!(
+                replication_write_may_pass_worm_gate(&state, &no_keys, &replication_opts(false, true)).expect("judged"),
+                "{mode}: the retention source timestamp lets LWW judge the default retention"
+            );
+
+            // Default retention plus a legal hold: both categories need a timestamp.
+            let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
+            assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
+
+            // A version whose default retention has already expired (old
+            // mod_time) is not locked by the default any more.
+            let expired_default = ObjectInfo {
+                mod_time: Some(make_datetime(2000, 1, 1)),
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &expired_default, &tagging_only).expect("judged"));
+
+            // A delete marker is never locked, so there is nothing to judge.
+            let delete_marker = ObjectInfo {
+                delete_marker: true,
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
+
+            // Cleared (empty) explicit keys fall back to the bucket default.
+            let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
+        }
+    }
+
+    /// The replication bypass never judges from a non-authoritative bucket
+    /// state or malformed persisted lock metadata; both are errors, not a pass.
+    #[test]
+    fn replication_write_worm_gate_fails_closed_on_unverifiable_lock_state() {
+        use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
+
+        let opts = replication_opts(true, true);
+        let err = replication_write_may_pass_worm_gate(
+            &ObjectLockConfigState::Fabricated,
+            &lock_object_info(std::collections::HashMap::new()),
+            &opts,
+        )
+        .expect_err("fabricated bucket lock metadata must not be judged");
+        assert!(err.to_string().contains("not authoritative"));
+
+        let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
+        let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
+            .expect_err("malformed legal hold must not be judged");
+        assert!(err.to_string().contains("legal-hold"));
+
+        let state = ObjectLockConfigState::Configured {
+            config: default_retention_config(ObjectLockRetentionMode::COMPLIANCE),
+            updated_at: OffsetDateTime::now_utc(),
+        };
+        let no_mod_time = ObjectInfo::default();
+        let err = replication_write_may_pass_worm_gate(&state, &no_mod_time, &opts)
+            .expect_err("default retention without a modification time must not be judged");
+        assert!(err.to_string().contains("modification time"));
+    }
+
+    /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
+    /// lock keys as empty strings (the MinIO on-disk shape, see
+    /// `parse_object_lock_retention`); that is "no lock", not corruption, and
+    /// must not wedge later explicit-version PUTs or deletes
+    /// (rustfs/backlog#1953).
+    #[test]
+    fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let cases: [(&str, &[&str]); 3] = [
+            (
+                "cleared retention",
+                &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
+            ),
+            ("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
+            (
+                "all cleared",
+                &[
+                    AMZ_OBJECT_LOCK_MODE_LOWER,
+                    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+                    AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
+                ],
+            ),
+        ];
+
+        for (case, keys) in cases {
+            let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
+            let obj_info = ObjectInfo {
+                user_defined: Arc::new(user_defined),
+                ..Default::default()
+            };
+
+            let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
+            assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
+        }
+    }
+
     #[test]
     fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
         let mut user_defined = std::collections::HashMap::new();

crates/ecstore/src/bucket/replication/mod.rs

@@ -44,11 +44,14 @@ mod replication_versioning_boundary;
 mod runtime_boundary;
 
 pub use replication_config_boundary::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
+    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
+    is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -13,9 +13,12 @@
 // limitations under the License.
 
 pub use rustfs_replication::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
+    ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
+    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -39,7 +39,7 @@ use super::replication_resync_boundary::{
 };
 use super::replication_resyncer::{
     ReplicationResyncer, get_heal_replicate_object_info, replicate_delete, replicate_delete_with_outcome, replicate_object,
-    replicate_object_with_outcome, save_resync_status,
+    replicate_object_with_outcome, update_resync_status_cas,
 };
 use super::replication_state::ReplicationStats;
 use super::replication_storage_boundary::{
@@ -1898,7 +1898,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
             }
         };
 
-        let mut bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
+        let bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
         if let Some(active) = bucket_status.targets_map.get(&opts.arn) {
             if active.resync_id == opts.resync_id {
                 self.resyncer
@@ -1924,26 +1924,43 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         }
 
         let now = OffsetDateTime::now_utc();
-        bucket_status.last_update = Some(now);
-        bucket_status.targets_map.insert(
-            opts.arn.clone(),
-            TargetReplicationResyncStatus {
-                start_time: Some(now),
-                last_update: Some(now),
-                resync_id: opts.resync_id.clone(),
-                resync_before_date: opts.resync_before,
-                resync_status: ResyncStatusType::ResyncPending,
-                failed_size: 0,
-                failed_count: 0,
-                replicated_size: 0,
-                replicated_count: 0,
-                bucket: opts.bucket.clone(),
-                object: String::new(),
-                error: None,
-            },
-        );
+        let admitted = TargetReplicationResyncStatus {
+            start_time: Some(now),
+            last_update: Some(now),
+            resync_id: opts.resync_id.clone(),
+            resync_before_date: opts.resync_before,
+            resync_status: ResyncStatusType::ResyncPending,
+            failed_size: 0,
+            failed_count: 0,
+            replicated_size: 0,
+            replicated_count: 0,
+            bucket: opts.bucket.clone(),
+            object: String::new(),
+            error: None,
+        };
 
-        save_resync_status(&opts.bucket, &bucket_status, self.storage.clone()).await?;
+        // The admission lock serializes competing admissions, but status
+        // writers (mark_status, the periodic saver) do not take it — write
+        // through the CAS so their concurrent updates to other targets are
+        // never lost, re-checking the conflict gate on each retry.
+        let (bucket_status, _) = update_resync_status_cas(&opts.bucket, self.storage.clone(), |persisted| {
+            if let Some(active) = persisted.targets_map.get(&opts.arn) {
+                if active.resync_id == opts.resync_id {
+                    return Ok(false);
+                }
+                if should_auto_resume_resync(active.resync_status) {
+                    return Err(EcstoreError::other(ResyncActiveConflictError {
+                        bucket: opts.bucket.clone(),
+                        arn: opts.arn.clone(),
+                        active_resync_id: active.resync_id.clone(),
+                    }));
+                }
+            }
+            persisted.last_update = Some(now);
+            persisted.targets_map.insert(opts.arn.clone(), admitted.clone());
+            Ok(true)
+        })
+        .await?;
         self.resyncer
             .status_map
             .write()
@@ -1953,6 +1970,83 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         Ok(true)
     }
 
+    /// Cancel the pending/started resync intent recorded for `arn`, if any,
+    /// because its remote target is being removed. Returns the canceled run.
+    ///
+    /// Runs under the bucket admission lock and reloads `resync.bin` from disk
+    /// before writing, like admission does: this node's `status_map` entry may
+    /// be stale relative to intents admitted by other nodes, and persisting it
+    /// would silently drop their durable restart intents.
+    pub async fn cancel_bucket_resync_for_removed_target(
+        self: Arc<Self>,
+        bucket: &str,
+        arn: &str,
+    ) -> Result<Option<ResyncOpts>, EcstoreError> {
+        let bucket = bucket.to_string();
+        let arn = arn.to_string();
+        tokio::spawn(async move { self.cancel_bucket_resync_for_removed_target_transaction(bucket, arn).await })
+            .await
+            .map_err(|error| EcstoreError::other(format!("replication resync cancellation task failed: {error}")))?
+    }
+
+    async fn cancel_bucket_resync_for_removed_target_transaction(
+        self: Arc<Self>,
+        bucket: String,
+        arn: String,
+    ) -> Result<Option<ResyncOpts>, EcstoreError> {
+        let admission_lock_key = ReplicationMetadataStore::resync_admission_lock_key(&bucket);
+        let admission_lock = self
+            .storage
+            .new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), &admission_lock_key)
+            .await?;
+        // Lock order: bucket resync admission lock -> resync status config-object lock.
+        let _admission_guard = admission_lock
+            .get_write_lock(ReplicationLockTiming::acquire_timeout())
+            .await
+            .map_err(EcstoreError::from)?;
+
+        let mut canceled: Option<ResyncOpts> = None;
+        let (final_map, _) = update_resync_status_cas(&bucket, self.storage.clone(), |persisted| {
+            canceled = None;
+            let Some(intent) = persisted.targets_map.get_mut(&arn) else {
+                return Ok(false);
+            };
+            if !should_auto_resume_resync(intent.resync_status) {
+                return Ok(false);
+            }
+            let now = OffsetDateTime::now_utc();
+            canceled = Some(ResyncOpts {
+                bucket: bucket.clone(),
+                arn: arn.clone(),
+                resync_id: intent.resync_id.clone(),
+                resync_before: intent.resync_before_date,
+            });
+            intent.resync_status = ResyncStatusType::ResyncCanceled;
+            intent.last_update = Some(now);
+            persisted.last_update = Some(now);
+            Ok(true)
+        })
+        .await?;
+
+        // Converge only the removed target's cached entry: cached progress
+        // counters for this node's other running targets stay authoritative.
+        {
+            let mut status_map = self.resyncer.status_map.write().await;
+            let cached = status_map
+                .entry(bucket.clone())
+                .or_insert_with(BucketReplicationResyncStatus::new);
+            if let Some(final_target) = final_map.targets_map.get(&arn) {
+                cached.targets_map.insert(arn.clone(), final_target.clone());
+                cached.last_update = final_map.last_update.or(cached.last_update);
+            }
+        }
+
+        if let Some(opts) = &canceled {
+            self.resyncer.cancel(opts).await;
+        }
+        Ok(canceled)
+    }
+
     pub async fn activate_bucket_resync(self: Arc<Self>, opts: ResyncOpts, recovering: bool) -> Result<(), EcstoreError> {
         let bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
         let Some(target_status) = bucket_status.targets_map.get(&opts.arn) else {
@@ -2710,6 +2804,11 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
     async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
     async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
+    async fn cancel_bucket_resync_for_removed_target(
+        self: Arc<Self>,
+        bucket: &str,
+        arn: &str,
+    ) -> Result<Option<ResyncOpts>, EcstoreError>;
     async fn admit_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<bool, EcstoreError>;
     async fn activate_bucket_resync(self: Arc<Self>, opts: ResyncOpts, recovering: bool) -> Result<(), EcstoreError>;
     async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError>;
@@ -2763,6 +2862,14 @@ impl<S: ReplicationStorage> ReplicationPoolTrait for ReplicationPool<S> {
         self.cancel_bucket_resync(opts).await
     }
 
+    async fn cancel_bucket_resync_for_removed_target(
+        self: Arc<Self>,
+        bucket: &str,
+        arn: &str,
+    ) -> Result<Option<ResyncOpts>, EcstoreError> {
+        ReplicationPool::<S>::cancel_bucket_resync_for_removed_target(self, bucket, arn).await
+    }
+
     async fn admit_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<bool, EcstoreError> {
         self.admit_bucket_resync(opts).await
     }
@@ -2855,7 +2962,7 @@ fn replicate_object_info_from_object_info(
         .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size().unwrap_or_default();
+    let asz = oi.get_actual_size_or_physical();
     let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
     let checksum = if ssec { oi.checksum.clone() } else { None };
 
@@ -3241,11 +3348,17 @@ mod tests {
 
         async fn put_object(
             &self,
-            _bucket: &str,
+            bucket: &str,
             object: &str,
             data: &mut Self::PutObjectReader,
             opts: &Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
+            let _lock_guard = if opts.no_lock {
+                None
+            } else {
+                let lock = self.new_ns_lock(bucket, object).await?;
+                Some(lock.get_write_lock(Duration::from_secs(10)).await?)
+            };
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3891,6 +4004,157 @@ mod tests {
         assert_eq!(pool.resyncer.cancel_tokens.read().await.len(), 1);
     }
 
+    /// Removing a target on node A must cancel only A's intent. Node A's cached
+    /// status map predates node B's admission, so a cancel that persisted the
+    /// cache would erase B's durable restart intent for `arn:second`.
+    #[tokio::test]
+    async fn removed_target_cancel_preserves_intents_admitted_on_other_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared.clone()))).await;
+        let node_b = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-b", shared.clone()))).await;
+        let bucket = "removed-target-cancel";
+
+        assert!(
+            node_a
+                .clone()
+                .admit_bucket_resync(test_resync_opts(bucket, "arn:first", "run-a"))
+                .await
+                .expect("node A admission should persist")
+        );
+        assert!(
+            node_b
+                .clone()
+                .admit_bucket_resync(test_resync_opts(bucket, "arn:second", "run-b"))
+                .await
+                .expect("node B admission should persist")
+        );
+        assert!(
+            !node_a.resyncer.status_map.read().await[bucket]
+                .targets_map
+                .contains_key("arn:second"),
+            "precondition: node A's cache must be stale relative to node B's admission"
+        );
+
+        let canceled = node_a
+            .clone()
+            .cancel_bucket_resync_for_removed_target(bucket, "arn:first")
+            .await
+            .expect("cancel should succeed");
+        assert_eq!(canceled.map(|opts| opts.resync_id), Some("run-a".to_string()));
+
+        let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
+            .expect("persisted status should decode");
+        assert_eq!(persisted.targets_map["arn:first"].resync_status, ResyncStatusType::ResyncCanceled);
+        assert_eq!(persisted.targets_map["arn:second"].resync_status, ResyncStatusType::ResyncPending);
+        assert_eq!(persisted.targets_map["arn:second"].resync_id, "run-b");
+        // Cache convergence is per-target: only the removed ARN is written
+        // back (a running target's cached progress counters stay
+        // authoritative), so node A's cache reflects the cancel while the
+        // persisted document remains the authority for `arn:second`.
+        assert_eq!(
+            node_a.resyncer.status_map.read().await[bucket].targets_map["arn:first"].resync_status,
+            ResyncStatusType::ResyncCanceled
+        );
+
+        let untouched = node_a
+            .clone()
+            .cancel_bucket_resync_for_removed_target(bucket, "arn:first")
+            .await
+            .expect("cancel of a terminal intent should be a no-op");
+        assert!(untouched.is_none());
+        assert!(
+            node_a
+                .clone()
+                .cancel_bucket_resync_for_removed_target(bucket, "arn:missing")
+                .await
+                .expect("cancel of an unknown arn should be a no-op")
+                .is_none()
+        );
+    }
+
+    /// The reviewer's resurrect scenario: after node A cancels `arn:first`, a
+    /// status write from node B — whose cache still holds the pre-cancel map —
+    /// must not flip `arn:first` back to `Pending` on disk. `mark_status` now
+    /// persists through the CAS with per-target guards instead of blind-saving
+    /// its cached whole-bucket map.
+    #[tokio::test]
+    async fn stale_peer_status_write_cannot_resurrect_canceled_intent() {
+        let shared = empty_resync_shared_state();
+        let node_a = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared.clone()))).await;
+        let node_b = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-b", shared.clone()))).await;
+        let bucket = "stale-peer-write";
+
+        assert!(
+            node_a
+                .clone()
+                .admit_bucket_resync(test_resync_opts(bucket, "arn:first", "run-a"))
+                .await
+                .expect("node A admission should persist")
+        );
+        assert!(
+            node_b
+                .clone()
+                .admit_bucket_resync(test_resync_opts(bucket, "arn:second", "run-b"))
+                .await
+                .expect("node B admission should persist")
+        );
+        // Seed node B's stale cache: it saw the map before A's cancel.
+        let pre_cancel = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
+            .expect("pre-cancel status should decode");
+        node_b
+            .resyncer
+            .status_map
+            .write()
+            .await
+            .insert(bucket.to_string(), pre_cancel);
+
+        node_a
+            .clone()
+            .cancel_bucket_resync_for_removed_target(bucket, "arn:first")
+            .await
+            .expect("cancel should succeed")
+            .expect("cancel should report the canceled run");
+
+        node_b
+            .resyncer
+            .mark_status(
+                ResyncStatusType::ResyncStarted,
+                test_resync_opts(bucket, "arn:second", "run-b"),
+                node_b.storage.clone(),
+            )
+            .await
+            .expect("peer status write should succeed");
+
+        let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
+            .expect("persisted status should decode");
+        assert_eq!(
+            persisted.targets_map["arn:first"].resync_status,
+            ResyncStatusType::ResyncCanceled,
+            "peer's stale cache must not resurrect the canceled intent"
+        );
+        assert_eq!(persisted.targets_map["arn:second"].resync_status, ResyncStatusType::ResyncStarted);
+
+        // And the reverse guard: a stale write for the canceled target itself
+        // is refused outright.
+        node_b
+            .resyncer
+            .mark_status(
+                ResyncStatusType::ResyncStarted,
+                test_resync_opts(bucket, "arn:first", "run-a"),
+                node_b.storage.clone(),
+            )
+            .await
+            .expect("guarded status write should be skipped, not fail");
+        let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
+            .expect("persisted status should decode");
+        assert_eq!(persisted.targets_map["arn:first"].resync_status, ResyncStatusType::ResyncCanceled);
+        assert_eq!(
+            node_b.resyncer.status_map.read().await[bucket].targets_map["arn:first"].resync_status,
+            ResyncStatusType::ResyncCanceled,
+            "the refused writer's cache must converge to the persisted terminal state"
+        );
+    }
+
     #[tokio::test]
     async fn admitted_resync_waits_for_target_metadata_commit_before_activation() {
         let shared = empty_resync_shared_state();
@@ -4030,6 +4294,77 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn concurrent_resync_status_cas_preserves_both_mutations() {
+        let shared = empty_resync_shared_state();
+        let mut seeded = BucketReplicationResyncStatus::new();
+        for arn in ["arn:a", "arn:b"] {
+            seeded.targets_map.insert(
+                arn.to_string(),
+                TargetReplicationResyncStatus {
+                    bucket: "cas-race".to_string(),
+                    resync_id: format!("run-{arn}"),
+                    resync_status: ResyncStatusType::ResyncPending,
+                    ..Default::default()
+                },
+            );
+        }
+        *shared.data.lock().expect("test data lock should not be poisoned") =
+            encode_resync_file(&seeded).expect("seeded resync status should encode");
+        shared.empty_object_exists.store(true, Ordering::SeqCst);
+        shared.etag_revision.store(1, Ordering::SeqCst);
+        shared.block_next_write.store(true, Ordering::SeqCst);
+        let node_a = Arc::new(LoadResyncNodeStore::new("cas-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("cas-node-b", shared.clone()));
+
+        let writer_a = tokio::spawn(async move {
+            update_resync_status_cas("cas-race", node_a, |status| {
+                status
+                    .targets_map
+                    .get_mut("arn:a")
+                    .expect("seeded target A should exist")
+                    .resync_status = ResyncStatusType::ResyncCanceled;
+                Ok(true)
+            })
+            .await
+        });
+
+        tokio::time::timeout(Duration::from_secs(10), shared.write_started.notified())
+            .await
+            .expect("writer A should pause after its precondition check");
+
+        let mut writer_b = tokio::spawn(async move {
+            update_resync_status_cas("cas-race", node_b, |status| {
+                status
+                    .targets_map
+                    .get_mut("arn:b")
+                    .expect("seeded target B should exist")
+                    .resync_status = ResyncStatusType::ResyncCompleted;
+                Ok(true)
+            })
+            .await
+        });
+        let writer_b_before_release = tokio::time::timeout(Duration::from_millis(250), &mut writer_b).await.ok();
+
+        shared.allow_write.notify_one();
+        writer_a
+            .await
+            .expect("writer A task should finish")
+            .expect("writer A should report a successful conditional save");
+        match writer_b_before_release {
+            Some(result) => result,
+            None => writer_b.await,
+        }
+        .expect("writer B task should finish")
+        .expect("writer B should retry and save its mutation");
+
+        let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
+            .expect("persisted resync status should decode");
+        assert_eq!(persisted.targets_map["arn:a"].resync_status, ResyncStatusType::ResyncCanceled);
+        assert_eq!(persisted.targets_map["arn:b"].resync_status, ResyncStatusType::ResyncCompleted);
+        assert!(!shared.last_put_no_lock.load(Ordering::SeqCst));
+    }
+
     #[test]
     fn replication_queue_admission_combines_target_results() {
         let mut admission = ReplicationQueueAdmission::Skipped;
@@ -4194,6 +4529,30 @@ mod tests {
         assert_eq!(ri.checksum, Some(checksum));
     }
 
+    #[test]
+    fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
+        let target = "arn:rustfs:replication:target-a";
+        let object = ObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            user_tags: Arc::new("owner=a3".to_string()),
+            ..Default::default()
+        };
+        let live =
+            replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
+        let persisted = live.to_mrf_entry();
+        let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
+        let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
+
+        assert_eq!(decoded[0].op, MrfOpKind::Metadata);
+        assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
+        let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
+        assert_eq!(replayed.op_type, ReplicationType::Metadata);
+        assert_eq!(replayed.user_tags, "owner=a3");
+        assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
+    }
+
     #[tokio::test]
     async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
         let (tx, mut rx) = mpsc::channel(1);

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -40,16 +40,17 @@ use super::replication_resync_boundary::ResyncStatusType;
 #[cfg(test)]
 use super::replication_resync_boundary::should_count_head_proxy_failure;
 use super::replication_resync_boundary::{
-    BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, encode_resync_file, is_version_id_mismatch,
-    resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
+    BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file,
+    is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
+    should_auto_resume_resync,
 };
 #[cfg(test)]
-use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX, decode_resync_file};
+use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX};
 #[cfg(test)]
 use super::replication_storage_boundary::ReplicationDeletedObject;
 use super::replication_storage_boundary::{
-    AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete,
-    ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
+    AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions,
+    ObjectToDelete, ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
 };
 use super::replication_target_boundary::{
     ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
@@ -76,7 +77,8 @@ use metrics::counter;
 use rmp_serde;
 use rustfs_s3_types::EventName;
 use rustfs_utils::http::{
-    AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
+    AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
+    has_internal_suffix, insert_str,
 };
 use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
 #[cfg(test)]
@@ -174,6 +176,14 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
     err.raw_response().is_some_and(|r| r.status().as_u16() == status)
 }
 
+fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
+    op_type == ReplicationType::Metadata
+        && object_info
+            .user_defined
+            .get(AMZ_BUCKET_REPLICATION_STATUS)
+            .is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
+}
+
 const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
 
 /// Targets that already produced a version-identity-drift warning this
@@ -419,7 +429,7 @@ impl ReplicationResyncer {
     where
         S: ReplicationObjectIO,
     {
-        let (bucket_status, status_duration) = {
+        let (updated_target, status_duration) = {
             let mut status_map = self.status_map.write().await;
             let now = OffsetDateTime::now_utc();
 
@@ -490,28 +500,62 @@ impl ReplicationResyncer {
 
             bucket_status.last_update = Some(now);
 
-            (bucket_status.clone(), status_duration)
+            (state.clone(), status_duration)
         };
 
-        save_resync_status(&opts.bucket, &bucket_status, obj_layer.clone()).await?;
-        if status != ResyncStatusType::ResyncCanceled {
-            let canceled_status = self
-                .status_map
-                .read()
-                .await
-                .get(&opts.bucket)
-                .filter(|current| {
-                    current.targets_map.get(&opts.arn).is_some_and(|target| {
-                        target.resync_id == opts.resync_id && target.resync_status == ResyncStatusType::ResyncCanceled
-                    })
-                })
-                .cloned();
-            if let Some(canceled_status) = canceled_status {
-                save_resync_status(&opts.bucket, &canceled_status, obj_layer).await?;
-                return Ok(());
+        // Persist through the CAS so a stale cached map can never clobber
+        // states other nodes finalized for other targets; re-run the staleness
+        // and canceled-is-terminal guards against the freshest persisted entry.
+        let updated_last_update = updated_target.last_update;
+        let (final_map, saved) = update_resync_status_cas(&opts.bucket, obj_layer, |persisted| {
+            if let Some(current) = persisted.targets_map.get(&opts.arn) {
+                if !resync_state_accepts_update(current, &opts) {
+                    debug!(
+                        event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                        bucket = %opts.bucket,
+                        arn = %opts.arn,
+                        incoming_resync_id = %opts.resync_id,
+                        current_resync_id = %current.resync_id,
+                        reason = "stale_status_update",
+                        "Skipped persisting stale resync status update"
+                    );
+                    return Ok(false);
+                }
+                if current.resync_status == ResyncStatusType::ResyncCanceled && status != ResyncStatusType::ResyncCanceled {
+                    debug!(
+                        event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                        bucket = %opts.bucket,
+                        arn = %opts.arn,
+                        incoming_status = %status,
+                        reason = "canceled_status_is_terminal",
+                        "Skipped resync status update after cancellation"
+                    );
+                    return Ok(false);
+                }
+            }
+            persisted.targets_map.insert(opts.arn.clone(), updated_target.clone());
+            persisted.last_update = updated_last_update;
+            Ok(true)
+        })
+        .await?;
+
+        // Converge this target's cached entry with what the persisted document
+        // decided (our update, or the newer/terminal state that outranked it).
+        {
+            let mut status_map = self.status_map.write().await;
+            if let Some(cached) = status_map.get_mut(&opts.bucket)
+                && let Some(final_target) = final_map.targets_map.get(&opts.arn)
+            {
+                cached.targets_map.insert(opts.arn.clone(), final_target.clone());
+                cached.last_update = final_map.last_update.or(cached.last_update);
             }
         }
-        if let Some(stats) = runtime_sources::replication_stats() {
+
+        if saved && let Some(stats) = runtime_sources::replication_stats() {
             stats.record_resync_status(&opts.bucket, status, status_duration).await;
         }
 
@@ -603,10 +647,16 @@ impl ReplicationResyncer {
                 }
                 _ = interval.tick() => {
 
-                    let status_map = self.status_map.read().await;
+                    let snapshot: Vec<(String, BucketReplicationResyncStatus)> = self
+                        .status_map
+                        .read()
+                        .await
+                        .iter()
+                        .map(|(bucket, status)| (bucket.clone(), status.clone()))
+                        .collect();
 
                     let mut update = false;
-                    for (bucket, status) in status_map.iter() {
+                    for (bucket, status) in &snapshot {
                         for target in status.targets_map.values() {
                             if target.last_update.is_none() {
                                 update = true;
@@ -622,7 +672,14 @@ impl ReplicationResyncer {
                             }
 
                         if update {
-                            if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
+                            // CAS-merge instead of a blind whole-map save: this
+                            // cache may lag other nodes' admissions and
+                            // cancellations, which must not be overwritten.
+                            let result = update_resync_status_cas(bucket, api.clone(), |persisted| {
+                                Ok(merge_local_resync_into_persisted(persisted, status))
+                            })
+                            .await;
+                            if let Err(err) = result {
                                 error!(
                                     event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
                                     component = LOG_COMPONENT_ECSTORE,
@@ -632,8 +689,8 @@ impl ReplicationResyncer {
                                     error = %err,
                                     "Failed to persist resync status"
                                 );
-                            } else {
-                                last_update_times.insert(bucket.clone(), status.last_update.expect("last_update should be set"));
+                            } else if let Some(last_update) = status.last_update {
+                                last_update_times.insert(bucket.clone(), last_update);
                             }
                         }
                     }
@@ -1412,7 +1469,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     };
     let mut replication_state = oi.replication_state();
     replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size().unwrap_or_default();
+    let actual_size = oi.get_actual_size_or_physical();
 
     Ok(ReplicateObjectInfo {
         name: oi.name.clone(),
@@ -1442,17 +1499,109 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     })
 }
 
-pub(crate) async fn save_resync_status<S: ReplicationObjectIO>(
+/// Upper bound on optimistic retries for a `resync.bin` compare-and-swap
+/// update before giving up; contention on one bucket's status is a handful of
+/// writers (status transitions, the periodic saver, admissions), not a crowd.
+const RESYNC_STATUS_CAS_MAX_ATTEMPTS: usize = 32;
+
+/// Read-merge-write `resync.bin` under an ETag compare-and-swap.
+///
+/// Every writer used to persist its node's cached whole-bucket map, so one
+/// node's stale cache could silently resurrect a state another node had
+/// already finalized (e.g. flip a just-canceled intent back to `Pending`).
+/// `apply` receives the freshest persisted map and mutates it in place,
+/// returning `Ok(false)` to skip the write. On a concurrent write the load +
+/// apply + save cycle is retried against the new document. Returns the final
+/// map and whether this call wrote it.
+pub(crate) async fn update_resync_status_cas<S, F>(
     bucket: &str,
-    status: &BucketReplicationResyncStatus,
     api: Arc<S>,
-) -> Result<()> {
-    let data = encode_resync_file(status)?;
-
+    mut apply: F,
+) -> Result<(BucketReplicationResyncStatus, bool)>
+where
+    S: ReplicationObjectIO,
+    F: FnMut(&mut BucketReplicationResyncStatus) -> Result<bool>,
+{
     let config_file = ReplicationMetadataStore::bucket_resync_file_path(bucket);
-    ReplicationConfigStore::save(api, &config_file, data).await?;
+    for _ in 0..RESYNC_STATUS_CAS_MAX_ATTEMPTS {
+        let (mut status, preconditions) =
+            match ReplicationConfigStore::read_no_lock_with_metadata(api.clone(), &config_file).await {
+                Ok((data, object_info)) => {
+                    let etag = object_info
+                        .etag
+                        .filter(|etag| !etag.trim().is_empty())
+                        .ok_or_else(|| Error::other("replication resync status has no ETag for conditional update"))?;
+                    let status = if data.is_empty() {
+                        BucketReplicationResyncStatus::new()
+                    } else {
+                        decode_resync_file(&data)?
+                    };
+                    (
+                        status,
+                        HTTPPreconditions {
+                            if_match: Some(etag),
+                            ..Default::default()
+                        },
+                    )
+                }
+                Err(Error::ConfigNotFound) => (
+                    BucketReplicationResyncStatus::new(),
+                    HTTPPreconditions {
+                        if_none_match: Some("*".to_string()),
+                        ..Default::default()
+                    },
+                ),
+                Err(err) => return Err(err),
+            };
+        if !apply(&mut status)? {
+            return Ok((status, false));
+        }
+        match ReplicationConfigStore::save_conditional(api.clone(), &config_file, encode_resync_file(&status)?, preconditions)
+            .await
+        {
+            Ok(()) => return Ok((status, true)),
+            Err(Error::PreconditionFailed) => continue,
+            Err(err) => return Err(err),
+        }
+    }
+    Err(Error::other("replication resync status conditional update did not converge"))
+}
 
-    Ok(())
+/// Merge this node's cached bucket resync map into the persisted map for the
+/// periodic saver. Per target: same run id overlays the fresher local state
+/// unless the persisted state is already terminal and the local one is not
+/// (a cancel/completion recorded by another node must stick); a different
+/// persisted run id means a newer admission elsewhere and is kept; targets
+/// unknown to disk are added. Returns whether `persisted` changed.
+pub(crate) fn merge_local_resync_into_persisted(
+    persisted: &mut BucketReplicationResyncStatus,
+    local: &BucketReplicationResyncStatus,
+) -> bool {
+    let mut changed = false;
+    for (arn, local_state) in &local.targets_map {
+        match persisted.targets_map.get(arn) {
+            Some(current) if current.resync_id == local_state.resync_id => {
+                let persisted_terminal = !should_auto_resume_resync(current.resync_status);
+                let local_terminal = !should_auto_resume_resync(local_state.resync_status);
+                if persisted_terminal && !local_terminal {
+                    continue;
+                }
+                if current != local_state {
+                    persisted.targets_map.insert(arn.clone(), local_state.clone());
+                    changed = true;
+                }
+            }
+            Some(_) => {}
+            None => {
+                persisted.targets_map.insert(arn.clone(), local_state.clone());
+                changed = true;
+            }
+        }
+    }
+    if changed && local.last_update.is_some() {
+        persisted.last_update = local.last_update;
+    }
+    changed
 }
 
 pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicationInfo, storage: Arc<S>) {
@@ -3494,6 +3643,7 @@ async fn resolve_replicate_all_action(
         start_time,
         ssec_audit_required,
     } = ctx;
+    let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
     let replication_action;
     match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
         Ok(oi) => {
@@ -3555,7 +3705,13 @@ async fn resolve_replicate_all_action(
                 // Version-ID format mismatch: retry without versionId and compare ETags.
                 match head_object_fallback(tgt_client, object).await {
                     Ok(Some(oi)) => {
-                        replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
+                        let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
+                        if require_existing_target && !etags_match {
+                            rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
+                        replication_action = if etags_match {
                             if ssec_audit_required
                                 && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
                             {
@@ -3568,6 +3724,11 @@ async fn resolve_replicate_all_action(
                         };
                     }
                     Ok(None) => {
+                        if require_existing_target {
+                            rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
                         replication_action = ReplicationAction::All;
                     }
                     Err(e2) => {
@@ -3593,7 +3754,12 @@ async fn resolve_replicate_all_action(
                         return None;
                     }
                 }
-            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
+            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
+                if require_existing_target {
+                    rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                    rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                    return None;
+                }
                 replication_action = ReplicationAction::All;
             } else {
                 rinfo.error = Some(e.to_string());
@@ -3868,6 +4034,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                 actual_size,
                 object_info.etag.clone().unwrap_or_default(),
                 object_info.mod_time,
+                &put_opts.internal,
             ),
         )
         .await
@@ -3886,6 +4053,87 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
 #[cfg(test)]
 mod tests {
     use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
+    fn resync_target_state(resync_id: &str, status: ResyncStatusType, replicated_count: i64) -> TargetReplicationResyncStatus {
+        TargetReplicationResyncStatus {
+            resync_id: resync_id.to_string(),
+            resync_status: status,
+            replicated_count,
+            ..Default::default()
+        }
+    }
+
+    /// Periodic-saver merge: fresher local progress overlays the same run,
+    /// but a terminal state persisted by another node must stick, a newer
+    /// admission elsewhere is kept, and locally-known targets are added.
+    #[test]
+    fn merge_local_resync_keeps_peer_terminal_and_newer_states() {
+        let mut persisted = BucketReplicationResyncStatus::new();
+        persisted.targets_map.insert(
+            "arn:same-run".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncStarted, 1),
+        );
+        persisted.targets_map.insert(
+            "arn:canceled".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncCanceled, 0),
+        );
+        persisted.targets_map.insert(
+            "arn:new-run".to_string(),
+            resync_target_state("run-2", ResyncStatusType::ResyncPending, 0),
+        );
+
+        let mut local = BucketReplicationResyncStatus::new();
+        local.targets_map.insert(
+            "arn:same-run".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncStarted, 9),
+        );
+        local.targets_map.insert(
+            "arn:canceled".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncPending, 0),
+        );
+        local.targets_map.insert(
+            "arn:new-run".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncStarted, 3),
+        );
+        local.targets_map.insert(
+            "arn:local-only".to_string(),
+            resync_target_state("run-1", ResyncStatusType::ResyncPending, 0),
+        );
+        local.last_update = Some(OffsetDateTime::now_utc());
+
+        assert!(merge_local_resync_into_persisted(&mut persisted, &local));
+        assert_eq!(persisted.targets_map["arn:same-run"].replicated_count, 9, "fresher local progress wins");
+        assert_eq!(
+            persisted.targets_map["arn:canceled"].resync_status,
+            ResyncStatusType::ResyncCanceled,
+            "peer terminal state must stick"
+        );
+        assert_eq!(
+            persisted.targets_map["arn:new-run"].resync_id, "run-2",
+            "newer admission elsewhere is kept"
+        );
+        assert!(persisted.targets_map.contains_key("arn:local-only"));
+        assert_eq!(persisted.last_update, local.last_update);
+    }
+
+    /// A terminal local state for the same run (completion/failure recorded by
+    /// this node) still overlays a non-terminal persisted state.
+    #[test]
+    fn merge_local_resync_reports_no_change_when_maps_agree() {
+        let mut persisted = BucketReplicationResyncStatus::new();
+        persisted
+            .targets_map
+            .insert("arn:same".to_string(), resync_target_state("run-1", ResyncStatusType::ResyncStarted, 5));
+        let local = persisted.clone();
+        assert!(!merge_local_resync_into_persisted(&mut persisted, &local));
+
+        let mut local = local.clone();
+        local
+            .targets_map
+            .insert("arn:same".to_string(), resync_target_state("run-1", ResyncStatusType::ResyncCompleted, 5));
+        assert!(merge_local_resync_into_persisted(&mut persisted, &local));
+        assert_eq!(persisted.targets_map["arn:same"].resync_status, ResyncStatusType::ResyncCompleted);
+    }
+
     use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
     use super::*;
     use s3s::dto::{
@@ -3921,6 +4169,113 @@ mod tests {
         })
     }
 
+    fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
+        use std::io::{Read, Write};
+
+        let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
+        let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
+        let handle = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
+            let mut request = [0_u8; 8192];
+            let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
+            assert!(bytes_read > 0, "test HTTP request should not be empty");
+            assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
+            write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("test HTTP response should be written");
+        });
+        (endpoint, handle)
+    }
+
+    #[tokio::test]
+    async fn replica_metadata_missing_target_stops_before_full_put() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            // Normal metadata writes replace REPLICA with per-target PENDING
+            // before constructing the worker request.
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            user_defined: Arc::new(HashMap::from([(
+                AMZ_BUCKET_REPLICATION_STATUS.to_string(),
+                ReplicationStatusType::Replica.as_str().to_string(),
+            )])),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(
+            rinfo.error.as_deref(),
+            Some("replica metadata target does not contain this object version")
+        );
+        server.join().expect("test HTTP server should finish");
+    }
+
+    #[tokio::test]
+    async fn source_metadata_missing_target_rebuilds_object() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(matches!(action, Some((ReplicationAction::All, _))));
+        assert!(rinfo.error.is_none());
+        server.join().expect("test HTTP server should finish");
+    }
+
     async fn register_test_target(target: &Arc<TargetClient>) {
         ReplicationTargetStore::register_test_target(target).await;
     }

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
             .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
         version_id: object_info.version_id.map(|version_id| version_id.to_string()),
         etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size().unwrap_or_default(),
+        actual_size: object_info.get_actual_size_or_physical(),
         delete_marker: object_info.delete_marker,
         content_type: object_info.content_type.as_deref(),
         content_encoding: object_info.content_encoding.as_deref(),
@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
     actual_size: String,
     source_etag: String,
     source_mtime: Option<OffsetDateTime>,
+    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
             // mtime must degrade to epoch so header() suppresses the header
             // instead of asserting the replication time as the object's mtime.
             source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
+            // Carry the per-category LWW timestamps on the complete request as
+            // well: the receiver's CompleteMultipartUpload options builder
+            // parses the same headers, so the multipart transport gets the
+            // same receiver-side LWW as the single-PUT transport
+            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
+            tagging_timestamp: source_internal.tagging_timestamp,
+            retention_timestamp: source_internal.retention_timestamp,
+            legalhold_timestamp: source_internal.legalhold_timestamp,
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -542,6 +551,20 @@ mod tests {
         assert!(replication_target_head_is_newer_null_version(&source, &target));
     }
 
+    #[test]
+    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
+        let source = ObjectInfo {
+            size: 128,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(replication_source_object(&source).actual_size, 128);
+    }
+
     #[test]
     fn replication_target_head_content_matches_compare_etag_only() {
         let source = ObjectInfo {
@@ -649,20 +672,39 @@ mod tests {
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
         let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
+        let source_internal = AdvancedPutOptions {
+            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
+            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
+            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
+            ..Default::default()
+        };
         let options = replication_complete_multipart_options(
             "1024".to_string(),
             "0123456789abcdef0123456789abcdef-3".to_string(),
             Some(source_mtime),
+            &source_internal,
         );
         assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
         assert_eq!(options.internal.source_mtime, source_mtime);
 
+        // The complete request must carry the same per-category LWW timestamps
+        // as the initiate request; the receiver reads them from the complete
+        // headers (rustfs/backlog#1953).
+        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
+        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
+        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
+
         // Absent source mtime must degrade to epoch (header suppressed), not
         // the AdvancedPutOptions default of now_utc() — that default would
         // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence.
-        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
+        // multipart HEAD convergence. Unset category timestamps stay epoch so
+        // header() keeps suppressing them.
+        let options_no_mtime =
+            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
         assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -32,6 +32,10 @@ pub struct Credentials {
     pub access_key: String,
     #[serde(rename = "secretKey")]
     pub secret_key: String,
+    // The aliases accept madmin's JSON tags (MinIO-written bucket-targets
+    // metadata and mc request bodies) without changing the snake_case
+    // persisted/peer wire format this struct serializes to.
+    #[serde(alias = "sessionToken")]
     pub session_token: Option<String>,
     pub expiration: Option<Timestamp>,
 }
@@ -202,12 +206,14 @@ pub struct BucketTarget {
     #[serde(default)]
     pub region: String,
 
-    #[serde(alias = "bandwidth", default)]
+    // madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
+    // alias kept for inputs written before the madmin tag was verified.
+    #[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
     pub bandwidth_limit: i64,
 
     #[serde(rename = "replicationSync", default)]
     pub replication_sync: bool,
-    #[serde(default)]
+    #[serde(alias = "storageclass", default)]
     pub storage_class: String,
     #[serde(rename = "skipTlsVerify", default)]
     pub skip_tls_verify: bool,
@@ -220,7 +226,7 @@ pub struct BucketTarget {
 
     #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
     pub reset_before_date: Option<OffsetDateTime>,
-    #[serde(default)]
+    #[serde(alias = "resetID", default)]
     pub reset_id: String,
     #[serde(rename = "totalDowntime", with = "duration_seconds", default)]
     pub total_downtime: Duration,
@@ -233,7 +239,7 @@ pub struct BucketTarget {
     #[serde(default)]
     pub latency: LatencyStat,
 
-    #[serde(default)]
+    #[serde(alias = "deploymentID", default)]
     pub deployment_id: String,
 
     #[serde(default)]
@@ -531,6 +537,85 @@ mod tests {
         assert_eq!(value["totalDowntime"], 90);
     }
 
+    #[test]
+    fn bucket_target_persisted_wire_keys_stay_snake_case() {
+        // bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
+        // in the admin set/remove handlers) and the msgpack struct-map form
+        // (`BucketTargets::marshal_msg`) both come straight from this struct's
+        // serde field names. madmin naming is applied only in the admin
+        // response layer (`remote_target_admin_json`); renaming here would
+        // silently break every existing deployment's persisted metadata.
+        let targets = BucketTargets {
+            targets: vec![BucketTarget {
+                credentials: Some(Credentials {
+                    access_key: "ak".to_string(),
+                    secret_key: "sk".to_string(),
+                    session_token: Some("token".to_string()),
+                    expiration: None,
+                }),
+                bandwidth_limit: 5,
+                storage_class: "STANDARD".to_string(),
+                reset_id: "reset-1".to_string(),
+                deployment_id: "deploy-1".to_string(),
+                ..Default::default()
+            }],
+        };
+
+        let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
+        let msgpack: serde_json::Value =
+            rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
+                .expect("msgpack struct map should decode into a JSON value");
+
+        for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
+            assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
+            assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
+            assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
+            assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
+            assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
+        }
+    }
+
+    #[test]
+    fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
+        // A MinIO-written bucket-targets.json carries madmin's JSON tags
+        // (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
+        // `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
+        // On migration these must land in the matching fields instead of
+        // silently defaulting (backlog#1951).
+        let targets: BucketTargets = serde_json::from_value(serde_json::json!({
+            "targets": [{
+                "sourcebucket": "src",
+                "endpoint": "minio.example:9000",
+                "credentials": {
+                    "accessKey": "ak",
+                    "secretKey": "sk",
+                    "sessionToken": "minio-session-token"
+                },
+                "targetbucket": "dst",
+                "type": "replication",
+                "replicationSync": true,
+                "bandwidthlimit": 107374182400i64,
+                "storageclass": "STANDARD",
+                "resetID": "reset-789",
+                "deploymentID": "deploy-123"
+            }]
+        }))
+        .expect("MinIO-written bucket-targets.json must deserialize");
+
+        let target = &targets.targets[0];
+        assert_eq!(target.bandwidth_limit, 107374182400);
+        assert_eq!(target.storage_class, "STANDARD");
+        assert_eq!(target.reset_id, "reset-789");
+        assert_eq!(target.deployment_id, "deploy-123");
+        assert_eq!(
+            target
+                .credentials
+                .as_ref()
+                .and_then(|credentials| credentials.session_token.as_deref()),
+            Some("minio-session-token")
+        );
+    }
+
     #[test]
     fn test_bucket_target_debug_redacts_credentials() {
         let target = BucketTarget {

crates/ecstore/src/cache_value/metacache_set.rs

@@ -218,6 +218,7 @@ pub struct ListPathRawOptions {
     pub path: String,
     pub recursive: bool,
     pub incl_deleted: bool,
+    pub skip_hidden_prefix_check: bool,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -249,6 +250,7 @@ impl Clone for ListPathRawOptions {
             path: self.path.clone(),
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
+            skip_hidden_prefix_check: self.skip_hidden_prefix_check,
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -274,6 +276,7 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         base_dir: opts.path.clone(),
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
+        skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -1098,11 +1101,13 @@ mod tests {
     #[test]
     fn walk_dir_options_preserve_zero_total_and_inherited_stall_timeouts() {
         let options = walk_dir_options(&ListPathRawOptions {
+            skip_hidden_prefix_check: true,
             walkdir_timeout: Some(Duration::ZERO),
             walkdir_stall_timeout: None,
             ..Default::default()
         });
 
+        assert!(options.skip_hidden_prefix_check);
         assert_eq!(options.timeout_ms, Some(0));
         assert_eq!(options.stall_timeout_ms, None);
         assert!(!options.skip_total_timeout);

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
 const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
+const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
     get_env_bool(
@@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
 }
 
-fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
+fn update_ns_scanner_capability_mac(
+    mac: &mut HmacSha256,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) {
     mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
     mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
     mac.update(challenge.as_bytes());
     mac.update(server_epoch.as_bytes());
+    if supports_tier_registry_generation {
+        // The optional response capability is part of the authenticated
+        // scope. A proxy cannot turn an old/unsupported peer into a worker
+        // that receives generation-fenced scanner work.
+        mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
+    }
 }
 
-fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
+fn generate_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     Ok(mac.finalize().into_bytes().to_vec())
 }
 
-fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
+fn verify_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     mac.verify_slice(proof)
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
 }
 
 pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
-    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
+    sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
 }
 
 pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
-    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
+    verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
+}
+
+pub fn sign_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
+    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
+}
+
+pub fn verify_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
+    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
 }
 
 #[derive(Clone, Copy)]
@@ -1709,13 +1749,28 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof =
-            generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
+    }
+
+    #[test]
+    fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
+        let secret = "test-scanner-capability-secret";
+        let challenge = Uuid::new_v4();
+        let server_epoch = Uuid::new_v4();
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
+            .expect("generation capability proof should be generated");
+
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("legacy capability proof should be generated");
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
     }
 
     /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
@@ -2939,6 +2994,7 @@ mod tests {
             dst_volume: "bucket".to_string(),
             dst_path: "object".to_string(),
             file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
+            scanner_publication_lease_token: Vec::new().into(),
         };
         let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
         let mut request = tonic::Request::new(());