chore(release): prepare 1.0.0-beta.10-preview.1 (#4898 )

Co-authored-by: heihutu <heihutu@gmail.com>
ci: install zip and aws CLI on self-hosted runners (#4897 )
2026-07-30 10:08:58 +00:00 · 2026-07-16 15:24:30 +08:00 · 2026-07-16 15:23:51 +08:00 · 2026-07-16 07:09:42 +00:00 · 2026-07-16 06:55:52 +00:00 · 2026-07-16 06:31:28 +00:00
251 changed files with 22595 additions and 3901 deletions
@@ -0,0 +1,26 @@
+## —— Coverage --------------------------------------------------------------------------------------
+
+# Local equivalent of the weekly coverage workflow (.github/workflows/coverage.yml,
+# backlog#1153 infra-5): same measurement scope (--workspace --exclude e2e_test,
+# nextest `ci` profile) and the same per-crate table. Slow — the instrumented
+# build cannot reuse your normal target cache and then runs the whole suite.
+# Doctests are not measured (needs nightly). Outputs land in target/llvm-cov/.
+.PHONY: coverage
+coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, writes target/llvm-cov/)
+	@if ! command -v cargo-llvm-cov >/dev/null 2>&1; then \
+		echo >&2 "❌ cargo-llvm-cov is required for 'make coverage' but was not found."; \
+		echo >&2 "   Install it with:"; \
+		echo >&2 "     cargo install cargo-llvm-cov --locked"; \
+		echo >&2 "     rustup component add llvm-tools-preview"; \
+		exit 1; \
+	fi
+	@if ! command -v cargo-nextest >/dev/null 2>&1; then \
+		echo >&2 "❌ cargo-nextest is required for 'make coverage' (see 'make test')."; \
+		echo >&2 "   Install it with: cargo install cargo-nextest --locked"; \
+		exit 1; \
+	fi
+	NEXTEST_PROFILE=ci cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
+	@mkdir -p target/llvm-cov
+	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
+	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
+	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
@@ -45,6 +45,15 @@ e2e-reliability = { max-threads = 1 }
 filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/))'
 test-group = 'ecstore-serial-flaky'

+# Serialize the multipart crash-consistency scenarios (dist-2, backlog#1150):
+# each spawns a 4-disk hermetic erasure set and drives full staged-upload +
+# commit + GET cycles — the same cross-disk-commit IO shape that made
+# concurrent_resend load-sensitive. Preventive serialization only, no retries.
+# The matching ci-profile override is after [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the 4-disk reliability / degraded-read e2e tests (see the
 # e2e-reliability test-group note above). The matching ci-profile override is at
 # the end of the file, after [profile.ci] is declared.
@@ -109,6 +118,13 @@ retries = 2
 filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
 test-group = 'e2e-reliability'

+# Serialize the multipart crash-consistency scenarios under the ci profile too
+# (see the matching default-profile override near the top). Not a quarantine:
+# no retries, just serialized 4-disk cross-disk-commit IO.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
+test-group = 'ecstore-serial-flaky'
+
 # ---------------------------------------------------------------------------
 # e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
 # ---------------------------------------------------------------------------
@@ -136,7 +152,7 @@ test-group = 'e2e-reliability'
 # the nightly profile derives its set as "the replication module MINUS this
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
-# regexes byte-identical. Count invariant: 20 here + 18 nightly = 38 total
+# regexes byte-identical. Count invariant: 20 here + 27 nightly = 47 total
 # (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
@@ -144,12 +160,38 @@ test-group = 'e2e-reliability'
 # the guard now honours an off-by-default opt-in and this suite's source servers
 # set it (RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET) — so the allowlist below is
 # restored.
+#
+# Security negative-auth subset (backlog#1151 sec-5): the three attacker-facing
+# S3 auth-rejection suites join the first clause above by module name —
+# presigned_negative (sec-2), negative_sigv4 (sec-1, header SigV4), and
+# admin_auth (sec-4, admin gate + root-credential lifecycle). All three use
+# RustFSTestEnvironment on a random port and are parallel-safe, so they meet the
+# smoke admission criteria unchanged. This is the wiring step that makes those
+# merged suites actually execute on every PR (they were dead until listed here).
+# A rename that drops any of them out of this filter would silently thin the
+# security gate with no CI signal, so scripts/check_security_smoke_count.sh owns
+# a count-floor guard over exactly this subset (infra-12 mechanism, floor in
+# .config/security-smoke-floor.txt), invoked from the e2e-tests job in ci.yml.
+# NOT here by topology: the GHSA-3p3x FTPS/WebDAV constant-time e2e
+# (protocols::test_protocol_core_suite) binds fixed ports and needs the
+# ftps,webdav features, so it cannot join this random-port, default-feature
+# profile; its GHSA-r5qv sibling is a unit test that already runs in the
+# test-and-lint `--all --exclude e2e_test` pass. See
+# docs/testing/security-regressions.md for the full CI-execution map.
+#
+# ILM tiering main path (backlog#1148 ilm-7): the `reliant::tiering::` clause
+# admits the hermetic transition e2e. Like the fast replication pair checks it
+# spawns a second independent single-node server (the cold RustFS tier), not a
+# cluster, so it keeps the lane's parallel-safe / no-external-dependency
+# properties. The RustFS warm backend has no loopback guard (that guard is
+# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
 [profile.e2e-smoke]
 default-filter = """
    package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative)_test::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud)_test::/)
        | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
        | test(/^reliant::lifecycle::/)
+        | test(/^reliant::tiering::/)
    )
 """
 fail-fast = false
@@ -160,10 +202,14 @@ fail-fast = false
 # backlog#1147 repl-1 (deps: ci-4). Runs the SLOW / cross-process replication
 # tests that are unfit for the per-PR e2e-smoke gate:
 #
-#   * 8 bucket-replication data-plane tests — they PUT/delete objects and poll
-#     until source and target converge; two replicate over HTTPS.
-#   * 9 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
+#   * 2 remote-target TLS validation tests.
+#   * 12 bucket-replication data-plane/helper tests — they PUT/delete objects
+#     and poll until source and target converge; two replicate over HTTPS, two
+#     pin active SSE failure contracts, and one guards event/history observers.
+#     The SSE-S3 contract remains ignored under backlog#1291.
+#   * 11 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
+#   * 1 `_real_three_node` site-replication test.
 #   * 1 `_real_single_node` service-account round-trip test.
 #
 # The set is defined as "everything in replication_extension_test that is NOT
@@ -199,3 +245,79 @@ fail-fast = false
 # Emitted to target/nextest/e2e-repl-nightly/junit.xml; uploaded by the nightly
 # workflow as the failure-triage artifact.
 path = "junit.xml"
+
+# ---------------------------------------------------------------------------
+# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
+# ---------------------------------------------------------------------------
+# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
+# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
+# object_lock (33), multipart_auth (109), quota, checksum, encryption,
+# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
+# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
+# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
+#
+# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
+#   * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
+#     ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
+#   * the 6 cluster suites that spin up a RustFSTestClusterEnvironment
+#     (cluster_concurrency, stale_multipart_cleanup_cluster,
+#     namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
+#     object_lambda) — too heavy for the merge budget; they run in ci-7's
+#     nightly 4-node lane.
+#   * replication_extension_test — repl-1 already splits it into the PR
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (27 slow) lanes and reserves
+#     it for those, so e2e-full does not double-run it.
+#   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
+#     manual-localhost:9000 reliant/policy tests are ci-13's migration.
+#
+# Each e2e test spawns its own single-node rustfs server on a random port with
+# an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
+# parallel-safe — the same property e2e-smoke relies on. The exception is the
+# 4-disk reliability / degraded-read fault-injection tests, serialized below
+# (identical to the ci profile) so several 4-disk servers never run at once.
+# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
+# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
+# product failures cannot be quarantined away with retries, so each family is
+# excluded here with its tracking issue, under the same discipline as the
+# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
+# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
+# negative-path siblings of each family stay in as regression guards.
+#   * rustfs#4842 — extract/snowball expand pipeline 500s (mtime=0
+#     OffsetDateTime deserialization + same-path failures).
+#   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
+#     archive even under ignore-errors semantics.
+#   * rustfs#4844 — anonymous POST-object with SSE-S3 / bucket-default SSE
+#     returns 500.
+#   * rustfs#4845 — 403 on allowed anonymous POST object-lock fields and on
+#     the list metadata=true extension.
+#   * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
+#     under parallel load (multi-node in-process clusters; natural home is
+#     ci-7's nightly cluster lane).
+[profile.e2e-full]
+default-filter = """
+    package(e2e_test)
+    & !test(/^protocols::/)
+    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^replication_extension_test::/)
+    & !test(/^multipart_auth_test::test_signed_put_object_extract_(accepts_compat_header|expands_tar_entries_with_prefix_headers|expands_tar_gz_archive|expands_tbz2_archive|expands_tgz_archive|expands_txz_archive|expands_tzst_archive|normalizes_prefix_header_value|preserves_directory_markers_by_default|preserves_object_lock_legal_hold|preserves_object_lock_retention|preserves_pax_metadata_and_version_id|preserves_request_metadata_on_extracted_objects|preserves_sse_c|preserves_sse_s3_and_redirect|preserves_storage_class|uses_bucket_default_sse_s3)$/)
+    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(prefers_exact_minio_prefix_over_suffix_fallback|supports_minio_prefix_and_directory_markers)$/)
+    & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
+    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
+    & !test(/^multipart_auth_test::test_anonymous_post_object_(accepts_sse_s3|rejects_sse_s3_missing_from_policy_conditions|uses_bucket_default_sse_kms|uses_bucket_default_sse_s3)$/)
+    & !test(/^multipart_auth_test::test_anonymous_post_object_(accepts_object_lock_legal_hold_field|accepts_object_lock_retention_fields)$/)
+    & !test(/^list_object(s_v2|_versions)_metadata_extension_test::/)
+    & !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
+"""
+fail-fast = false
+
+[profile.e2e-full.junit]
+# Emitted to target/nextest/e2e-full/junit.xml; uploaded by the e2e-full job.
+path = "junit.xml"
+
+# Serialize the 4-disk reliability / degraded-read e2e tests under e2e-full too
+# (see the e2e-reliability test-group note near the top of this file). Not a
+# quarantine: no retries, just single-threaded so several 4-disk servers never
+# run concurrently.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
+test-group = 'e2e-reliability'
@@ -0,0 +1,12 @@
+# Committed floor for the number of security negative-auth tests selected by the
+# e2e-smoke PR profile (see scripts/check_security_smoke_count.sh, backlog#1151
+# sec-5).
+#
+# The floor equals the exact count of e2e_test cases whose name starts with a
+# security module prefix (negative_sigv4_test, presigned_negative_test,
+# admin_auth_test) that the [profile.e2e-smoke] default-filter in
+# .config/nextest.toml selects, at the time this file was last updated. CI fails
+# if the selected count drops below this number, so a rename or removal that
+# thins the security smoke gate must update this file in the same PR.
+# Adding tests does not require a bump, but bumping keeps the guard tight.
+16
@@ -56,6 +56,7 @@ runs:
          libssl-dev \
          ripgrep \
          unzip \
+          zip \
          protobuf-compiler

    - name: Install protoc
@@ -136,11 +136,13 @@ jobs:
            echo "⚡ Manual/scheduled build detected"
          fi

-          echo "should_build=$should_build" >> $GITHUB_OUTPUT
-          echo "build_type=$build_type" >> $GITHUB_OUTPUT
-          echo "version=$version" >> $GITHUB_OUTPUT
-          echo "short_sha=$short_sha" >> $GITHUB_OUTPUT
-          echo "is_prerelease=$is_prerelease" >> $GITHUB_OUTPUT
+          {
+            echo "should_build=$should_build"
+            echo "build_type=$build_type"
+            echo "version=$version"
+            echo "short_sha=$short_sha"
+            echo "is_prerelease=$is_prerelease"
+          } >> "$GITHUB_OUTPUT"

          echo "📊 Build Summary:"
          echo "  - Should build: $should_build"
@@ -220,8 +222,8 @@ jobs:
    name: Build RustFS
    needs: [ build-check, prepare-platform-matrix ]
    if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
-    runs-on: ${{ matrix.platform == 'linux' && fromJSON('["self-hosted","linux","sm-standard-2"]') || matrix.os }}
-    timeout-minutes: 90
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 150
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      # Release binaries ship without dial9 telemetry and therefore do not need
@@ -437,11 +439,11 @@ jobs:
            # Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
            PACKAGE_NAME="${PACKAGE_BASENAME}-v${PACKAGE_VERSION}"
          fi
-
+          
          # Create zip packages for all platforms
          # Ensure zip is available
          if ! command -v zip &> /dev/null; then
-            if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+            if [[ "${{ matrix.os }}" == "ubuntu-latest" || "${{ matrix.platform }}" == "linux" ]]; then
              sudo apt-get update && sudo apt-get install -y zip
            fi
          fi
@@ -493,7 +495,7 @@ jobs:
            if [[ "${{ matrix.platform }}" == "windows" ]]; then
              dir
            else
-              ls -lh ${PACKAGE_NAME}.zip
+              ls -lh "${PACKAGE_NAME}.zip"
            fi
          else
            echo "❌ Failed to create package: ${PACKAGE_NAME}.zip"
@@ -542,11 +544,13 @@ jobs:
            fi
          fi

-          echo "package_name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT
-          echo "package_file=${PACKAGE_NAME}.zip" >> $GITHUB_OUTPUT
-          echo "latest_files=${LATEST_FILES}" >> $GITHUB_OUTPUT
-          echo "build_type=${BUILD_TYPE}" >> $GITHUB_OUTPUT
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          {
+            echo "package_name=${PACKAGE_NAME}"
+            echo "package_file=${PACKAGE_NAME}.zip"
+            echo "latest_files=${LATEST_FILES}"
+            echo "build_type=${BUILD_TYPE}"
+            echo "version=${VERSION}"
+          } >> "$GITHUB_OUTPUT"

          echo "📦 Package created: ${PACKAGE_NAME}.zip"
          if [[ -n "$LATEST_FILES" ]]; then
@@ -579,9 +583,19 @@ jobs:
            exit 0
          fi

+          # The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
+          # images did). Install it on demand so R2 uploads survive a fresh runner
+          # instead of hard-failing here.
          if ! command -v aws >/dev/null 2>&1; then
-            echo "❌ aws CLI not found on runner; cannot upload to R2"
-            exit 1
+            echo "aws CLI not found on runner; installing..."
+            if command -v apt-get >/dev/null 2>&1; then
+              sudo apt-get update && sudo apt-get install -y awscli
+            elif command -v brew >/dev/null 2>&1; then
+              brew install awscli
+            else
+              echo "❌ aws CLI missing and no apt-get/brew to install it; cannot upload to R2"
+              exit 1
+            fi
          fi

          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
@@ -743,8 +757,8 @@ jobs:
            RELEASE_URL=$(gh release view "$TAG" --json url --jq '.url')
          fi

-          echo "release_id=$RELEASE_ID" >> $GITHUB_OUTPUT
-          echo "release_url=$RELEASE_URL" >> $GITHUB_OUTPUT
+          echo "release_id=$RELEASE_ID" >> "$GITHUB_OUTPUT"
+          echo "release_url=$RELEASE_URL" >> "$GITHUB_OUTPUT"
          echo "Created release: $RELEASE_URL"

  # Prepare and upload release assets
@@ -793,9 +807,9 @@ jobs:
          cd ./release-assets

          # Generate checksums for all files (including latest versions)
-          if ls *.zip >/dev/null 2>&1; then
-            sha256sum *.zip > SHA256SUMS
-            sha512sum *.zip > SHA512SUMS
+          if compgen -G "*.zip" >/dev/null; then
+            sha256sum -- *.zip > SHA256SUMS
+            sha512sum -- *.zip > SHA512SUMS
          fi

          cd ..
@@ -435,6 +435,16 @@ jobs:
      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

+      # Guard the security negative-auth smoke subset (backlog#1151 sec-5)
+      # against a rename or deletion silently dropping it out of the e2e-smoke
+      # filter. The script lists what the profile selects and fails if the count
+      # of security auth-rejection tests falls below the committed floor in
+      # .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
+      # before the smoke suite so a thinned gate fails fast; the `nextest list`
+      # here compiles the e2e_test binaries the run below reuses.
+      - name: Check security smoke subset count floor
+        run: ./scripts/check_security_smoke_count.sh check
+
      # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
      # profile.e2e-smoke default-filter in .config/nextest.toml is the single
      # wiring mechanism for e2e tests in CI — extend that filter instead of
@@ -468,6 +478,59 @@ jobs:
          path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
          retention-days: 3

+  e2e-full:
+    name: End-to-End Tests (full merge gate)
+    # Merge gate only (backlog#1149 ci-5): the never-automated user-visible
+    # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
+    # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
+    # every PR, so it is gated to main pushes, the merge queue, and manual
+    # dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
+    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'merge_group' ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    needs: [ build-rustfs-debug-binary ]
+    runs-on: sm-standard-2
+    timeout-minutes: 55
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      # Download after the cache restore so the freshly built binary from the
+      # build job always wins over anything restored into target/debug.
+      - name: Download debug binary
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          name: rustfs-debug-binary
+          path: target/debug
+
+      - name: Make binary executable
+        run: chmod +x ./target/debug/rustfs
+
+      # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
+      # default-filter in .config/nextest.toml is the single wiring mechanism —
+      # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
+      # debug binary; each test spawns its own rustfs server on a random port.
+      - name: Run e2e full suite
+        run: cargo nextest run --profile e2e-full -p e2e_test
+
+      - name: Upload junit
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-full-junit-${{ github.run_number }}
+          path: target/nextest/e2e-full/junit.xml
+          retention-days: 7
+
  e2e-tests-rio-v2:
    name: End-to-End Tests (rio-v2)
    needs: [ build-rustfs-debug-binary-rio-v2 ]
@@ -494,6 +557,14 @@ jobs:
      - name: Setup Rust toolchain for s3s-e2e installation
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

+      # The sm-standard-* custom runner images (introduced in #4884) ship no C
+      # toolchain, unlike GitHub-hosted ubuntu-latest. Installing s3s-e2e below
+      # compiles it from source on a cache miss, and build scripts need cc.
+      - name: Install build tools for s3s-e2e compilation
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake pkg-config libssl-dev
+
      - name: Install s3s-e2e test tool
        uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
        with:
@@ -0,0 +1,122 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+#
+# NON-BLOCKING by design: this workflow only runs on schedule and manual
+# dispatch, so it never attaches a status to a PR and must never be made a
+# required check. It exists to give coverage a visible baseline and trend
+# (per-crate table in the job summary, lcov artifact kept 90 days) — the
+# per-crate ratchet for the security-critical crates builds on it later
+# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+#
+# Measurement scope matches the PR test gate (ci.yml "Run tests"):
+# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
+# NOT measured (ci.yml runs them uninstrumented; `cargo llvm-cov` needs a
+# nightly toolchain to cover doctests). Trend-comparison workflow:
+# docs/testing/README.md "Coverage" section. `make coverage` is the local
+# equivalent. Scheduled failures alert via the ci-8 composite action.
+
+name: coverage
+
+on:
+  workflow_dispatch:
+  schedule:
+    # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
+    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
+    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
+    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
+    - cron: "0 7 * * 0"
+
+# Only alert-on-failure needs more than read access; it declares its own
+# job-level `issues: write`.
+permissions:
+  contents: read
+
+jobs:
+  coverage:
+    name: Workspace coverage (weekly)
+    runs-on: sm-standard-4
+    # The instrumented build cannot reuse the regular CI cache (different
+    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
+    # full suite; give it double the test job's 60-minute budget.
+    timeout-minutes: 120
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
+      # retries=0 plus the quarantine list and the ecstore-serial-flaky
+      # serialization. Set via env because `cargo llvm-cov`'s own --profile
+      # flag selects the *cargo build* profile, not the nextest profile.
+      NEXTEST_PROFILE: ci
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-coverage
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: cargo-llvm-cov
+
+      - name: Install llvm-tools component
+        run: rustup component add llvm-tools-preview
+
+      - name: Run instrumented test suite
+        run: cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
+
+      - name: Generate lcov and JSON reports
+        run: |
+          mkdir -p target/llvm-cov
+          cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
+          cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
+
+      - name: Write per-crate summary
+        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload coverage artifact
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: coverage-lcov-${{ github.run_number }}
+          path: |
+            target/llvm-cov/lcov.info
+            target/llvm-cov/coverage.json
+          retention-days: 90
+          if-no-files-found: ignore
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [coverage]
+    # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
+    # manual workflow_dispatch runs stay quiet so a debugging run never files a
+    # spurious alert.
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -247,13 +247,15 @@ jobs:
            esac
          fi

-          echo "should_build=$should_build" >> $GITHUB_OUTPUT
-          echo "should_push=$should_push" >> $GITHUB_OUTPUT
-          echo "build_type=$build_type" >> $GITHUB_OUTPUT
-          echo "version=$version" >> $GITHUB_OUTPUT
-          echo "short_sha=$short_sha" >> $GITHUB_OUTPUT
-          echo "is_prerelease=$is_prerelease" >> $GITHUB_OUTPUT
-          echo "create_latest=$create_latest" >> $GITHUB_OUTPUT
+          {
+            echo "should_build=$should_build"
+            echo "should_push=$should_push"
+            echo "build_type=$build_type"
+            echo "version=$version"
+            echo "short_sha=$short_sha"
+            echo "is_prerelease=$is_prerelease"
+            echo "create_latest=$create_latest"
+          } >> "$GITHUB_OUTPUT"

          echo "🐳 Docker Build Summary:"
          echo "  - Should build: $should_build"
@@ -320,7 +322,6 @@ jobs:
        run: |
          BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
          VERSION="${{ needs.build-check.outputs.version }}"
-          SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
          CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
          VARIANT_SUFFIX="${{ matrix.suffix }}"

@@ -343,8 +344,8 @@ jobs:
              ;;
          esac

-          echo "docker_release=$DOCKER_RELEASE" >> $GITHUB_OUTPUT
-          echo "docker_channel=$DOCKER_CHANNEL" >> $GITHUB_OUTPUT
+          echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
+          echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT"

          echo "🐳 Docker build parameters:"
          echo "  - Original version: $VERSION"
@@ -376,7 +377,7 @@ jobs:
          fi

          # Output tags
-          echo "tags=$TAGS" >> $GITHUB_OUTPUT
+          echo "tags=$TAGS" >> "$GITHUB_OUTPUT"

          # Generate labels
          LABELS="org.opencontainers.image.title=RustFS"
@@ -387,7 +388,7 @@ jobs:
          LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
          LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"

-          echo "labels=$LABELS" >> $GITHUB_OUTPUT
+          echo "labels=$LABELS" >> "$GITHUB_OUTPUT"

          echo "🐳 Generated Docker tags:"
          echo "$TAGS" | tr ',' '\n' | sed 's/^/  - /'
@@ -15,20 +15,24 @@
 # Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
 #
 # The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the 20
-# FAST bucket-replication tests. This scheduled lane runs the remaining 18
+# FAST replication tests. This scheduled lane runs the remaining 27
 # heavier replication e2e tests that are unfit for a per-PR gate:
 #
-#   * 8 bucket-replication data-plane tests (PUT/delete + poll for convergence;
-#     two replicate over HTTPS).
-#   * 9 `_real_dual_node` site-replication tests (each spawns TWO rustfs
+#   * 2 remote-target TLS validation tests.
+#   * 12 bucket-replication data-plane/helper tests (PUT/delete + poll for
+#     convergence; two replicate over HTTPS, two pin active SSE failure
+#     contracts, and one guards event/history observers). The SSE-S3 contract
+#     remains ignored under backlog#1291.
+#   * 11 `_real_dual_node` site-replication tests (each spawns TWO rustfs
 #     servers and drives the cross-process site-replication control plane).
+#   * 1 `_real_three_node` site-replication test.
 #   * 1 `_real_single_node` service-account round-trip test.
 #
 # The selection is the [profile.e2e-repl-nightly] default-filter in
 # .config/nextest.toml — the single wiring mechanism (repl-1 / ci-4). Do NOT
 # add ad-hoc cargo-test steps here; change the filterset instead.
 #
-# Explicit division of labor: these 18 tests run ONLY here, never double-run
+# Explicit division of labor: these 27 tests run ONLY here, never double-run
 # in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
 # consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
 # into it rather than growing a second scheduled entrypoint.
@@ -80,7 +84,12 @@ jobs:
          python-version: "3.12"

      - name: Install awscurl
-        run: python3 -m pip install --user --upgrade pip awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip awscurl
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"

      # Build the rustfs binary once up front. The e2e tests spawn it as a
      # child process (crates/e2e_test/src/common.rs) and will build it on
@@ -0,0 +1,70 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MinIO on-disk interop: prove RustFS reads MinIO-written erasure-coded SSE
+# objects with byte-identical data and correct logical size.
+#
+# This is NOT a PR gate. The fixtures are real MinIO backend trees generated on
+# the fly (they are gitignored, never committed), so the job regenerates them
+# each run with Docker and then runs the `#[ignore]` reader tests in
+# crates/ecstore/tests/minio_generated_read_test.rs.
+#
+# Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
+# unlike the self-hosted fleet, whose pods drift in Docker/pip availability
+# (see the infra note in e2e-s3tests.yml). Nightly + manual only.
+name: minio-interop
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Nightly at 03:17 UTC (offset from other nightly jobs).
+    - cron: "17 3 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  minio-interop:
+    name: MinIO interop (EC + SSE read parity)
+    # Skip on forks: needs the repo's runners and is not a contributor gate.
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    env:
+      # Fixed 32-byte test KMS key baked into the fixture lab; not a secret.
+      RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-minio-interop
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Generate real MinIO fixtures via Docker
+        run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh
+
+      - name: Run MinIO interop reader tests
+        run: |
+          cargo nextest run --run-ignored ignored-only \
+            -p rustfs-ecstore --features rio-v2 \
+            -E 'binary(minio_generated_read_test)'
@@ -135,6 +135,10 @@ jobs:
        run: |
          docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
          docker rm -f rustfs-mint >/dev/null 2>&1 || true
+          # The four disks share one physical device on the runner (a single
+          # loopback filesystem), so the local physical-disk-independence guard
+          # would refuse to start. Bypass it — this is the CI use case the guard
+          # explicitly sanctions via RUSTFS_UNSAFE_BYPASS_DISK_CHECK.
          docker run -d --name rustfs-mint \
            --network rustfs-net \
            -p 9000:9000 \
@@ -142,6 +146,7 @@ jobs:
            -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
            -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
            -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
+            -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
            -v /tmp/rustfs-mint:/data \
            rustfs-ci

@@ -41,6 +41,10 @@ on:
        type: boolean
  pull_request:
    types: [labeled, synchronize, reopened]
+  push:
+    # Every main commit pre-builds and caches its release binary (perf-3) so the
+    # nightly A/B restores a ready baseline instead of paying the double build.
+    branches: [main]

 permissions:
  contents: read
@@ -58,22 +62,81 @@ env:
  RUST_BACKTRACE: 1

 jobs:
+  # perf-3: on every push to main, build the release binary once and cache it
+  # keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
+  # perf-7 PR gate) restore this instead of paying the ~32min-per-side source
+  # build. That double build is what pushed the expanded 24-cell nightly past its
+  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
+  # builds off the shared cargo cache keep each push cheap, and building on the
+  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
+  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
+  # are cancelled ~7/8 of the time and are not a reliable baseline.
+  build-baseline-cache:
+    name: Build + cache baseline binary
+    if: github.event_name == 'push'
+    runs-on: sm-standard-2
+    # Latest-wins: consumers only ever restore the binary for the *current*
+    # origin/main tip, so when pushes land faster than the ~65min build, a
+    # superseded build's output is dead weight — cancel it instead of stacking
+    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
+    # most costs one same-commit self-heal in the A/B job.
+    concurrency:
+      group: perf-baseline-build-main
+      cancel-in-progress: true
+    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
+    # single release build past 60min on this runner — every cache build on
+    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
+    # time of 1h0m0s") and the cache never populated. The measured binary must
+    # keep the production profile, so the budget absorbs the build instead.
+    timeout-minutes: 100
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build release rustfs
+        run: cargo build --release --bin rustfs
+
+      - name: Stage binary for cache
+        run: |
+          set -euo pipefail
+          mkdir -p baseline-bin
+          cp target/release/rustfs baseline-bin/rustfs
+
+      - name: Cache baseline binary by SHA
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ github.sha }}
+
  warp-ab:
    name: Warp A/B budget gate
-    # Opt-in on PRs: only run when the `perf-ab` label is present, and for
-    # `labeled` events only when the label being added is `perf-ab` itself —
-    # adding an unrelated label to an opted-in PR must not re-run the gate.
-    # Always run on schedule / manual dispatch.
+    # Always run on schedule / manual dispatch. Opt-in on PRs: only when the
+    # `perf-ab` label is present, and for `labeled` events only when the label
+    # being added is `perf-ab` itself (adding an unrelated label to an opted-in
+    # PR must not re-run the gate). Never on push — that event only feeds
+    # build-baseline-cache above.
    if: >-
-      github.event_name != 'pull_request' ||
-      (contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
       (github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
    runs-on: sm-standard-2
-    # Phase-0 stopgap: the baseline+candidate release double-build alone is
-    # ~65min on this runner, so 90min left no room for a real full-matrix
-    # measurement (the earlier nightly runs only ever failed *before*
-    # measuring). 120min gives the 24-cell short matrix headroom until perf-3
-    # caches the baseline binary and restores a tighter budget.
+    # With perf-3's cached baseline binary the common (cache-hit) nightly is
+    # measurement-only and finishes well under 50min. This ceiling stays
+    # generous only to absorb the same-commit cache-miss self-heal (~65min
+    # single build with the post-#4806 LTO profile + measurement). A timeout
+    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
+    # not just failure). perf-6 recalibrates the budget once the noise study
+    # lands.
    timeout-minutes: 120
    steps:
      - name: Checkout repository
@@ -110,21 +173,106 @@ jobs:
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

+      # perf-3: resolve the commits so the cache can be keyed by SHA. The
+      # baseline is origin/main; the candidate is the checked-out ref. On the
+      # nightly (checkout == main) they are the same commit, so one cached binary
+      # serves both phases and the run does zero source builds.
+      - name: Resolve baseline / candidate commits
+        id: commits
+        run: |
+          set -euo pipefail
+          baseline_sha="$(git rev-parse origin/main)"
+          candidate_sha="$(git rev-parse HEAD)"
+          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
+          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
+          echo "baseline commit: $baseline_sha"
+          echo "candidate commit: $candidate_sha"
+
+      # Exact-key restore of the baseline binary built by build-baseline-cache
+      # when origin/main last landed. A miss (binary evicted or not built yet)
+      # leaves cache-hit unset and the rig falls back to a source build.
+      - name: Restore cached baseline binary
+        id: baseline_cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
+
+      # Self-heal: on a nightly/dispatch run where the candidate commit IS the
+      # baseline commit, a cache miss would make the rig build the same commit
+      # twice (~65min per side with the post-#4806 LTO profile — no job budget
+      # fits that). Build it once here, reuse it for both phases, and save it
+      # back to the cache so the next run hits.
+      - name: Build baseline on cache miss (same-commit self-heal)
+        id: selfheal
+        if: >-
+          steps.baseline_cache.outputs.cache-hit != 'true' &&
+          steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
+        run: |
+          set -euo pipefail
+          cargo build --release --bin rustfs
+          mkdir -p baseline-bin
+          cp target/release/rustfs baseline-bin/rustfs
+          echo "built=true" >> "$GITHUB_OUTPUT"
+
+      - name: Save self-healed baseline to cache
+        if: steps.selfheal.outputs.built == 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
+
      - name: Run warp A/B and gate
        id: ab
        run: |
          set -euo pipefail
-          # Budget note: the baseline+candidate release double-build (~65 min,
-          # cached away later by perf-3) dominates the 90-min job, so the
-          # measurement runs a short warp matrix — duration/rounds/cooldown are
-          # kept small to fit all 24 cells (6 workloads x 2 phases x 2 drive-sync)
-          # under budget rather than dropping cells. --health-timeout 180 outlasts
-          # the server's own 120s startup-readiness budget, which is what the
-          # rig's previous 60s health poll undershot (the first two nightly
-          # failures). perf-6 will recalibrate these once the pipeline is green.
+          # Budget note: with perf-3's cached baseline the nightly does no source
+          # build on a cache hit, so the wall-clock is dominated by the short warp
+          # matrix — duration/rounds/cooldown are kept small to fit all 24 cells
+          # (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
+          # --health-timeout 180 outlasts the server's own 120s startup-readiness
+          # budget, which the rig's previous 60s health poll undershot (the first
+          # two nightly failures). perf-6 recalibrates these once the noise study
+          # lands.
          duration="${{ github.event.inputs.duration || '12s' }}"
-          args=(--baseline-ref origin/main
-            --duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
+          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
+          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
+          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
+          selfheal_built="${{ steps.selfheal.outputs.built }}"
+
+          args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
+
+          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" ]]; then
+            chmod +x baseline-bin/rustfs
+            base_bin="$PWD/baseline-bin/rustfs"
+            args+=(--baseline-bin "$base_bin")
+            if [[ "$baseline_hit" == "true" ]]; then
+              base_src="actions-cache (rustfs-baseline-$baseline_sha)"
+            else
+              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
+            fi
+            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
+              # Nightly on main: the candidate is the same commit as the baseline,
+              # so reuse the one binary for both phases and skip all builds.
+              args+=(--candidate-bin "$base_bin" --skip-build)
+              cand_src="same binary as baseline (same commit)"
+            else
+              cand_src="source build of the checked-out ref"
+            fi
+          else
+            # Cache miss with candidate != baseline (opt-in PR gate only): fall
+            # back to the source double-build. With the post-#4806 LTO profile
+            # this will overrun the job budget and alert; rerun once the push
+            # cache build for origin/main has completed, or wait for perf-7's
+            # merge-base caching.
+            args+=(--baseline-ref origin/main)
+            base_src="source build of origin/main (cache miss)"
+            cand_src="source build of the checked-out ref"
+          fi
+
+          args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
+          args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")
+
          if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
            args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
          fi
@@ -201,12 +349,8 @@ jobs:
        run: |
          gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"

-      # TODO(ci-8/perf-2): scheduled/dispatch failures are currently silent. Once
-      # ci-8 lands the .github/actions/schedule-failure-issue composite action,
-      # perf-2 adds a step here guarded by
-      #   if: failure() && github.event_name != 'pull_request'
-      # that calls it (label perf-nightly-failure, append to an existing open
-      # issue) instead of hand-rolling gh CLI dedup. Do not implement it here.
+      # Scheduled failure alerting is handled by the alert-on-failure job below
+      # (perf-2 consuming ci-8's schedule-failure-issue composite action).

      - name: Enforce gate
        if: always()
@@ -224,7 +368,16 @@ jobs:
    # `always()` is required: without it this job is skipped when a needed
    # job fails. Alerts only for scheduled (nightly) runs (backlog#1149
    # ci-8); PR and manual dispatch failures are already watched by a human.
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    # `cancelled` is included alongside `failure` on purpose: a job that hits
+    # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
+    # timeouts went silent precisely because the guard was failure-only. The
+    # composite action already reports cancelled/timed-out jobs in the issue
+    # body. (Scheduled runs get a unique concurrency group with
+    # cancel-in-progress off, so a cancellation here means a timeout/manual
+    # abort, never a superseding run.)
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
@@ -1,45 +1,7 @@
 {
    "version": "0.2.0",
    "configurations": [
-        {
-            "name": "Debug RustFS observability (OTLP)",
-            "type": "lldb",
-            "request": "launch",
-            "cargo": {
-                "args": [
-                    "build",
-                    "--bin=rustfs",
-                    "--package=rustfs"
-                ],
-                "filter": {
-                    "name": "rustfs",
-                    "kind": "bin"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}",
-            "env": {
-                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=info,iam=info",
-                "RUST_BACKTRACE": "full",
-                "RUSTFS_ACCESS_KEY": "rustfsadmin",
-                "RUSTFS_SECRET_KEY": "rustfsadmin",
-                "RUSTFS_VOLUMES": "./target/observability/data{1...4}",
-                "RUSTFS_ADDRESS": ":9000",
-                "RUSTFS_CONSOLE_ENABLE": "true",
-                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
-                "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
-                "RUSTFS_OBS_ENDPOINT": "http://127.0.0.1:4318",
-                "RUSTFS_OBS_TRACES_EXPORT_ENABLED": "true",
-                "RUSTFS_OBS_METRICS_EXPORT_ENABLED": "true",
-                "RUSTFS_OBS_LOGS_EXPORT_ENABLED": "true",
-                "RUSTFS_OBS_USE_STDOUT": "true",
-                "RUSTFS_OBS_LOG_DIRECTORY": "./target/observability/logs",
-                "RUSTFS_OBS_METER_INTERVAL": "5",
-                "RUSTFS_OBS_SERVICE_NAME": "rustfs-observability-local",
-                "RUSTFS_OBS_ENVIRONMENT": "development"
-            }
-        },
-        {
+          {
            "type": "lldb",
            "request": "launch",
            "name": "Debug(only) executable 'rustfs'",
@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Helm Ingress**: `customAnnotations` are now merged with class-specific annotations (nginx/traefik) instead of being ignored when `ingress.className` is set.

 ### Added
+- **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
+  - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
+  - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window
+  - Pre-flight stream validation, and a bounded failed-events store (count and TTL). Only a non-retryable rejection is recorded in the failed-events store. A retryable condition keeps the entry on the live queue until it is delivered
+  - Operator guide at `docs/operations/nats-jetstream.md`
 - **OpenStack Keystone Authentication Integration**: Full support for OpenStack Keystone authentication via X-Auth-Token headers
  - Tower-based middleware (`KeystoneAuthLayer`) self-contained within `rustfs-keystone` crate
  - Task-local storage for async-safe credential passing between middleware and auth handlers
@@ -31,6 +31,8 @@ make build-docker BUILD_OS=ubuntu22.04
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
 - CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
+- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
+  policy: [docs/testing/README.md](docs/testing/README.md)
 - Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
  [docs/operations/tier-ilm-debugging.md](docs/operations/tier-ilm-debugging.md)

@@ -68,6 +68,8 @@ make pre-pr

 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.

+> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
+
 ### 🔒 Automated Pre-commit Hooks
 #### What `make pre-commit` and `make pre-pr` actually run

@@ -52,6 +52,7 @@ members = [
    "crates/extension-schema", # Extension schema contracts
    "crates/signer", # client signer
    "crates/storage-api", # Storage API contracts
+    "crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
    "crates/targets", # Target-specific configurations and utilities
    "crates/trusted-proxies", # Trusted proxies management
    "crates/tls-runtime", # Project-wide TLS runtime foundation
@@ -67,7 +68,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.96.0"
-version = "1.0.0-beta.8"
+version = "1.0.0-beta.10-preview.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -84,55 +85,56 @@ redundant_clone = "warn"

 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-beta.8" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.8" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.8" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.8" }
-rustfs-common = { path = "crates/common", version = "1.0.0-beta.8" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.8" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-beta.8" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.8" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.8" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.8" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.8" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.8" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.8" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.8" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.8" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.8" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.8" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.8" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.8" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.8" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.8" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.8" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.8" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.8" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.8" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.8" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.8" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.8" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.8" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.8" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.8" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.8" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.8" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.8" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.8" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.8" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.8" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.8" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.8" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.8" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.8" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.8" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.8" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.8" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.10-preview.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.10-preview.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.10-preview.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.10-preview.1" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.10-preview.1" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.10-preview.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.10-preview.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.10-preview.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.10-preview.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.10-preview.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.10-preview.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.10-preview.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.10-preview.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.10-preview.1" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.10-preview.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.10-preview.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.10-preview.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.10-preview.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.10-preview.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.10-preview.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.10-preview.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.10-preview.1" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.10-preview.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.10-preview.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.10-preview.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.10-preview.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.10-preview.1" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.10-preview.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.10-preview.1" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.10-preview.1" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.10-preview.1" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.10-preview.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.10-preview.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.10-preview.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.10-preview.1" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.10-preview.1" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.10-preview.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.10-preview.1" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.10-preview.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.10-preview.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.10-preview.1" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.10-preview.1" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.10-preview.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.10-preview.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.10-preview.1" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
-async_zip = { version = "0.0.18", default-features = false, features = ["tokio", "deflate"] }
-mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "tracing"] }
+async_zip = { default-features = false, version = "0.0.18" }
+mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
@@ -143,32 +145,32 @@ futures-core = "0.3.32"
 futures-lite = "2.6.1"
 futures-util = "0.3.32"
 pollster = "1.0.1"
-pulsar = { version = "6.8.0", default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
-lapin = { version = "4.10.0", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
-hyper = { version = "1.10.1", features = ["http2", "http1", "server"] }
-hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
-hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+pulsar = { default-features = false, version = "6.8.0" }
+lapin = { default-features = false, version = "4.10.0" }
+hyper = { version = "1.10.1" }
+hyper-rustls = { default-features = false, version = "0.27.9" }
+hyper-util = { version = "0.1.20" }
 http = "1.4.2"
-http-body = "1.0.1"
-http-body-util = "0.1.3"
+http-body = "1.1.0"
+http-body-util = "0.1.4"
 minlz = "1.2.3"
-reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { default-features = false, version = "0.13.4" }
 rustfs-kafka-async = { version = "1.2.0" }
-socket2 = { version = "0.6.4", features = ["all"] }
-tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] }
-tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
+socket2 = { version = "0.6.5" }
+tokio = { version = "1.52.3" }
+tokio-rustls = { default-features = false, version = "0.26.4" }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
-tokio-util = { version = "0.7.18", features = ["io", "compat"] }
-tonic = { version = "0.14.6", features = ["gzip", "deflate"] }
+tokio-util = { version = "0.7.18" }
+tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
-tower = { version = "0.5.3", features = ["timeout"] }
-tower-http = { version = "0.7.0", features = ["cors"] }
+tower = { version = "0.5.3" }
+tower-http = { version = "0.7.0" }

 # Serialization and Data Formats
 apache-avro = "0.21.0"
-bytes = { version = "1.12.1", features = ["serde"] }
+bytes = { version = "1.12.1" }
 bytesize = "2.4.2"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -177,8 +179,8 @@ prost = "0.14.4"
 quick-xml = "0.41.0"
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
-serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.150", features = ["raw_value"] }
+serde = { version = "1.0.228" }
+serde_json = { version = "1.0.150" }
 serde_urlencoded = "0.7.1"

 # Cryptography and Security
@@ -186,33 +188,33 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.0", features = ["rand_core"] }
+aes-gcm = { version = "=0.11.0" }
 argon2 = { version = "=0.6.0-rc.8" }
 blake2 = "=0.11.0-rc.6"
 chacha20poly1305 = { version = "=0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
-jsonwebtoken = { version = "10.4.0", features = ["aws_lc_rs"] }
-openidconnect = { version = "4.0", default-features = false, features = ["accept-rfc3339-timestamps"] }
+jsonwebtoken = { version = "10.4.0" }
+openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { version = "0.23.41", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls = { default-features = false, version = "0.23.42" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.0"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
-zeroize = { version = "1.9.0", features = ["derive"] }
+zeroize = { version = "1.9.0" }

 # Time and Date
-chrono = { version = "0.4.45", features = ["serde"] }
+chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.32", features = ["serde"] }
-time = { version = "0.3.53", features = ["std", "parsing", "formatting", "macros", "serde"] }
+jiff = { version = "0.2.32" }
+time = { version = "0.3.53" }

 # Database
-deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
-tokio-postgres = { version = "0.7.18", default-features = false, features = ["runtime", "with-serde_json-1"] }
+deadpool-postgres = { version = "0.14" }
+tokio-postgres = { default-features = false, version = "0.7.18" }
 tokio-postgres-rustls = "0.14.0"

 # Utilities and Tools
@@ -223,34 +225,34 @@ atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.9.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-s3 = { version = "1.138.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
-aws-smithy-http-client = { version = "1.2.0", default-features = false, features = ["default-client", "rustls-aws-lc"] }
-aws-smithy-runtime-api = { version = "1.13.0", features = ["http-1x"] }
+aws-sdk-s3 = { default-features = false, version = "1.138.0" }
+aws-smithy-http-client = { default-features = false, version = "1.2.0" }
+aws-smithy-runtime-api = { version = "1.13.0" }
 aws-smithy-types = { version = "1.6.1" }
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.4"
-clap = { version = "4.6.1", features = ["derive", "env"] }
-const-str = { version = "1.1.0", features = ["std", "proc"] }
+clap = { version = "4.6.2" }
+const-str = { version = "1.1.0" }
 convert_case = "0.11.0"
-criterion = { version = "0.8", features = ["html_reports"] }
+criterion = { version = "0.8" }
 crossbeam-queue = "0.3.13"
 crossbeam-channel = "0.5.16"
 crossbeam-deque = "0.8.7"
 crossbeam-utils = "0.8.22"
-datafusion = { git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
+datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
 derive_builder = "0.20.2"
 enumset = "1.1.13"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.15.0"
-google-cloud-auth = "1.13.0"
-hashbrown = { version = "0.17.1", features = ["serde", "rayon"] }
+google-cloud-storage = "1.16.0"
+google-cloud-auth = "1.14.0"
+hashbrown = { version = "0.17.1" }
 hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
-ipnetwork = { version = "0.21.1", features = ["serde"] }
+ipnetwork = { version = "0.21.1" }
 lazy_static = "1.5.0"
 libc = "0.2.186"
 libsystemd = "0.7.2"
@@ -261,7 +263,7 @@ matchit = "0.9.2"
 md-5 = "0.11.0"
 md5 = "0.8.1"
 mime_guess = "2.0.5"
-moka = { version = "0.12.15", features = ["future"] }
+moka = { version = "0.12.15" }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
 nvml-wrapper = "0.12.1"
@@ -271,27 +273,27 @@ path-clean = "1.0.1"
 percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.2", features = ["serde"] }
+rand = { version = "0.10.2" }
 ratelimit = "0.10.1"
 rayon = "1.12.0"
-reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "7.0.1", features = ["simd-accel"] }
+reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.0" }
 #reed-solomon-erasure = { version = "6.0", features = ["simd-accel"], git = "https://github.com/houseme/reed-solomon-erasure",rev = "main" }
 reed-solomon-simd = "3.1.0"
-regex = { version = "1.13.0" }
-rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
-redis = { version = "1.3.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
-rustix = { version = "1.1.4", features = ["fs"] }
+regex = { version = "1.13.1" }
+rumqttc = { package = "rumqttc-next", version = "0.33.2" }
+redis = { version = "1.4.0" }
+rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { version = "0.14.1", features = ["minio"] }
+s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "ce69c3f10824535c7c24b2f71cdb2aaa4dffb5e0" }
 serial_test = "3.5.0"
-shadow-rs = { version = "2.0.0", default-features = false }
+shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
-smallvec = { version = "1.15.2", features = ["serde"] }
+smallvec = { version = "1.15.2" }
 smartstring = "1.0.1"
-snap = "1.1.1"
-starshard = { version = "2.2.1", features = ["rayon", "async", "serde"] }
-strum = { version = "0.28.0", features = ["derive"] }
+snap = "1.1.2"
+starshard = { version = "2.2.2" }
+strum = { version = "0.28.0" }
 sysinfo = "0.39.6"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
@@ -301,15 +303,15 @@ tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
 tracing-error = "0.2.1"
 tracing-opentelemetry = { version = "0.33" }
-tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
+tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.23.5", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.24.0" }
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 windows = { version = "0.62.2" }
-xxhash-rust = { version = "0.8.16", features = ["xxh64", "xxh3"] }
+xxhash-rust = { version = "0.8.17" }
 zip = "8.6.0"
 zstd = "0.13.3"

@@ -317,19 +319,19 @@ zstd = "0.13.3"
 metrics = "0.24.6"
 dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.32.0" }
-opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] }
-opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] }
-opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] }
-opentelemetry-semantic-conventions = { version = "0.32.1", features = ["semconv_experimental"] }
+opentelemetry-appender-tracing = { version = "0.32.0" }
+opentelemetry-otlp = { version = "0.32.0" }
+opentelemetry_sdk = { version = "0.32.1" }
+opentelemetry-semantic-conventions = { version = "0.32.1" }
 opentelemetry-stdout = { version = "0.32.0" }
-pyroscope = { version = "2.1.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.1.0" }

 # FTP and SFTP
-libunftp = { version = "0.23.0", features = ["experimental"] }
+libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "10.0.1" }
 rcgen = "0.14.8"
-russh = { version = "0.62.2", features = ["serde"] }
+russh = { version = "0.62.2" }
 russh-sftp = "2.3.0"

 # WebDAV
@@ -339,7 +341,7 @@ dav-server = "0.11.0"
 mimalloc = "0.1"
 hotpath = "0.21"
 # Snapshot testing for output format regression detection
-insta = { version = "1.48", features = ["yaml", "json"] }
+insta = { version = "1.48" }

 [workspace.metadata.cargo-shear]
 ignored = ["rustfs"]
@@ -353,12 +355,34 @@ debug = "line-tables-only"

 [profile.release]
 opt-level = 3
+lto = "thin"
+codegen-units = 1
+debug = 0
+split-debuginfo = "off"
+strip = "symbols"

 [profile.production]
 inherits = "release"
-lto = "fat"
-codegen-units = 1

 [profile.profiling]
 inherits = "release"
 debug = true
+strip = "none"
+
+# Pin hyper to a revision that carries the HTTP/1 "flush buffered data before
+# shutdown" fix (hyperium/hyper#4018, commit 72046cc7). This lands as a
+# `[patch.crates-io]` entry — not on the `hyper` workspace dependency — so that
+# every consumer in the tree, including the transitive `hyper-util` server path
+# (`conn::auto` / `GracefulShutdown`) that actually drives our connections,
+# resolves to the fixed hyper rather than the buggy crates.io copy.
+#
+# hyper <= 1.10.1 can call `poll_shutdown()` on the socket while response bytes
+# are still buffered (a prior `poll_flush()` returned `Poll::Pending` and the
+# result was discarded). A backpressured / slow-reading peer then receives a
+# graceful FIN before the full Content-Length body is flushed, which standard S3
+# clients (minio-go / warp) report as `unexpected EOF` on large-object GET under
+# load. The fix is not in any crates.io release yet as of hyper 1.10.1; drop
+# this patch once a released version (> 1.10.1) contains commit 72046cc7.
+# See rustfs/backlog#1232.
+[patch.crates-io]
+hyper = { git = "https://github.com/hyperium/hyper.git", rev = "ccc1e850dc0cda3e71b0acd11f60ca3d48d09034" }
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-FROM rust:1.95-trixie
+FROM rust:1.97-trixie

 RUN set -eux; \
    export DEBIAN_FRONTEND=noninteractive; \
@@ -32,7 +32,7 @@ ARG RUSTFS_BUILD_FEATURES=""
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.95-trixie AS builder
+FROM rust:1.97-trixie AS builder

 # Re-declare args after FROM
 ARG TARGETPLATFORM
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10-preview.1
 ```

 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10-preview.1
 ```

 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
@@ -217,7 +217,7 @@ setup_rust_environment() {
    # Set up environment variables for musl targets
    if [[ "$PLATFORM" == *"musl"* ]]; then
        print_message $YELLOW "Setting up environment for musl target..."
-        export RUSTFLAGS="--cfg tokio_unstable -C target-feature=-crt-static"
+        export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-feature=-crt-static"

        # For cargo-zigbuild, set up additional environment variables
        if command -v cargo-zigbuild &> /dev/null; then
@@ -434,7 +434,7 @@ build_binary() {
        fi
    else
        # Native compilation
-        build_cmd="RUSTFLAGS='--cfg tokio_unstable -Clink-arg=-lm' cargo build"
+        build_cmd="RUSTFLAGS='${RUSTFLAGS:+$RUSTFLAGS }-Clink-arg=-lm' cargo build"
    fi

    if [ "$BUILD_TYPE" = "release" ]; then
@@ -27,17 +27,17 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi

 [dependencies]
 rustfs-targets = { workspace = true }
-rustfs-config = { workspace = true, features = ["audit", "constants", "server-config-model"] }
+rustfs-config = { workspace = true, features = ["audit", "server-config-model"] }
 rustfs-s3-types = { workspace = true }
-chrono = { workspace = true }
-const-str = { workspace = true }
+chrono = { workspace = true, features = ["serde"] }
+const-str = { workspace = true, features = ["std", "proc"] }
 futures = { workspace = true }
-hashbrown = { workspace = true }
+hashbrown = { workspace = true, features = ["serde", "rayon"] }
 metrics = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = ["raw_value"] }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }

 [dev-dependencies]
@@ -32,6 +32,7 @@ const EVENT_AUDIT_BATCH_DISPATCH_COMPLETED: &str = "audit_batch_dispatch_complet
 const EVENT_AUDIT_TARGET_STATE_CHANGED: &str = "audit_target_state_changed";
 const EVENT_AUDIT_REPLAY_DELIVERED: &str = "audit_replay_delivered";
 const EVENT_AUDIT_REPLAY_RETRY_SCHEDULED: &str = "audit_replay_retry_scheduled";
+const EVENT_AUDIT_REPLAY_RETRY_EXHAUSTED: &str = "audit_replay_retry_exhausted";
 const EVENT_AUDIT_REPLAY_DROPPED: &str = "audit_replay_dropped";
 const EVENT_AUDIT_REPLAY_STREAM_STATUS: &str = "audit_replay_stream_status";

@@ -281,6 +282,7 @@ impl AuditPipeline {
                let delivery = target.delivery_snapshot();
                AuditTargetMetricSnapshot {
                    failed_messages: delivery.failed_messages,
+                    failed_store_length: delivery.failed_store_length,
                    queue_length: delivery.queue_length,
                    target_id: target.id().to_string(),
                    total_messages: delivery.total_messages,
@@ -463,18 +465,16 @@ impl AuditRuntimeFacade {
                            target.record_final_failure();
                            observability::record_target_failure();
                        }
-                        ReplayEvent::RetryExhausted { key, target } => {
+                        ReplayEvent::RetryExhausted { detail, key, target } => {
                            warn!(
-                                event = EVENT_AUDIT_REPLAY_DROPPED,
+                                event = EVENT_AUDIT_REPLAY_RETRY_EXHAUSTED,
                                component = LOG_COMPONENT_AUDIT,
                                subsystem = LOG_SUBSYSTEM_PIPELINE,
                                target_id = %target.id(),
                                replay_key = %key,
-                                reason = "retry_exhausted",
-                                "audit replay delivery"
+                                error = %detail,
+                                "audit replay retry budget exhausted, entry stays queued and retries"
                            );
-                            target.record_final_failure();
-                            observability::record_target_failure();
                        }
                        ReplayEvent::UnreadableEntry { key, error, target } => {
                            warn!(
@@ -30,6 +30,7 @@ const EVENT_AUDIT_CONFIG_RELOADED: &str = "audit_config_reloaded";
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct AuditTargetMetricSnapshot {
    pub failed_messages: u64,
+    pub failed_store_length: u64,
    pub queue_length: u64,
    pub target_id: String,
    pub total_messages: u64,
@@ -26,13 +26,14 @@ categories = ["web-programming", "development-tools", "network-programming"]
 documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"

 [dependencies]
-bytes = { workspace = true }
+bytes = { workspace = true, features = ["serde"] }
 crc-fast = { workspace = true }
 http = { workspace = true }
 base64-simd = { workspace = true }
 md-5 = { workspace = true }
 sha1 = { workspace = true }
 sha2 = { workspace = true }
+xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }

 [dev-dependencies]
 pretty_assertions = { workspace = true }
@@ -36,7 +36,7 @@ impl fmt::Display for UnknownChecksumAlgorithmError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
-            r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256")"#,
+            r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256", "sha512", "xxhash3", "xxhash64", "xxhash128")"#,
            self.checksum_algorithm
        )
    }
@@ -16,13 +16,20 @@ use crate::base64;
 use http::header::{HeaderMap, HeaderValue};

 use crate::Crc64Nvme;
-use crate::{CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256};
+use crate::{
+    CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256, Sha512,
+    Xxhash3, Xxhash64, Xxhash128,
+};

 pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
 pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
 pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
 pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256";
 pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme";
+pub const SHA_512_HEADER_NAME: &str = "x-amz-checksum-sha512";
+pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
+pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
+pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";

 #[allow(dead_code)]
 pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
@@ -85,6 +92,30 @@ impl HttpChecksum for Sha256 {
    }
 }

+impl HttpChecksum for Sha512 {
+    fn header_name(&self) -> &'static str {
+        SHA_512_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Xxhash3 {
+    fn header_name(&self) -> &'static str {
+        XXHASH_3_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Xxhash64 {
+    fn header_name(&self) -> &'static str {
+        XXHASH_64_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Xxhash128 {
+    fn header_name(&self) -> &'static str {
+        XXHASH_128_HEADER_NAME
+    }
+}
+
 impl HttpChecksum for Md5 {
    fn header_name(&self) -> &'static str {
        MD5_HEADER_NAME
@@ -35,6 +35,10 @@ pub const CRC_32_C_NAME: &str = "crc32c";
 pub const CRC_64_NVME_NAME: &str = "crc64nvme";
 pub const SHA_1_NAME: &str = "sha1";
 pub const SHA_256_NAME: &str = "sha256";
+pub const SHA_512_NAME: &str = "sha512";
+pub const XXHASH_3_NAME: &str = "xxhash3";
+pub const XXHASH_64_NAME: &str = "xxhash64";
+pub const XXHASH_128_NAME: &str = "xxhash128";
 pub const MD5_NAME: &str = "md5";

 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -46,6 +50,10 @@ pub enum ChecksumAlgorithm {
    Sha1,
    Sha256,
    Crc64Nvme,
+    Sha512,
+    Xxhash3,
+    Xxhash64,
+    Xxhash128,
 }

 impl FromStr for ChecksumAlgorithm {
@@ -62,6 +70,14 @@ impl FromStr for ChecksumAlgorithm {
            Ok(Self::Sha256)
        } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) {
            Ok(Self::Crc64Nvme)
+        } else if checksum_algorithm.eq_ignore_ascii_case(SHA_512_NAME) {
+            Ok(Self::Sha512)
+        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_3_NAME) {
+            Ok(Self::Xxhash3)
+        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_64_NAME) {
+            Ok(Self::Xxhash64)
+        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_128_NAME) {
+            Ok(Self::Xxhash128)
        } else {
            Err(UnknownChecksumAlgorithmError::new(checksum_algorithm))
        }
@@ -76,6 +92,10 @@ impl ChecksumAlgorithm {
            Self::Crc64Nvme => Box::<Crc64Nvme>::default(),
            Self::Sha1 => Box::<Sha1>::default(),
            Self::Sha256 => Box::<Sha256>::default(),
+            Self::Sha512 => Box::<Sha512>::default(),
+            Self::Xxhash3 => Box::<Xxhash3>::default(),
+            Self::Xxhash64 => Box::<Xxhash64>::default(),
+            Self::Xxhash128 => Box::<Xxhash128>::default(),
        }
    }

@@ -86,6 +106,10 @@ impl ChecksumAlgorithm {
            Self::Crc64Nvme => CRC_64_NVME_NAME,
            Self::Sha1 => SHA_1_NAME,
            Self::Sha256 => SHA_256_NAME,
+            Self::Sha512 => SHA_512_NAME,
+            Self::Xxhash3 => XXHASH_3_NAME,
+            Self::Xxhash64 => XXHASH_64_NAME,
+            Self::Xxhash128 => XXHASH_128_NAME,
        }
    }
 }
@@ -285,6 +309,165 @@ impl Checksum for Sha256 {
        Self::size()
    }
 }
+
+#[derive(Debug, Default)]
+struct Sha512 {
+    hasher: sha2::Sha512,
+}
+
+impl Sha512 {
+    fn update(&mut self, bytes: &[u8]) {
+        use sha2::Digest;
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        use sha2::Digest;
+        Bytes::copy_from_slice(self.hasher.finalize().as_slice())
+    }
+
+    fn size() -> u64 {
+        use sha2::Digest;
+        sha2::Sha512::output_size() as u64
+    }
+}
+
+impl Checksum for Sha512 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes);
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+/// XXH3 (64-bit) hasher with the canonical seed of 0.
+///
+/// The raw digest is a `u64` serialized as 8 big-endian bytes so that the value
+/// matches the server-side (`rustfs-rio`) computation for the same algorithm.
+struct Xxhash3 {
+    hasher: xxhash_rust::xxh3::Xxh3,
+}
+
+impl Default for Xxhash3 {
+    fn default() -> Self {
+        Self {
+            hasher: xxhash_rust::xxh3::Xxh3::new(),
+        }
+    }
+}
+
+impl Xxhash3 {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        8
+    }
+}
+
+impl Checksum for Xxhash3 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+/// XXH3 (128-bit) hasher with the canonical seed of 0.
+///
+/// The raw digest is a `u128` serialized as 16 big-endian bytes.
+struct Xxhash128 {
+    hasher: xxhash_rust::xxh3::Xxh3,
+}
+
+impl Default for Xxhash128 {
+    fn default() -> Self {
+        Self {
+            hasher: xxhash_rust::xxh3::Xxh3::new(),
+        }
+    }
+}
+
+impl Xxhash128 {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        Bytes::copy_from_slice(self.hasher.digest128().to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        16
+    }
+}
+
+impl Checksum for Xxhash128 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+/// XXH64 hasher with the canonical seed of 0.
+///
+/// The raw digest is a `u64` serialized as 8 big-endian bytes.
+struct Xxhash64 {
+    hasher: xxhash_rust::xxh64::Xxh64,
+}
+
+impl Default for Xxhash64 {
+    fn default() -> Self {
+        Self {
+            hasher: xxhash_rust::xxh64::Xxh64::new(0),
+        }
+    }
+}
+
+impl Xxhash64 {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        8
+    }
+}
+
+impl Checksum for Xxhash64 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
 #[allow(dead_code)]
 #[derive(Debug, Default)]
 struct Md5 {
@@ -455,4 +638,97 @@ mod tests {
        let error = "MD5".parse::<ChecksumAlgorithm>().expect_err("md5 should not parse");
        assert_eq!("MD5", error.checksum_algorithm());
    }
+
+    #[test]
+    fn test_additional_algorithms_parse_and_round_trip() {
+        // The AWS 2026-04 additional checksum algorithms must be recognised
+        // (case-insensitively) and round-trip through as_str().
+        for (name, expected) in [
+            ("sha512", ChecksumAlgorithm::Sha512),
+            ("SHA512", ChecksumAlgorithm::Sha512),
+            ("xxhash3", ChecksumAlgorithm::Xxhash3),
+            ("XXHASH3", ChecksumAlgorithm::Xxhash3),
+            ("xxhash64", ChecksumAlgorithm::Xxhash64),
+            ("xxhash128", ChecksumAlgorithm::Xxhash128),
+        ] {
+            let parsed = name.parse::<ChecksumAlgorithm>().expect("algorithm should parse");
+            assert_eq!(parsed, expected);
+            assert_eq!(expected.as_str().parse::<ChecksumAlgorithm>().unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_unknown_algorithm_never_panics_and_fails_closed() {
+        // Fail-closed contract: an unknown or garbage algorithm name must return
+        // an error instead of panicking or silently substituting another hasher.
+        for name in ["", "xxhash", "sha3", "crc16", "not-a-real-algo", "🦀"] {
+            assert!(name.parse::<ChecksumAlgorithm>().is_err(), "unknown algorithm {name:?} must fail closed");
+        }
+    }
+
+    #[test]
+    fn test_sha512_matches_direct_computation() {
+        use crate::Sha512;
+        use crate::http::SHA_512_HEADER_NAME;
+        use sha2::{Digest, Sha512 as Sha512Ref};
+
+        let mut checksum = Sha512::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let header = Box::new(checksum).headers();
+        let encoded = header.get(SHA_512_HEADER_NAME).expect("sha512 header present");
+        let got = base64_encoded_checksum_to_hex_string(encoded);
+
+        let mut reference = Sha512Ref::new();
+        reference.update(TEST_DATA.as_bytes());
+        let expected = reference.finalize().iter().fold(String::from("0x"), |mut acc, b| {
+            write!(acc, "{b:02X?}").unwrap();
+            acc
+        });
+        assert_eq!(got, expected);
+    }
+
+    #[test]
+    fn test_xxhash3_matches_direct_computation_big_endian_seed0() {
+        use crate::Xxhash3;
+        use xxhash_rust::xxh3::Xxh3;
+
+        let mut checksum = Xxhash3::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let raw = Box::new(checksum).finalize();
+
+        let mut reference = Xxh3::new();
+        reference.update(TEST_DATA.as_bytes());
+        assert_eq!(raw.len(), 8);
+        assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice());
+    }
+
+    #[test]
+    fn test_xxhash128_matches_direct_computation_big_endian_seed0() {
+        use crate::Xxhash128;
+        use xxhash_rust::xxh3::Xxh3;
+
+        let mut checksum = Xxhash128::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let raw = Box::new(checksum).finalize();
+
+        let mut reference = Xxh3::new();
+        reference.update(TEST_DATA.as_bytes());
+        assert_eq!(raw.len(), 16);
+        assert_eq!(&raw[..], reference.digest128().to_be_bytes().as_slice());
+    }
+
+    #[test]
+    fn test_xxhash64_matches_direct_computation_big_endian_seed0() {
+        use crate::Xxhash64;
+        use xxhash_rust::xxh64::Xxh64;
+
+        let mut checksum = Xxhash64::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let raw = Box::new(checksum).finalize();
+
+        let mut reference = Xxh64::new(0);
+        reference.update(TEST_DATA.as_bytes());
+        assert_eq!(raw.len(), 8);
+        assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice());
+    }
 }
@@ -28,14 +28,14 @@ categories = ["web-programming", "development-tools", "data-structures"]
 workspace = true

 [dependencies]
-tokio = { workspace = true }
-tonic = { workspace = true }
-uuid = { workspace = true }
-chrono = { workspace = true }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
+tonic = { workspace = true, features = ["gzip", "deflate"] }
+uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
+chrono = { workspace = true, features = ["serde"] }
 metrics = { workspace = true }
-serde = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
 rmp-serde = { workspace = true }
-s3s = { workspace = true }
+s3s = { workspace = true, features = ["minio"] }
 tracing = { workspace = true }

 [lib]
@@ -13,15 +13,15 @@ categories = ["concurrency", "filesystem"]
 [dependencies]
 # Internal crates
 rustfs-io-core = { workspace = true }
-serde = { workspace = true }
+serde = { workspace = true, features = ["derive"] }

 # Async runtime
-tokio = { workspace = true, features = ["sync"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }

 # Logging
 tracing = { workspace = true }

 [dev-dependencies]
-insta = { workspace = true }
-serde_json = { workspace = true }
-tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread"] }
+insta = { workspace = true, features = ["yaml", "json"] }
+serde_json = { workspace = true, features = ["raw_value"] }
+tokio = { workspace = true, features = ["test-util", "macros", "rt-multi-thread", "fs"] }
@@ -25,9 +25,9 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
 categories = ["web-programming", "development-tools", "config"]

 [dependencies]
-const-str = { workspace = true, optional = true }
-serde = { workspace = true, optional = true }
-serde_json = { workspace = true, optional = true }
+const-str = { workspace = true, optional = true, features = ["std", "proc"] }
+serde = { workspace = true, optional = true, features = ["derive"] }
+serde_json = { workspace = true, optional = true, features = ["raw_value"] }

 [lints]
 workspace = true
@@ -25,8 +25,11 @@ pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KE
 pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
 pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
 pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+pub const ENV_AUDIT_NATS_JETSTREAM_ENABLE: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_ENABLE";
+pub const ENV_AUDIT_NATS_JETSTREAM_STREAM_NAME: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_STREAM_NAME";
+pub const ENV_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS";

-pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 16] = &[
    ENV_AUDIT_NATS_ENABLE,
    ENV_AUDIT_NATS_ADDRESS,
    ENV_AUDIT_NATS_SUBJECT,
@@ -40,6 +43,9 @@ pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
    ENV_AUDIT_NATS_TLS_REQUIRED,
    ENV_AUDIT_NATS_QUEUE_DIR,
    ENV_AUDIT_NATS_QUEUE_LIMIT,
+    ENV_AUDIT_NATS_JETSTREAM_ENABLE,
+    ENV_AUDIT_NATS_JETSTREAM_STREAM_NAME,
+    ENV_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS,
 ];

 pub const AUDIT_NATS_KEYS: &[&str] = &[
@@ -56,5 +62,8 @@ pub const AUDIT_NATS_KEYS: &[&str] = &[
    crate::NATS_TLS_REQUIRED,
    crate::NATS_QUEUE_DIR,
    crate::NATS_QUEUE_LIMIT,
+    crate::NATS_JETSTREAM_ENABLE,
+    crate::NATS_JETSTREAM_STREAM_NAME,
+    crate::NATS_JETSTREAM_ACK_TIMEOUT_SECS,
    crate::COMMENT_KEY,
 ];
@@ -0,0 +1,52 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/// Enable or disable per-client rate limiting for the S3 API.
+///
+/// When enabled (and `RUSTFS_API_RATE_LIMIT_RPM` > 0), requests are throttled
+/// per client IP using a token bucket; over-limit requests receive
+/// `429 Too Many Requests` with a `Retry-After` header. Internode RPC/gRPC,
+/// health probes, and the console (which has its own limiter) are exempt.
+/// Environment variable: RUSTFS_API_RATE_LIMIT_ENABLE
+/// Example: RUSTFS_API_RATE_LIMIT_ENABLE=true
+pub const ENV_API_RATE_LIMIT_ENABLE: &str = "RUSTFS_API_RATE_LIMIT_ENABLE";
+
+/// Default for `RUSTFS_API_RATE_LIMIT_ENABLE`.
+///
+/// Disabled by default: RustFS ships permissive and operators opt in to
+/// abuse-protection hardening. When disabled the request path is unchanged.
+pub const DEFAULT_API_RATE_LIMIT_ENABLE: bool = false;
+
+/// Sustained S3 API request budget per client IP, in requests per minute.
+///
+/// `0` means unlimited (rate limiting stays inert even when enabled).
+/// Environment variable: RUSTFS_API_RATE_LIMIT_RPM
+/// Example: RUSTFS_API_RATE_LIMIT_RPM=6000
+pub const ENV_API_RATE_LIMIT_RPM: &str = "RUSTFS_API_RATE_LIMIT_RPM";
+
+/// Default for `RUSTFS_API_RATE_LIMIT_RPM`.
+///
+/// `0` (unlimited) so that setting only the enable switch cannot throttle
+/// traffic by surprise; operators must choose an explicit budget.
+pub const DEFAULT_API_RATE_LIMIT_RPM: u32 = 0;
+
+/// Burst capacity per client IP (maximum tokens in the bucket).
+///
+/// Allows short spikes above the sustained rate. `0` means "same as RPM".
+/// Environment variable: RUSTFS_API_RATE_LIMIT_BURST
+/// Example: RUSTFS_API_RATE_LIMIT_BURST=200
+pub const ENV_API_RATE_LIMIT_BURST: &str = "RUSTFS_API_RATE_LIMIT_BURST";
+
+/// Default for `RUSTFS_API_RATE_LIMIT_BURST` (`0` = same as RPM).
+pub const DEFAULT_API_RATE_LIMIT_BURST: u32 = 0;
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+pub(crate) mod api;
 pub(crate) mod app;
 pub(crate) mod body_limits;
 pub(crate) mod capacity;
@@ -79,6 +79,13 @@ pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const NATS_TLS_REQUIRED: &str = "tls_required";
 pub const NATS_QUEUE_DIR: &str = "queue_dir";
 pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+pub const NATS_JETSTREAM_ENABLE: &str = "jetstream_enable";
+pub const NATS_JETSTREAM_STREAM_NAME: &str = "jetstream_stream_name";
+pub const NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "jetstream_ack_timeout_secs";
+
+pub const NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS: u64 = 30;
+pub const NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS: u64 = 10;
+pub const NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS: u64 = 120;

 pub const PULSAR_BROKER: &str = "broker";
 pub const PULSAR_TOPIC: &str = "topic";
@@ -15,6 +15,8 @@
 #[cfg(feature = "constants")]
 pub mod constants;
 #[cfg(feature = "constants")]
+pub use constants::api::*;
+#[cfg(feature = "constants")]
 pub use constants::app::*;
 #[cfg(feature = "constants")]
 pub use constants::body_limits::*;
@@ -26,6 +26,9 @@ pub const NOTIFY_NATS_KEYS: &[&str] = &[
    crate::NATS_TLS_REQUIRED,
    crate::NATS_QUEUE_DIR,
    crate::NATS_QUEUE_LIMIT,
+    crate::NATS_JETSTREAM_ENABLE,
+    crate::NATS_JETSTREAM_STREAM_NAME,
+    crate::NATS_JETSTREAM_ACK_TIMEOUT_SECS,
    crate::COMMENT_KEY,
 ];

@@ -42,8 +45,11 @@ pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_
 pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
 pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
 pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+pub const ENV_NOTIFY_NATS_JETSTREAM_ENABLE: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_ENABLE";
+pub const ENV_NOTIFY_NATS_JETSTREAM_STREAM_NAME: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_STREAM_NAME";
+pub const ENV_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS";

-pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 16] = &[
    ENV_NOTIFY_NATS_ENABLE,
    ENV_NOTIFY_NATS_ADDRESS,
    ENV_NOTIFY_NATS_SUBJECT,
@@ -57,4 +63,7 @@ pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
    ENV_NOTIFY_NATS_TLS_REQUIRED,
    ENV_NOTIFY_NATS_QUEUE_DIR,
    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+    ENV_NOTIFY_NATS_JETSTREAM_ENABLE,
+    ENV_NOTIFY_NATS_JETSTREAM_STREAM_NAME,
+    ENV_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS,
 ];
@@ -37,10 +37,6 @@ pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
 pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
 pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
 pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
-/// Stable OpenTelemetry service instance identity for this RustFS process.
-pub const ENV_OBS_INSTANCE_ID: &str = "RUSTFS_OBS_INSTANCE_ID";
-/// Optional stable RustFS deployment identity used to correlate cluster telemetry.
-pub const ENV_OBS_CLUSTER_ID: &str = "RUSTFS_OBS_CLUSTER_ID";

 /// Per-signal enable/disable flags (default: true)
 pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
@@ -135,8 +131,6 @@ mod tests {
        assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
        assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
        assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
-        assert_eq!(ENV_OBS_INSTANCE_ID, "RUSTFS_OBS_INSTANCE_ID");
-        assert_eq!(ENV_OBS_CLUSTER_ID, "RUSTFS_OBS_CLUSTER_ID");
        assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
        assert_eq!(ENV_OBS_LOG_STDOUT_ENABLED, "RUSTFS_OBS_LOG_STDOUT_ENABLED");
        assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
@@ -27,9 +27,9 @@ categories = ["web-programming", "development-tools", "data-structures", "securi
 [dependencies]
 base64-simd = { workspace = true }
 hmac = { workspace = true }
-rand = { workspace = true }
-serde = { workspace = true }
-serde_json.workspace = true
+rand = { workspace = true, features = ["serde"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = ["raw_value"] }
 sha2 = { workspace = true }
 time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }

@@ -29,23 +29,23 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
 workspace = true

 [dependencies]
-aes-gcm = { workspace = true, optional = true }
+aes-gcm = { workspace = true, optional = true, features = ["rand_core"] }
 argon2 = { workspace = true, optional = true }
 chacha20poly1305 = { workspace = true, optional = true }
-jsonwebtoken = { workspace = true }
+jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
 base64-simd = { workspace = true }
 pbkdf2 = { workspace = true, optional = true }
-rand = { workspace = true }
+rand = { workspace = true, features = ["serde"] }
 rsa = { workspace = true, features = ["sha2"] }
 serde = { workspace = true, features = ["derive"] }
 sha2 = { workspace = true, optional = true }
 thiserror.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }


 [dev-dependencies]
 test-case.workspace = true
-time.workspace = true
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }

 [features]
 default = ["crypto", "fips"]
@@ -28,7 +28,7 @@ categories = ["data-structures", "filesystem"]
 workspace = true

 [dependencies]
-serde = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
 path-clean = { workspace = true }
 rmp-serde = { workspace = true }
 async-trait = { workspace = true }
@@ -38,44 +38,47 @@ futures.workspace = true
 rustfs-lock.workspace = true
 rustfs-protos.workspace = true
 rmp-serde.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-tonic = { workspace = true }
-tokio = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = ["raw_value"] }
+tonic = { workspace = true, features = ["gzip", "deflate"] }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
 tokio-stream = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
-bytes.workspace = true
+bytes = { workspace = true, features = ["serde"] }
 serial_test = { workspace = true }
-aws-sdk-s3.workspace = true
+aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-config = { workspace = true }
-aws-smithy-http-client.workspace = true
+aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
 async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
-reqwest = { workspace = true }
+http-body-util.workspace = true
+hyper = { workspace = true, features = ["http2", "http1", "server"] }
+hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "multipart"] }
 rustfs-signer.workspace = true
 tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
-uuid = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
+uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 urlencoding.workspace = true
 walkdir.workspace = true
 base64 = { workspace = true }
-rand = { workspace = true }
-chrono = { workspace = true }
+rand = { workspace = true, features = ["serde"] }
+chrono = { workspace = true, features = ["serde"] }
 md5 = { workspace = true }
 sha2 = { workspace = true }
 astral-tokio-tar = { workspace = true }
-s3s = { workspace = true }
+s3s = { workspace = true, features = ["minio"] }
 zstd.workspace = true
-time.workspace = true
-suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
+suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
 rcgen.workspace = true
 local-ip-address.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
-russh = { workspace = true }
+rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+russh = { workspace = true, features = ["serde"] }
 russh-sftp = { workspace = true }
 zip.workspace = true
-clap.workspace = true
+clap = { workspace = true, features = ["derive", "env"] }
@@ -0,0 +1,448 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Systematic HTTP e2e for the admin IAM management surface (backlog#1154
+//! peri-2, first batch): the full user / canned-policy / service-account CRUD
+//! lifecycle over real signed HTTP against a real binary, plus a non-admin
+//! denial probe for every management endpoint (reusing the sec-4 assertion
+//! pattern — the authorization gate itself is pinned by `admin_auth_test`).
+//!
+//! Before this suite the ~40 admin handler modules only had helper-level unit
+//! tests; no test proved that `mc admin user add` style flows work end to end
+//! (create -> attach policy -> the credential actually gains S3 access ->
+//! service account inherits it -> deletion revokes it).
+//!
+//! Later batches tracked on backlog#1154: config get/set, info, pools status,
+//! group lifecycle, import/export IAM.
+
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::{Client, Config};
+use http::header::{CONTENT_TYPE, HOST};
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use tokio::time::{Duration, sleep};
+
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+type BoxError = Box<dyn Error + Send + Sync>;
+
+/// Signs and sends an admin HTTP request with the given credential, returning
+/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
+/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
+async fn admin_request(
+    base_url: &str,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(StatusCode, String), BoxError> {
+    let url = format!("{base_url}{path_and_query}");
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let mut builder = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if body.is_some() {
+        builder = builder.header(CONTENT_TYPE, "application/json");
+    }
+
+    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
+    let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request = local_http_client().request(reqwest_method, &url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    let response = request.send().await?;
+    let status = response.status();
+    let text = response.text().await.unwrap_or_default();
+    Ok((status, text))
+}
+
+/// Root-credential admin request that must succeed; returns the response body.
+async fn admin_ok(
+    env: &RustFSTestEnvironment,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> Result<String, BoxError> {
+    let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
+    }
+    Ok(text)
+}
+
+fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
+    let config = Config::builder()
+        .credentials_provider(Credentials::new(access_key, secret_key, None, None, "e2e-admin-crud"))
+        .region(Region::new("us-east-1"))
+        .endpoint_url(url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+/// Retries an S3 PUT until IAM propagation makes it succeed (bounded).
+async fn wait_for_s3_put(client: &Client, bucket: &str, key: &str, within: Duration) -> TestResult {
+    let deadline = tokio::time::Instant::now() + within;
+    loop {
+        match client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(b"peri-2 probe"))
+            .send()
+            .await
+        {
+            Ok(_) => return Ok(()),
+            Err(err) => {
+                if tokio::time::Instant::now() >= deadline {
+                    return Err(format!("PUT {bucket}/{key} never succeeded: {err:?}").into());
+                }
+            }
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+/// A canned policy granting full S3 access to one bucket.
+fn bucket_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+/// Full user -> policy -> service-account lifecycle, proving each management
+/// call takes effect on the data plane, not just that the endpoint answers 200.
+#[tokio::test]
+#[serial]
+async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "peri2-crud";
+    let user = "peri2user";
+    let user_secret = "peri2usersecret";
+    let policy = "peri2-rw";
+
+    let root_client = env.create_s3_client();
+    root_client.create_bucket().bucket(bucket).send().await?;
+
+    // --- canned policy CRUD ---------------------------------------------------
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy}"),
+        Some(bucket_rw_policy(bucket)),
+    )
+    .await?;
+
+    let info = admin_ok(
+        &env,
+        http::Method::GET,
+        &format!("/rustfs/admin/v3/info-canned-policy?name={policy}"),
+        None,
+    )
+    .await?;
+    let info_json: serde_json::Value = serde_json::from_str(&info)?;
+    let statement_json = info_json.get("Policy").cloned().unwrap_or(info_json.clone());
+    assert!(
+        statement_json.to_string().contains("s3:*"),
+        "info-canned-policy must round-trip the policy document: {info}"
+    );
+
+    let listed = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-canned-policies", None).await?;
+    assert!(listed.contains(policy), "list-canned-policies must contain {policy}: {listed}");
+
+    // --- user CRUD --------------------------------------------------------------
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(serde_json::json!({ "secretKey": user_secret, "status": "enabled" }).to_string()),
+    )
+    .await?;
+
+    let users = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    assert!(users.contains(user), "list-users must contain {user}: {users}");
+
+    let user_info = admin_ok(&env, http::Method::GET, &format!("/rustfs/admin/v3/user-info?accessKey={user}"), None).await?;
+    let user_info: rustfs_madmin::UserInfo = serde_json::from_str(&user_info)
+        .map_err(|e| format!("user-info response must decode as rustfs_madmin::UserInfo: {e}"))?;
+    assert_eq!(user_info.status, rustfs_madmin::AccountStatus::Enabled);
+
+    // The fresh user is authenticatable but unauthorized for the bucket.
+    let user_client = build_s3_client(&env.url, user, user_secret);
+    let denied = user_client
+        .put_object()
+        .bucket(bucket)
+        .key("before-attach")
+        .body(ByteStream::from_static(b"x"))
+        .send()
+        .await;
+    assert!(denied.is_err(), "user without a policy must not be able to write to {bucket}");
+
+    // --- attach policy: the credential actually gains S3 access -----------------
+    admin_ok(
+        &env,
+        http::Method::POST,
+        "/rustfs/admin/v3/idp/builtin/policy/attach",
+        Some(serde_json::json!({ "policies": [policy], "user": user }).to_string()),
+    )
+    .await?;
+
+    wait_for_s3_put(&user_client, bucket, "after-attach.txt", Duration::from_secs(10)).await?;
+
+    let user_info = admin_ok(&env, http::Method::GET, &format!("/rustfs/admin/v3/user-info?accessKey={user}"), None).await?;
+    let user_info: rustfs_madmin::UserInfo = serde_json::from_str(&user_info)?;
+    assert!(
+        user_info.policy_name.as_deref().is_some_and(|p| p.contains(policy)),
+        "user-info must reflect the attached policy, got {:?}",
+        user_info.policy_name
+    );
+
+    // --- service account: create for the user, credential works, info/list pin it
+    let sa_resp = admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-service-accounts",
+        Some(serde_json::json!({ "targetUser": user }).to_string()),
+    )
+    .await?;
+    let sa_json: serde_json::Value = serde_json::from_str(&sa_resp)?;
+    let sa_ak = sa_json["credentials"]["accessKey"]
+        .as_str()
+        .ok_or(format!("add-service-accounts response missing credentials.accessKey: {sa_resp}"))?
+        .to_string();
+    let sa_sk = sa_json["credentials"]["secretKey"]
+        .as_str()
+        .ok_or(format!("add-service-accounts response missing credentials.secretKey: {sa_resp}"))?
+        .to_string();
+
+    let sa_client = build_s3_client(&env.url, &sa_ak, &sa_sk);
+    wait_for_s3_put(&sa_client, bucket, "via-service-account.txt", Duration::from_secs(10)).await?;
+
+    let sa_info = admin_ok(
+        &env,
+        http::Method::GET,
+        &format!("/rustfs/admin/v3/info-service-account?accessKey={sa_ak}"),
+        None,
+    )
+    .await?;
+    let sa_info: serde_json::Value = serde_json::from_str(&sa_info)?;
+    assert_eq!(
+        sa_info["parentUser"].as_str(),
+        Some(user),
+        "info-service-account must name the parent user: {sa_info}"
+    );
+
+    let sa_list = admin_ok(
+        &env,
+        http::Method::GET,
+        &format!("/rustfs/admin/v3/list-service-accounts?user={user}"),
+        None,
+    )
+    .await?;
+    let sa_list: rustfs_madmin::ListServiceAccountsResp = serde_json::from_str(&sa_list)
+        .map_err(|e| format!("list-service-accounts must decode as rustfs_madmin::ListServiceAccountsResp: {e}"))?;
+    assert!(
+        sa_list.accounts.iter().any(|a| a.access_key == sa_ak),
+        "list-service-accounts must contain {sa_ak}"
+    );
+
+    // --- deletion revokes access -------------------------------------------------
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/delete-service-account?accessKey={sa_ak}"),
+        None,
+    )
+    .await?;
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
+    loop {
+        let revoked = sa_client
+            .put_object()
+            .bucket(bucket)
+            .key("after-sa-delete")
+            .body(ByteStream::from_static(b"x"))
+            .send()
+            .await;
+        if revoked.is_err() {
+            break;
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err("deleted service account credential still works".into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+
+    // Disable then remove the user; the credential must stop working.
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-status?accessKey={user}&status=disabled"),
+        None,
+    )
+    .await?;
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
+    loop {
+        let disabled = user_client
+            .put_object()
+            .bucket(bucket)
+            .key("after-disable")
+            .body(ByteStream::from_static(b"x"))
+            .send()
+            .await;
+        if disabled.is_err() {
+            break;
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err("disabled user credential still works".into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user}"),
+        None,
+    )
+    .await?;
+    let users = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    assert!(!users.contains(user), "removed user must disappear from list-users: {users}");
+
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-canned-policy?name={policy}"),
+        None,
+    )
+    .await?;
+    let (status, _) = admin_request(
+        &env.url,
+        http::Method::GET,
+        &format!("/rustfs/admin/v3/info-canned-policy?name={policy}"),
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert!(!status.is_success(), "info-canned-policy must fail after remove-canned-policy");
+
+    env.stop_server();
+    Ok(())
+}
+
+/// Every management endpoint in the first batch must deny an authenticated but
+/// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the
+/// gate implementation itself is owned by sec-4 / admin_auth_test).
+#[tokio::test]
+#[serial]
+async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let limited = "peri2limited";
+    let limited_secret = "peri2limitedsecret";
+    let other = "peri2other";
+    for (user, secret) in [(limited, limited_secret), (other, "peri2othersecret")] {
+        admin_ok(
+            &env,
+            http::Method::PUT,
+            &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+            Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
+        )
+        .await?;
+    }
+
+    // Admin-only management endpoints. Targeted probes aim at a DIFFERENT user
+    // (`other`): operations a principal performs on itself run in `deny_only`
+    // mode (see should_check_deny_only in rustfs/src/admin/handlers/user.rs)
+    // and would not be denied. Also deliberately excludes the self-service
+    // account endpoints (add/list service accounts), which any authenticated
+    // user may call for themselves.
+    let probes: Vec<(http::Method, String, Option<String>)> = vec![
+        (
+            http::Method::PUT,
+            "/rustfs/admin/v3/add-user?accessKey=peri2evil".to_string(),
+            Some(serde_json::json!({ "secretKey": "evilsecret123", "status": "enabled" }).to_string()),
+        ),
+        (http::Method::GET, "/rustfs/admin/v3/list-users".to_string(), None),
+        (http::Method::GET, format!("/rustfs/admin/v3/user-info?accessKey={other}"), None),
+        (
+            http::Method::PUT,
+            format!("/rustfs/admin/v3/set-user-status?accessKey={other}&status=disabled"),
+            None,
+        ),
+        (http::Method::DELETE, format!("/rustfs/admin/v3/remove-user?accessKey={other}"), None),
+        (
+            http::Method::PUT,
+            "/rustfs/admin/v3/add-canned-policy?name=peri2evilpolicy".to_string(),
+            Some(bucket_rw_policy("peri2-any")),
+        ),
+        (http::Method::GET, "/rustfs/admin/v3/list-canned-policies".to_string(), None),
+        (http::Method::GET, "/rustfs/admin/v3/info-canned-policy?name=readwrite".to_string(), None),
+        (
+            http::Method::DELETE,
+            "/rustfs/admin/v3/remove-canned-policy?name=readwrite".to_string(),
+            None,
+        ),
+        (
+            http::Method::POST,
+            "/rustfs/admin/v3/idp/builtin/policy/attach".to_string(),
+            Some(serde_json::json!({ "policies": ["readwrite"], "user": other }).to_string()),
+        ),
+    ];
+
+    for (method, path, body) in probes {
+        let (status, text) = admin_request(&env.url, method.clone(), &path, body, limited, limited_secret).await?;
+        assert_eq!(
+            status,
+            StatusCode::FORBIDDEN,
+            "non-admin credential must get 403 on {method} {path}, got {status}: {text}"
+        );
+        assert!(text.contains("AccessDenied"), "denial on {method} {path} must carry AccessDenied: {text}");
+    }
+
+    env.stop_server();
+    Ok(())
+}
@@ -0,0 +1,109 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E coverage for the opt-in per-client S3 API rate limit (backlog#1191):
+//! the layer must be wired into the real server stack, reject over-limit
+//! clients with `429` + `Retry-After`, keep health probes exempt, and stay
+//! completely inert with default configuration.
+
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use serial_test::serial;
+use tracing::info;
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+#[tokio::test]
+#[serial]
+async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    // Burst 50 leaves headroom for the readiness-poll ListBuckets calls that
+    // share the loopback client IP; refill (60 rpm = 1/s) is slow enough that
+    // a rapid burst below reliably exhausts the bucket.
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_API_RATE_LIMIT_ENABLE", "true"),
+            ("RUSTFS_API_RATE_LIMIT_RPM", "60"),
+            ("RUSTFS_API_RATE_LIMIT_BURST", "50"),
+        ],
+    )
+    .await?;
+
+    let client = local_http_client();
+    let list_buckets_url = format!("{}/", env.url);
+
+    let mut throttled = None;
+    let mut allowed = 0usize;
+    for _ in 0..80 {
+        let response = client.get(&list_buckets_url).send().await?;
+        if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
+            throttled = Some(response);
+            break;
+        }
+        allowed += 1;
+    }
+
+    let throttled = throttled.unwrap_or_else(|| panic!("no 429 within 80 rapid requests ({allowed} allowed) at burst 50"));
+    assert!(allowed > 0, "healthy traffic below the burst must not be throttled");
+    info!("rate limit engaged after {allowed} allowed requests");
+
+    let retry_after = throttled
+        .headers()
+        .get(reqwest::header::RETRY_AFTER)
+        .and_then(|v| v.to_str().ok())
+        .and_then(|v| v.parse::<u64>().ok())
+        .expect("429 must carry a numeric Retry-After header");
+    assert!(retry_after >= 1, "Retry-After must be at least one second, got {retry_after}");
+    assert_eq!(
+        throttled.headers().get("x-ratelimit-limit").and_then(|v| v.to_str().ok()),
+        Some("60"),
+        "429 must expose the configured limit"
+    );
+
+    let body = throttled.text().await?;
+    assert!(body.contains("<Code>TooManyRequests</Code>"), "S3-style error body expected: {body}");
+
+    // Health probes stay exempt even while the client budget is exhausted.
+    let health = client.get(format!("{}/health", env.url)).send().await?;
+    assert_ne!(
+        health.status(),
+        reqwest::StatusCode::TOO_MANY_REQUESTS,
+        "health probes must never be rate limited"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn api_rate_limit_stays_inert_by_default() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let client = local_http_client();
+    let list_buckets_url = format!("{}/", env.url);
+
+    for i in 0..80 {
+        let response = client.get(&list_buckets_url).send().await?;
+        assert_ne!(
+            response.status(),
+            reqwest::StatusCode::TOO_MANY_REQUESTS,
+            "request {i} was throttled although rate limiting is disabled by default"
+        );
+    }
+
+    Ok(())
+}
@@ -19,8 +19,10 @@
 mod tests {
    use crate::common::{RustFSTestEnvironment, init_logging};
    use aws_sdk_s3::Client;
+    use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
    use aws_sdk_s3::primitives::ByteStream;
    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
+    use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
    use base64::Engine;
    use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
    use serial_test::serial;
@@ -31,6 +33,25 @@ mod tests {
        env.create_s3_client()
    }

+    /// Client with boto/SDK automatic checksum calculation disabled, so the ONLY
+    /// checksum on the wire is the one the test injects. Needed because the SDK's
+    /// default (crc32) would otherwise collide with the additional-algorithm header
+    /// we inject via `mutate_request` for XXHash/SHA-512/MD5 (which have no typed
+    /// SDK builder). Mirrors the boto3 `request_checksum_calculation=when_required`.
+    fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> Client {
+        let creds = Credentials::new(&env.access_key, &env.secret_key, None, None, "e2e-additional-checksum");
+        let config = aws_sdk_s3::Config::builder()
+            .credentials_provider(creds)
+            .region(Region::new("us-east-1"))
+            .endpoint_url(format!("http://{}", env.address))
+            .force_path_style(true)
+            .behavior_version_latest()
+            .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
+            .http_client(SmithyHttpClientBuilder::new().build_http())
+            .build();
+        Client::from_conf(config)
+    }
+
    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        match client.create_bucket().bucket(bucket).send().await {
            Ok(_) => {
@@ -459,4 +480,87 @@ mod tests {
            "Multipart object should report the same full-object CRC64NVME as direct upload"
        );
    }
+
+    /// Integration test for the AWS 2026-04 additional checksum algorithms
+    /// (XXHash3/64/128, SHA-512, MD5). aws_sdk_s3 has no typed builder for these, so the
+    /// `x-amz-checksum-<algo>` header is injected via `mutate_request` (value computed by
+    /// rustfs-rio, which is byte-for-byte identical to awscrt). Verifies the server
+    /// verifies-on-write: a correct value is accepted and the object stored; a wrong
+    /// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header
+    /// echo round-trip is additionally exercised by the boto3+awscrt e2e.
+    #[tokio::test]
+    #[serial]
+    async fn test_additional_checksums_verify_on_write() {
+        init_logging();
+        info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client_no_auto_checksum(&env);
+        let bucket = "test-additional-checksums";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let content: &[u8] = b"additional-checksum verify-on-write payload for xxhash/sha512/md5";
+
+        for (ty, header) in [
+            (RioChecksumType::XXHASH3, "x-amz-checksum-xxhash3"),
+            (RioChecksumType::XXHASH64, "x-amz-checksum-xxhash64"),
+            (RioChecksumType::XXHASH128, "x-amz-checksum-xxhash128"),
+            (RioChecksumType::SHA512, "x-amz-checksum-sha512"),
+            (RioChecksumType::MD5, "x-amz-checksum-md5"),
+        ] {
+            // Correct checksum -> accepted, and the object is stored intact.
+            let good = Checksum::new_from_data(ty, content).expect("compute checksum").encoded;
+            let ok_key = format!("ok-{header}");
+            let put = client
+                .put_object()
+                .bucket(bucket)
+                .key(&ok_key)
+                .body(ByteStream::from_static(content))
+                .customize()
+                .mutate_request(move |req| {
+                    req.headers_mut().insert(header, good.clone());
+                })
+                .send()
+                .await;
+            assert!(put.is_ok(), "{header}: correct checksum must be accepted: {:?}", put.err());
+            let got = client
+                .get_object()
+                .bucket(bucket)
+                .key(&ok_key)
+                .send()
+                .await
+                .expect("GetObject");
+            let body = got.body.collect().await.expect("collect body").into_bytes();
+            assert_eq!(body.as_ref(), content, "{header}: stored body must match uploaded content");
+
+            // Wrong checksum -> rejected with BadDigest, and nothing is stored.
+            let bad = Checksum::new_from_data(ty, b"a totally different payload")
+                .expect("compute checksum")
+                .encoded;
+            let bad_key = format!("bad-{header}");
+            let put_bad = client
+                .put_object()
+                .bucket(bucket)
+                .key(&bad_key)
+                .body(ByteStream::from_static(content))
+                .customize()
+                .mutate_request(move |req| {
+                    req.headers_mut().insert(header, bad.clone());
+                })
+                .send()
+                .await;
+            assert!(put_bad.is_err(), "{header}: a mismatched checksum must be rejected");
+            let msg = format!("{:?}", put_bad.err().unwrap());
+            assert!(
+                msg.contains("BadDigest") || msg.to_lowercase().contains("digest") || msg.to_lowercase().contains("checksum"),
+                "{header}: expected a BadDigest/checksum error, got: {msg}"
+            );
+            let head = client.head_object().bucket(bucket).key(&bad_key).send().await;
+            assert!(head.is_err(), "{header}: nothing must be stored after a rejected PutObject");
+
+            info!("PASSED additional-checksum verify-on-write: {header}");
+        }
+    }
 }
@@ -446,6 +446,13 @@ impl RustFSTestEnvironment {
        self.start_rustfs_server_inner(extra_args, &[], false).await
    }

+    pub async fn start_rustfs_server_without_cleanup_with_env(
+        &mut self,
+        extra_env: &[(&str, &str)],
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(vec![], extra_env, false).await
+    }
+
    /// Wait for RustFS server to be ready.
    ///
    /// A listening TCP port is not sufficient here: the process may accept
@@ -514,6 +521,30 @@ impl RustFSTestEnvironment {
            }
        }
    }
+
+    /// Restart this server in place, preserving its on-disk data directory,
+    /// listen address, and credentials.
+    ///
+    /// Failure-recovery tests (backlog#1147 repl-5) need the *same* instance to
+    /// come back up after a crash/stop with its persisted state intact:
+    /// per-object replication status in `xl.meta`, the MRF recovery file, and
+    /// resync metadata all live under `temp_dir`, which this method keeps. The
+    /// address is reused too (the server sets `SO_REUSEADDR`/`SO_REUSEPORT`), so
+    /// existing S3 clients keep working without reconfiguration.
+    ///
+    /// The previous process is stopped first. `cleanup_existing` is false so a
+    /// sibling server sharing the harness (e.g. a replication target on another
+    /// port) is left untouched — only this instance is recycled. Pass the same
+    /// `extra_args` / `extra_env` the instance was originally started with;
+    /// background tasks read their env once at startup.
+    pub async fn restart_server_preserving_data(
+        &mut self,
+        extra_args: Vec<&str>,
+        extra_env: &[(&str, &str)],
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.stop_server();
+        self.start_rustfs_server_inner(extra_args, extra_env, false).await
+    }
 }

 impl Drop for RustFSTestEnvironment {
@@ -0,0 +1,191 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Over-the-wire smoke net for the embedded console listener (backlog#1154
+//! peri-4). The console is a second HTTP listener that serves the full S3/admin
+//! surface plus the unauthenticated console routes under `/rustfs/console`;
+//! before this test its behaviour was only covered by in-process router tests
+//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
+//! credentials through the public console endpoints or fail-opened the
+//! protected surface on the console port had no regression net.
+//!
+//! Pinned wire contract (real binary, real TCP):
+//!   * `/rustfs/console/version` and `/rustfs/console/license` answer 200 JSON
+//!     without authentication, with complete fields and no credential material.
+//!   * `/rustfs/console/license` exposes only the coarse `licensed` flag.
+//!   * the console SPA prefix dispatches to the static handler, never to the
+//!     S3 API (no S3 error XML), whether or not console assets are embedded.
+//!   * unauthenticated requests to the admin API and the S3 root on the console
+//!     listener are denied (403 AccessDenied) — the extra listener does not
+//!     fail-open the protected surface.
+//!   * a listener with the console disabled (the main S3 port here) does not
+//!     serve the unauthenticated console endpoints at all.
+
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use serial_test::serial;
+use std::error::Error;
+use tokio::time::{Duration, sleep};
+
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+/// Polls the console version endpoint until the console listener accepts
+/// requests. The harness readiness check only covers the S3 listener; the
+/// console listener of the same process may come up moments later.
+async fn wait_for_console_ready(console_base: &str) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let client = local_http_client();
+    let url = format!("{console_base}/rustfs/console/version");
+    let mut last_err = String::new();
+    for _ in 0..40 {
+        match client.get(&url).send().await {
+            Ok(response) if response.status() == reqwest::StatusCode::OK => return Ok(response),
+            Ok(response) => last_err = format!("status {}", response.status()),
+            Err(err) => last_err = err.to_string(),
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+    Err(format!("console listener at {console_base} never became ready: {last_err}").into())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_console_over_the_wire_smoke() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    // A unique secret makes the credential-leak assertions below sharp: any
+    // occurrence of it in a console response body is a genuine leak, not a
+    // collision with a common default string.
+    env.access_key = format!("peri4ak{}", uuid::Uuid::new_v4().simple());
+    env.secret_key = format!("peri4sk{}", uuid::Uuid::new_v4().simple());
+
+    let console_port = RustFSTestEnvironment::find_available_port().await?;
+    let console_address = format!("127.0.0.1:{console_port}");
+    let console_base = format!("http://{console_address}");
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_CONSOLE_ENABLE", "true"),
+            ("RUSTFS_CONSOLE_ADDRESS", console_address.as_str()),
+        ],
+    )
+    .await?;
+
+    let client = local_http_client();
+
+    // --- /rustfs/console/version: 200 JSON, complete fields, no secrets ------
+    let version_response = wait_for_console_ready(&console_base).await?;
+    let content_type = version_response
+        .headers()
+        .get(reqwest::header::CONTENT_TYPE)
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string();
+    assert!(content_type.starts_with("application/json"), "version content-type: {content_type}");
+    let version_body = version_response.text().await?;
+    let version_json: serde_json::Value = serde_json::from_str(&version_body)?;
+    for field in ["version", "version_info", "date"] {
+        assert!(
+            version_json[field].as_str().is_some_and(|v| !v.is_empty()),
+            "console version field {field} missing or empty: {version_body}"
+        );
+    }
+    assert!(
+        !version_body.contains(&env.secret_key) && !version_body.contains(&env.access_key),
+        "console version response leaks credentials: {version_body}"
+    );
+
+    // --- /rustfs/console/license: coarse licensed flag only ------------------
+    let license_response = client.get(format!("{console_base}/rustfs/console/license")).send().await?;
+    assert_eq!(license_response.status(), reqwest::StatusCode::OK);
+    let license_body = license_response.text().await?;
+    let license_json: serde_json::Value = serde_json::from_str(&license_body)?;
+    assert!(
+        license_json["licensed"].is_boolean(),
+        "license payload must expose a licensed bool: {license_body}"
+    );
+    let license_keys: Vec<&String> = license_json.as_object().map(|o| o.keys().collect()).unwrap_or_default();
+    assert_eq!(
+        license_keys,
+        vec!["licensed"],
+        "public license endpoint must not expose license metadata: {license_body}"
+    );
+    assert!(
+        !license_body.contains(&env.secret_key) && !license_body.contains(&env.access_key),
+        "console license response leaks credentials: {license_body}"
+    );
+
+    // --- console SPA prefix dispatches to the static handler, not the S3 API -
+    // With release console assets embedded this is 200 text/html; a from-source
+    // binary embeds an empty static dir and serves the handler's 404 fallback.
+    // Either way it must never fall through to an S3 handler (S3 error XML).
+    let spa_response = client.get(format!("{console_base}/rustfs/console/")).send().await?;
+    let spa_status = spa_response.status();
+    let spa_content_type = spa_response
+        .headers()
+        .get(reqwest::header::CONTENT_TYPE)
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string();
+    let spa_body = spa_response.text().await?;
+    assert!(
+        !spa_body.contains("<Error>"),
+        "console SPA route fell through to the S3 API: status {spa_status}, body {spa_body}"
+    );
+    match spa_status {
+        reqwest::StatusCode::OK => {
+            assert!(
+                spa_content_type.starts_with("text/html"),
+                "embedded console index must be html, got {spa_content_type}"
+            );
+        }
+        reqwest::StatusCode::NOT_FOUND => {
+            assert!(
+                spa_body.contains("RustFS"),
+                "asset-less console 404 must come from the console static handler: {spa_body}"
+            );
+        }
+        other => panic!("console SPA route returned unexpected status {other}: {spa_body}"),
+    }
+
+    // --- protected surface on the console listener stays authenticated -------
+    let admin_response = client.get(format!("{console_base}/rustfs/admin/v3/info")).send().await?;
+    assert_eq!(
+        admin_response.status(),
+        reqwest::StatusCode::FORBIDDEN,
+        "unauthenticated admin API on the console listener must be denied"
+    );
+    let admin_body = admin_response.text().await?;
+    assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
+
+    let s3_root_response = client.get(format!("{console_base}/")).send().await?;
+    assert_eq!(
+        s3_root_response.status(),
+        reqwest::StatusCode::FORBIDDEN,
+        "unauthenticated S3 root on the console listener must be denied"
+    );
+
+    // --- console disabled on a listener means no console surface at all ------
+    // The main S3 listener of this same process runs with the console disabled,
+    // so its console paths must not answer with the unauthenticated console
+    // payloads (they fall through to the authenticated S3 surface instead).
+    let s3_listener_console = client.get(format!("{}/rustfs/console/version", env.url)).send().await?;
+    assert_ne!(
+        s3_listener_console.status(),
+        reqwest::StatusCode::OK,
+        "console endpoints must not be served on a console-disabled listener"
+    );
+
+    env.stop_server();
+    Ok(())
+}
@@ -67,6 +67,10 @@ mod bucket_policy_check_test;
 #[cfg(test)]
 mod security_boundary_test;

+// Opt-in per-client S3 API rate limiting (backlog#1191)
+#[cfg(test)]
+mod api_rate_limit_test;
+
 // Admin authorization gate: non-admin denial + root-credential lifecycle (backlog#1151 sec-4)
 #[cfg(test)]
 mod admin_auth_test;
@@ -191,6 +195,30 @@ mod stale_multipart_cleanup_cluster_test;
 #[cfg(test)]
 mod object_lambda_test;

+// S3 event-notification webhook delivery end-to-end (backlog#1154 peri-1):
+// configure webhook target -> PutBucketNotificationConfiguration -> object
+// operation -> event delivered, plus filter negatives and store-queue redelivery.
+#[cfg(test)]
+mod notification_webhook_test;
+
+// TLS certificate hot-reload live-listener e2e (backlog#1154 peri-5): swap
+// certificates without a restart, existing sessions survive, bad material is
+// fail-safe (old certificate keeps serving, failure is logged).
+#[cfg(test)]
+mod tls_hot_reload_test;
+
+// Console listener over-the-wire smoke (backlog#1154 peri-4): public console
+// endpoints answer without credentials or leaks, the SPA prefix never falls
+// through to the S3 API, and the protected surface stays authenticated.
+#[cfg(test)]
+mod console_smoke_test;
+
+// Admin IAM management CRUD e2e (backlog#1154 peri-2): user / canned-policy /
+// service-account lifecycle over signed HTTP with data-plane effect assertions,
+// plus non-admin 403 probes per endpoint (sec-4 pattern).
+#[cfg(test)]
+mod admin_iam_crud_test;
+
 // Replication extension end-to-end regression tests
 #[cfg(test)]
 mod replication_extension_test;
@@ -0,0 +1,594 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression net for the S3 event-notification pipeline
+//! (backlog#1154 peri-1). Proves the full "configure webhook target ->
+//! PutBucketNotificationConfiguration -> object operation -> event delivered"
+//! chain against a real rustfs binary and a real HTTP receiver, which no other
+//! test covers: the target-plugin suites stop at broker integration and the
+//! object-lambda suite only exercises the webhook target as a transform
+//! function, never as an S3 event sink.
+//!
+//! Coverage:
+//!   * PUT / multipart-complete / DELETE each deliver one event with the correct
+//!     eventName, bucket, key, versionId and eTag.
+//!   * prefix/suffix filters drop non-matching keys (rule-engine gate).
+//!   * an event queued while the target endpoint is unreachable is redelivered
+//!     from the on-disk store once the endpoint recovers (store-and-forward).
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
+    NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
+};
+use http::header::{CONTENT_TYPE, HOST};
+use local_ip_address::local_ip;
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serde_json::Value;
+use serial_test::serial;
+use std::error::Error;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant, timeout};
+
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+type BoxError = Box<dyn Error + Send + Sync>;
+
+/// Region embedded in the target ARN. Only the `id:name` tail of the ARN is used
+/// for target lookup (see `process_queue_configurations`), so the region is
+/// nominal, but it must be present for `ARN::parse` to succeed.
+const NOTIFY_REGION: &str = "us-east-1";
+
+/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
+/// so the ARN a notification rule references is
+/// `arn:rustfs:sqs:<region>:<name>:webhook`.
+fn target_arn(target_name: &str) -> String {
+    format!("arn:rustfs:sqs:{NOTIFY_REGION}:{target_name}:webhook")
+}
+
+// ---------------------------------------------------------------------------
+// In-test HTTP event receiver
+// ---------------------------------------------------------------------------
+
+/// Reads one HTTP request off the stream, returning its method and body. Handles
+/// the target's HEAD reachability probe (no body) and POST event deliveries.
+async fn read_http_message(stream: &mut tokio::net::TcpStream) -> Result<(String, Vec<u8>), BoxError> {
+    let mut buffer = Vec::new();
+    let mut chunk = [0_u8; 4096];
+
+    let header_end = loop {
+        let read = stream.read(&mut chunk).await?;
+        if read == 0 {
+            return Err("connection closed before request headers were complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+        if let Some(pos) = buffer.windows(4).position(|w| w == b"\r\n\r\n") {
+            break pos;
+        }
+    };
+
+    let header_text = std::str::from_utf8(&buffer[..header_end])?;
+    let mut lines = header_text.split("\r\n");
+    let request_line = lines.next().ok_or("missing request line")?;
+    let method = request_line.split_whitespace().next().ok_or("missing method")?.to_string();
+
+    let mut content_length = 0usize;
+    for line in lines {
+        if let Some((name, value)) = line.split_once(':')
+            && name.trim().eq_ignore_ascii_case("content-length")
+        {
+            content_length = value.trim().parse::<usize>().unwrap_or(0);
+        }
+    }
+
+    let body_offset = header_end + 4;
+    while buffer.len().saturating_sub(body_offset) < content_length {
+        let read = stream.read(&mut chunk).await?;
+        if read == 0 {
+            return Err("connection closed before request body was complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+    }
+
+    Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
+}
+
+/// Drives an accepted listener: answers every request 200 OK (so the target's
+/// HEAD reachability probe reports it online) and forwards each non-empty POST
+/// body, parsed as the S3 event envelope JSON, to `tx`.
+fn serve_event_collector(listener: TcpListener, tx: mpsc::UnboundedSender<Value>) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            let Ok((mut stream, _)) = listener.accept().await else {
+                return;
+            };
+            let tx = tx.clone();
+            tokio::spawn(async move {
+                if let Ok(Ok((method, body))) = timeout(Duration::from_secs(5), read_http_message(&mut stream)).await {
+                    let _ = stream
+                        .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+                        .await;
+                    let _ = stream.shutdown().await;
+                    if method == "POST"
+                        && !body.is_empty()
+                        && let Ok(value) = serde_json::from_slice::<Value>(&body)
+                    {
+                        let _ = tx.send(value);
+                    }
+                }
+            });
+        }
+    })
+}
+
+/// Binds a fresh collector on a random port. Returns its `/events`
+/// endpoint URL, a receiver of parsed event envelopes, and the serving task.
+async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>), BoxError> {
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let port = listener.local_addr()?.port();
+    let endpoint_ip = local_ip()?;
+    let (tx, rx) = mpsc::unbounded_channel();
+    let handle = serve_event_collector(listener, tx);
+    Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
+}
+
+/// Decoded object key of the first record in an event envelope.
+fn event_key(envelope: &Value) -> Option<String> {
+    let raw = envelope["Records"][0]["s3"]["object"]["key"].as_str()?;
+    Some(
+        urlencoding::decode(raw)
+            .map(|d| d.into_owned())
+            .unwrap_or_else(|_| raw.to_string()),
+    )
+}
+
+/// Waits until an event targeting `key` whose `EventName` starts with
+/// `event_prefix` arrives, returning the full envelope. Other envelopes are
+/// discarded, so a lingering duplicate of an earlier event (delivery is
+/// at-least-once) or the create event of a key that is later deleted never
+/// satisfies a wait for the opposite event type.
+async fn wait_for_event(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    key: &str,
+    event_prefix: &str,
+    within: Duration,
+) -> Result<Value, BoxError> {
+    let deadline = Instant::now() + within;
+    loop {
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            return Err(format!("no {event_prefix}* event for key {key} within {within:?}").into());
+        }
+        match timeout(remaining, rx.recv()).await {
+            Ok(Some(envelope)) => {
+                let key_matches = event_key(&envelope).as_deref() == Some(key);
+                let name_matches = envelope["EventName"].as_str().is_some_and(|n| n.starts_with(event_prefix));
+                if key_matches && name_matches {
+                    return Ok(envelope);
+                }
+            }
+            Ok(None) => return Err("event collector channel closed".into()),
+            Err(_) => return Err(format!("no {event_prefix}* event for key {key} within {within:?}").into()),
+        }
+    }
+}
+
+/// Collects every event until `stop_key` is seen (plus a short grace window to
+/// catch stragglers), or until `max_wait` elapses. Used to prove a negative:
+/// non-matching keys must never appear even though a matching sentinel does.
+async fn collect_until(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    stop_key: &str,
+    max_wait: Duration,
+    grace: Duration,
+) -> Vec<Value> {
+    let hard_deadline = Instant::now() + max_wait;
+    let mut soft_deadline: Option<Instant> = None;
+    let mut collected = Vec::new();
+    loop {
+        let deadline = soft_deadline.unwrap_or(hard_deadline).min(hard_deadline);
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            break;
+        }
+        match timeout(remaining, rx.recv()).await {
+            Ok(Some(envelope)) => {
+                let matched = event_key(&envelope).as_deref() == Some(stop_key);
+                collected.push(envelope);
+                if matched && soft_deadline.is_none() {
+                    soft_deadline = Some(Instant::now() + grace);
+                }
+            }
+            Ok(None) => break,
+            Err(_) => break,
+        }
+    }
+    collected
+}
+
+// ---------------------------------------------------------------------------
+// Admin target configuration (signed admin HTTP)
+// ---------------------------------------------------------------------------
+
+async fn signed_admin_request(
+    env: &RustFSTestEnvironment,
+    method: http::Method,
+    url: &str,
+    body: Option<Vec<u8>>,
+) -> Result<reqwest::Response, BoxError> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let mut builder = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if body.is_some() {
+        builder = builder.header(CONTENT_TYPE, "application/json");
+    }
+
+    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        builder.body(Body::empty())?,
+        content_len,
+        &env.access_key,
+        &env.secret_key,
+        "",
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request = crate::common::local_http_client().request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    Ok(request.send().await?)
+}
+
+async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
+    let payload = serde_json::json!({
+        "notify_enabled": true,
+        "audit_enabled": false,
+    });
+    let url = format!("{}/rustfs/admin/v3/module-switches", env.url);
+    let response = signed_admin_request(env, http::Method::PUT, &url, Some(payload.to_string().into_bytes())).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let text = response.text().await.unwrap_or_default();
+        return Err(format!("failed to enable notify module: {status} {text}").into());
+    }
+    Ok(())
+}
+
+/// Registers a webhook notification target with a persistent queue directory, so
+/// delivery goes through the durable store-and-forward path.
+async fn configure_webhook_target(env: &RustFSTestEnvironment, target_name: &str, endpoint: &str) -> TestResult {
+    let queue_dir = format!("{}/notify-queue-{target_name}", env.temp_dir);
+    tokio::fs::create_dir_all(&queue_dir).await?;
+    let payload = serde_json::json!({
+        "key_values": [
+            { "key": "endpoint", "value": endpoint },
+            { "key": "queue_dir", "value": queue_dir },
+        ]
+    });
+    let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}", env.url);
+    let response = signed_admin_request(env, http::Method::PUT, &url, Some(payload.to_string().into_bytes())).await?;
+    let status = response.status();
+    if status != StatusCode::OK {
+        let text = response.text().await.unwrap_or_default();
+        return Err(format!("failed to configure webhook target {target_name}: {status} {text}").into());
+    }
+    Ok(())
+}
+
+/// Polls the admin target ARN list until the freshly configured target is
+/// registered in the runtime, so notification rules and object events do not
+/// race target activation.
+async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
+    let suffix = format!(":{target_name}:webhook");
+    let url = format!("{}/rustfs/admin/v3/target/arns", env.url);
+    for _ in 0..80 {
+        let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
+        if response.status() == StatusCode::OK {
+            let arns: Vec<String> = serde_json::from_slice(&response.bytes().await?)?;
+            if arns.iter().any(|arn| arn.ends_with(&suffix)) {
+                return Ok(());
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+    Err(format!("target {target_name} was not registered in admin ARNs").into())
+}
+
+/// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
+/// filtered to `prefix` + `suffix`.
+async fn put_notification_config(client: &Client, bucket: &str, target_name: &str, prefix: &str, suffix: &str) -> TestResult {
+    let key_filter = S3KeyFilter::builder()
+        .filter_rules(FilterRule::builder().name(FilterRuleName::Prefix).value(prefix).build())
+        .filter_rules(FilterRule::builder().name(FilterRuleName::Suffix).value(suffix).build())
+        .build();
+    let queue = QueueConfiguration::builder()
+        .id(format!("{target_name}-rule"))
+        .queue_arn(target_arn(target_name))
+        .events(Event::from("s3:ObjectCreated:*"))
+        .events(Event::from("s3:ObjectRemoved:*"))
+        .filter(NotificationConfigurationFilter::builder().key(key_filter).build())
+        .build()?;
+    let config = NotificationConfiguration::builder().queue_configurations(queue).build();
+    client
+        .put_bucket_notification_configuration()
+        .bucket(bucket)
+        .notification_configuration(config)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn enable_versioning(client: &Client, bucket: &str) -> TestResult {
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+fn trimmed_etag(value: Option<&str>) -> Option<String> {
+    value.map(|e| e.trim_matches('"').to_string())
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+/// PUT / multipart-complete / DELETE each deliver one event with correct fields,
+/// and the prefix/suffix filter drops non-matching keys.
+#[tokio::test]
+#[serial]
+async fn test_webhook_event_delivery_and_filtering() -> TestResult {
+    init_logging();
+
+    let (endpoint, mut rx, handle) = spawn_event_collector().await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    enable_notify_module(&env).await?;
+
+    let bucket = "peri1-events";
+    let target = "peri1events";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    enable_versioning(&client, bucket).await?;
+
+    configure_webhook_target(&env, target, &endpoint).await?;
+    wait_for_target_registered(&env, target).await?;
+    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+
+    // --- PUT: ObjectCreated:Put with exact eTag + versionId ------------------
+    let put_key = "uploads/report.dat";
+    let put = client
+        .put_object()
+        .bucket(bucket)
+        .key(put_key)
+        .body(ByteStream::from_static(b"peri-1 put body"))
+        .send()
+        .await?;
+    let put_version = put
+        .version_id()
+        .ok_or("PUT response missing versionId (versioning not enabled?)")?;
+
+    let created = wait_for_event(&mut rx, put_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    let record = &created["Records"][0];
+    let object = &record["s3"]["object"];
+    assert_eq!(
+        created["EventName"].as_str(),
+        Some("s3:ObjectCreated:Put"),
+        "envelope EventName: {created}"
+    );
+    assert!(
+        record["eventName"]
+            .as_str()
+            .is_some_and(|n| n.starts_with("s3:ObjectCreated:")),
+        "record eventName: {record}"
+    );
+    assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
+    assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
+    assert_eq!(
+        trimmed_etag(object["eTag"].as_str()),
+        trimmed_etag(put.e_tag()),
+        "eTag in event: {object}"
+    );
+
+    // --- Multipart complete: ObjectCreated:CompleteMultipartUpload -----------
+    let mp_key = "uploads/multi.dat";
+    let created_mp = client.create_multipart_upload().bucket(bucket).key(mp_key).send().await?;
+    let upload_id = created_mp.upload_id().ok_or("missing upload id")?;
+    let part = client
+        .upload_part()
+        .bucket(bucket)
+        .key(mp_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from_static(b"peri-1 multipart part body"))
+        .send()
+        .await?;
+    let complete = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(mp_key)
+        .upload_id(upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(part.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mp_event = wait_for_event(&mut rx, mp_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    let mp_record = &mp_event["Records"][0];
+    assert_eq!(
+        mp_event["EventName"].as_str(),
+        Some("s3:ObjectCreated:CompleteMultipartUpload"),
+        "multipart EventName: {mp_event}"
+    );
+    assert_eq!(mp_record["s3"]["bucket"]["name"].as_str(), Some(bucket));
+    assert_eq!(
+        trimmed_etag(mp_record["s3"]["object"]["eTag"].as_str()),
+        trimmed_etag(complete.e_tag()),
+        "multipart eTag in event: {mp_record}"
+    );
+
+    // --- Filter: non-matching prefix and suffix must never be delivered ------
+    let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
+    let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
+    let sentinel = "uploads/keep.dat";
+    for key in [wrong_prefix, wrong_suffix, sentinel] {
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(b"filter probe"))
+            .send()
+            .await?;
+    }
+    let collected = collect_until(&mut rx, sentinel, Duration::from_secs(20), Duration::from_secs(2)).await;
+    let seen: Vec<String> = collected.iter().filter_map(event_key).collect();
+    assert!(
+        seen.iter().any(|k| k == sentinel),
+        "matching sentinel {sentinel} was not delivered; saw {seen:?}"
+    );
+    assert!(
+        !seen.iter().any(|k| k == wrong_prefix),
+        "wrong-prefix key {wrong_prefix} bypassed the filter; saw {seen:?}"
+    );
+    assert!(
+        !seen.iter().any(|k| k == wrong_suffix),
+        "wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
+    );
+
+    // --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
+    let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
+    let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
+    let removed_record = &removed["Records"][0];
+    assert!(
+        removed["EventName"]
+            .as_str()
+            .is_some_and(|n| n.starts_with("s3:ObjectRemoved:")),
+        "delete EventName: {removed}"
+    );
+    if let Some(marker_version) = delete.version_id() {
+        assert_eq!(
+            removed_record["s3"]["object"]["versionId"].as_str(),
+            Some(marker_version),
+            "delete-marker versionId in event: {removed_record}"
+        );
+    }
+
+    env.stop_server();
+    handle.abort();
+    Ok(())
+}
+
+/// An event queued while the target endpoint is unreachable survives on the
+/// durable store and is redelivered once the endpoint comes back.
+#[tokio::test]
+#[serial]
+#[ignore = "FAILING deterministically on main since it landed (#4821): the target is created but never appears in /rustfs/admin/v3/target/arns, so wait_for_target_registered times out. Quarantined per the flake policy; remove with the fix for rustfs#4852"]
+async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    enable_notify_module(&env).await?;
+
+    let bucket = "peri1-redeliver";
+    let target = "peri1redeliver";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+
+    // Configure the target while its endpoint is reachable so activation and
+    // ARN registration complete deterministically. Registering against a dead
+    // endpoint stalls behind the reachability probe's timeout and flakes the
+    // registration wait on loaded runners (observed on the merge run of
+    // rustfs#4821).
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let port = listener.local_addr()?.port();
+    let endpoint_ip = local_ip()?;
+    let endpoint = format!("http://{endpoint_ip}.nip.io:{port}/events");
+    let (setup_tx, _setup_rx) = mpsc::unbounded_channel();
+    let setup_handle = serve_event_collector(listener, setup_tx);
+
+    configure_webhook_target(&env, target, &endpoint).await?;
+    wait_for_target_registered(&env, target).await?;
+    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+
+    // Take the endpoint down (drops the listener, so connections are refused —
+    // a retryable NotConnected), then PUT: the event cannot be delivered and
+    // must survive on the durable queue store.
+    setup_handle.abort();
+    let _ = setup_handle.await;
+
+    let key = "uploads/redeliver.dat";
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"queued while target down"))
+        .send()
+        .await?;
+
+    // Hold the endpoint down long enough for at least one replay attempt to
+    // fail (the replay worker scans the store every 500ms), so recovery below
+    // exercises real redelivery rather than a first-attempt success.
+    tokio::time::sleep(Duration::from_secs(2)).await;
+
+    // Bring the endpoint back on the same port; the replay worker retries with
+    // exponential backoff and delivers the queued event.
+    let listener = TcpListener::bind(("0.0.0.0", port)).await?;
+    let (tx, mut rx) = mpsc::unbounded_channel();
+    let handle = serve_event_collector(listener, tx);
+
+    let redelivered = wait_for_event(&mut rx, key, "s3:ObjectCreated:", Duration::from_secs(45)).await?;
+    assert_eq!(
+        redelivered["EventName"].as_str(),
+        Some("s3:ObjectCreated:Put"),
+        "redelivered EventName: {redelivered}"
+    );
+    assert_eq!(redelivered["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
+
+    env.stop_server();
+    handle.abort();
+    Ok(())
+}
@@ -22,3 +22,4 @@ mod lifecycle;
 mod lock;
 mod node_interact_test;
 mod sql;
+mod tiering;
@@ -0,0 +1,380 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Hermetic ILM transition main-path end-to-end test (backlog#1148 ilm-7).
+//!
+//! Two embedded `rustfs` servers, each with independent credentials, port and
+//! data directory:
+//!
+//! * `cold` — a second RustFS server wired as a [`TierType::RustFS`] remote tier.
+//! * `hot` — the source server that transitions objects to `cold`.
+//!
+//! There are no containers, no external S3 backend and no `awscurl`: the
+//! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like
+//! the other admin-API e2e suites in this crate. The RustFS warm backend has no
+//! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier
+//! to `cold` over `http://127.0.0.1:<port>`.
+//!
+//! A single test drives the full transition main path and pins the chain
+//! required by ilm-7:
+//!   1. `AddTier(RustFS)` on `hot` targeting `cold` — the real connectivity /
+//!      in-use probe runs (no `force`), so this also proves the tier is reachable.
+//!   2. A `Transition Days=0` rule installed before a multipart PUT transitions
+//!      the object immediately (the completion path enqueues it; the 1s scanner
+//!      cycle is only a backstop).
+//!   3. `HEAD` reports `x-amz-storage-class == <tier name>` and no `x-amz-restore`.
+//!   4. `GET` streams byte-identical data back through the warm backend, and the
+//!      content-type and user metadata survive the round trip (rustfs#2246).
+//!   5. Range `GET` within a part and across the part boundary read the correct
+//!      bytes from the tier (backlog#807).
+//!   6. The remote object is present in the cold-tier bucket after transition.
+//!   7. `DeleteObject` on `hot` drives free-version cleanup: the cold-tier copy
+//!      eventually disappears and the hot object is gone (no local residue).
+
+use crate::common::{RustFSTestEnvironment, local_http_client};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    BucketLifecycleConfiguration, CompletedMultipartUpload, CompletedPart, ExpirationStatus, LifecycleRule, LifecycleRuleFilter,
+    Transition, TransitionStorageClass,
+};
+use http::Method;
+use http::header::HOST;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use std::time::{Duration as StdDuration, Instant};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const TIER_NAME: &str = "COLDTIER";
+const TIER_BUCKET: &str = "ilm7-cold-tier";
+const TIER_PREFIX: &str = "tiered";
+const SOURCE_BUCKET: &str = "ilm7-hot";
+const OBJECT_KEY: &str = "tier/report.bin";
+const CONTENT_TYPE: &str = "application/x-ilm7";
+const USER_META_KEY: &str = "ilm7-origin";
+const USER_META_VAL: &str = "hermetic-transition";
+
+/// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
+/// internal part boundary sits at this offset.
+const PART0_SIZE: usize = 5 * 1024 * 1024;
+/// 1 MiB tail so the completed object is genuinely multipart (two parts).
+const PART1_SIZE: usize = 1024 * 1024;
+const OBJECT_SIZE: usize = PART0_SIZE + PART1_SIZE;
+
+/// Deterministic, position-dependent payload: adjacent offsets differ, so a
+/// misaligned range read is caught.
+fn payload() -> Vec<u8> {
+    (0..OBJECT_SIZE).map(|i| (i % 251) as u8).collect()
+}
+
+/// Sign and send an admin request in-process (no `awscurl`).
+///
+/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed
+/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being
+/// pre-hashed. Returns the response status and body text.
+async fn signed_admin_request(
+    base_url: &str,
+    method: Method,
+    path: &str,
+    body: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{base_url}{path}");
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
+
+    let request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+    let client = local_http_client();
+    let mut request_builder = client.request(method, url.as_str());
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if !body_bytes.is_empty() {
+        request_builder = request_builder.body(body_bytes);
+    }
+    let response = request_builder.send().await?;
+    let status = response.status();
+    let text = response.text().await?;
+    Ok((status, text))
+}
+
+/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
+///
+/// No `force`, so the server runs the real in-use / connectivity probe against
+/// `cold` (which requires the tier bucket to already exist there).
+async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
+    let body = serde_json::json!({
+        "type": "rustfs",
+        "rustfs": {
+            "name": TIER_NAME,
+            "endpoint": cold.url.as_str(),
+            "accessKey": cold.access_key.as_str(),
+            "secretKey": cold.secret_key.as_str(),
+            "bucket": TIER_BUCKET,
+            "prefix": TIER_PREFIX,
+            "region": "us-east-1",
+            "storageClass": ""
+        }
+    })
+    .to_string();
+
+    let (status, resp) = signed_admin_request(
+        &hot.url,
+        Method::PUT,
+        "/rustfs/admin/v3/tier",
+        Some(&body),
+        &hot.access_key,
+        &hot.secret_key,
+    )
+    .await?;
+    if !status.is_success() {
+        return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
+    }
+    Ok(())
+}
+
+/// A current-version `Transition Days=0` rule scoped to the object's prefix.
+fn transition_rule() -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
+    Ok(LifecycleRule::builder()
+        .id("ilm7-transition")
+        .filter(LifecycleRuleFilter::builder().prefix("tier/").build())
+        .transitions(
+            Transition::builder()
+                .days(0)
+                .storage_class(TransitionStorageClass::from(TIER_NAME))
+                .build(),
+        )
+        .status(ExpirationStatus::Enabled)
+        .build()?)
+}
+
+/// Upload `data` as a two-part multipart object with a content-type and one
+/// user-metadata entry.
+async fn put_multipart_object(client: &Client, bucket: &str, key: &str, data: &[u8]) -> TestResult {
+    let create = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .content_type(CONTENT_TYPE)
+        .metadata(USER_META_KEY, USER_META_VAL)
+        .send()
+        .await?;
+    let upload_id = create
+        .upload_id()
+        .ok_or("CreateMultipartUpload returned no upload id")?
+        .to_string();
+
+    let mut completed = Vec::new();
+    for (idx, chunk) in [&data[..PART0_SIZE], &data[PART0_SIZE..]].into_iter().enumerate() {
+        let part_number = (idx + 1) as i32;
+        let uploaded = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(&upload_id)
+            .part_number(part_number)
+            .body(ByteStream::from(chunk.to_vec()))
+            .send()
+            .await?;
+        completed.push(
+            CompletedPart::builder()
+                .part_number(part_number)
+                .e_tag(uploaded.e_tag().unwrap_or_default())
+                .build(),
+        );
+    }
+
+    client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
+        .send()
+        .await?;
+    Ok(())
+}
+
+/// Number of objects currently stored in the cold-tier bucket.
+async fn cold_tier_object_count(cold_client: &Client) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+    let resp = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
+    Ok(resp.contents().len())
+}
+
+/// Poll `HEAD` until the object's storage class is the tier name (transition
+/// complete), or fail after `deadline`.
+async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
+    let start = Instant::now();
+    loop {
+        let head = client.head_object().bucket(bucket).key(key).send().await?;
+        if head.storage_class().map(|sc| sc.as_str()) == Some(TIER_NAME) {
+            return Ok(());
+        }
+        if start.elapsed() >= deadline {
+            return Err(format!(
+                "object {bucket}/{key} was not transitioned to {TIER_NAME} within {}s (storage_class={:?})",
+                deadline.as_secs(),
+                head.storage_class()
+            )
+            .into());
+        }
+        tokio::time::sleep(StdDuration::from_millis(500)).await;
+    }
+}
+
+/// Poll until the cold-tier bucket is empty (remote free-version cleanup done),
+/// or fail after `deadline`.
+async fn wait_for_cold_tier_empty(cold_client: &Client, deadline: StdDuration) -> TestResult {
+    let start = Instant::now();
+    loop {
+        let count = cold_tier_object_count(cold_client).await?;
+        if count == 0 {
+            return Ok(());
+        }
+        if start.elapsed() >= deadline {
+            return Err(format!(
+                "cold-tier bucket still holds {count} object(s) {}s after DeleteObject; \
+                 free-version remote cleanup did not converge",
+                deadline.as_secs()
+            )
+            .into());
+        }
+        tokio::time::sleep(StdDuration::from_millis(500)).await;
+    }
+}
+
+/// GET `bytes=start-end` (inclusive) and assert it equals `data[start..=end]`.
+async fn assert_range(client: &Client, start: usize, end: usize, data: &[u8]) -> TestResult {
+    let range = format!("bytes={start}-{end}");
+    let resp = client
+        .get_object()
+        .bucket(SOURCE_BUCKET)
+        .key(OBJECT_KEY)
+        .range(&range)
+        .send()
+        .await?;
+    let got = resp.body.collect().await?.into_bytes();
+    let expected = &data[start..=end];
+    assert_eq!(got.len(), expected.len(), "range {range}: length mismatch");
+    assert_eq!(got.as_ref(), expected, "range {range}: bytes mismatch reading from the tier");
+    Ok(())
+}
+
+/// Full ilm-7 hermetic transition main path across two embedded RustFS servers.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_hermetic_transition_main_path() -> TestResult {
+    // Cold-tier server (independent credentials). It is a passive tier target,
+    // so it needs no lifecycle/scanner configuration.
+    let mut cold = RustFSTestEnvironment::new().await?;
+    cold.access_key = "coldtieradmin".to_string();
+    cold.secret_key = "coldtiersecret".to_string();
+    cold.start_rustfs_server_without_cleanup(vec![]).await?;
+    let cold_client = cold.create_s3_client();
+    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
+
+    // Hot/source server. A 1s scanner cycle is a backstop; transition is
+    // primarily driven immediately by the multipart completion path.
+    let mut hot = RustFSTestEnvironment::new().await?;
+    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1")])
+        .await?;
+    let hot_client = hot.create_s3_client();
+
+    // Wire the RustFS remote tier (real connectivity probe, no force).
+    add_rustfs_tier(&hot, &cold).await?;
+
+    // Source bucket + Days=0 transition rule installed BEFORE the write so the
+    // completion path enqueues the transition immediately.
+    hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?;
+    let lifecycle = BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?;
+    hot_client
+        .put_bucket_lifecycle_configuration()
+        .bucket(SOURCE_BUCKET)
+        .lifecycle_configuration(lifecycle)
+        .send()
+        .await?;
+
+    let data = payload();
+    put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?;
+
+    // 1) Transition completes: HEAD reports the tier name and no restore state.
+    wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?;
+    let head = hot_client.head_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
+    assert_eq!(
+        head.storage_class().map(|sc| sc.as_str()),
+        Some(TIER_NAME),
+        "transitioned object must report the tier name as its storage class"
+    );
+    assert!(
+        head.restore().is_none(),
+        "a freshly transitioned object must not advertise x-amz-restore, got {:?}",
+        head.restore()
+    );
+
+    // 2) The remote object now lives in the cold-tier bucket.
+    assert!(
+        cold_tier_object_count(&cold_client).await? >= 1,
+        "cold-tier bucket must hold the transitioned object"
+    );
+
+    // 3) GET streams identical bytes back through the warm backend; content-type
+    //    and user metadata survive the transition round trip (rustfs#2246).
+    let get = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
+    assert_eq!(get.content_type(), Some(CONTENT_TYPE), "content-type must survive transition");
+    assert_eq!(
+        get.metadata().and_then(|m| m.get(USER_META_KEY)).map(String::as_str),
+        Some(USER_META_VAL),
+        "user metadata must survive transition"
+    );
+    let body = get.body.collect().await?.into_bytes();
+    assert_eq!(body.len(), data.len(), "full transitioned GET length mismatch");
+    assert_eq!(body.as_ref(), data.as_slice(), "full transitioned GET must be byte-identical");
+
+    // 4) Range GET within a single part and across the part boundary
+    //    (backlog#807): both must read the correct bytes from the tier.
+    assert_range(&hot_client, 1000, 1099, &data).await?;
+    assert_range(&hot_client, PART0_SIZE - 5, PART0_SIZE + 4, &data).await?;
+
+    // 5) DeleteObject drives free-version cleanup. The local object is gone
+    //    immediately; the remote copy is removed asynchronously.
+    hot_client
+        .delete_object()
+        .bucket(SOURCE_BUCKET)
+        .key(OBJECT_KEY)
+        .send()
+        .await?;
+
+    let get_after = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await;
+    let err = get_after.expect_err("hot object must be gone immediately after delete");
+    assert_eq!(
+        err.as_service_error().and_then(|e| e.code()),
+        Some("NoSuchKey"),
+        "hot GET after delete must be NoSuchKey, got {err:?}"
+    );
+
+    wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?;
+
+    Ok(())
+}
@@ -0,0 +1,306 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Live-listener TLS certificate hot-reload e2e (backlog#1154 peri-5).
+//!
+//! `crates/tls-runtime` had ~20 unit tests but no test ever rotated a
+//! certificate under a listening HTTPS server, so the operational promise
+//! "swap certificates without a restart" (RUSTFS_TLS_RELOAD_ENABLE) had no
+//! automated proof. This suite pins, against a real binary over real TLS
+//! handshakes:
+//!
+//!   * the server starts with certificate A and serves its fingerprint;
+//!   * after the on-disk material is replaced with certificate B, new
+//!     connections receive B within the reload interval — no restart;
+//!   * a connection established under A keeps working across the swap
+//!     (existing sessions are not torn down);
+//!   * replacing the material with garbage does not take down the listener or
+//!     the previously loaded certificate (fail-safe, no fail-open) and leaves
+//!     an assertable `tls_reload_failed` log event.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use rcgen::generate_simple_self_signed;
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, ClientConnection, DigitallySignedStruct, Error as RustlsError, SignatureScheme, StreamOwned};
+use serial_test::serial;
+use sha2::{Digest, Sha256};
+use std::error::Error;
+use std::io::{Read, Write};
+use std::net::TcpStream;
+use std::sync::Arc;
+use tokio::time::{Duration, sleep};
+
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+type BoxError = Box<dyn Error + Send + Sync>;
+
+const CERT_FILE: &str = "rustfs_cert.pem";
+const KEY_FILE: &str = "rustfs_key.pem";
+/// Minimum value the server accepts for RUSTFS_TLS_RELOAD_INTERVAL (seconds).
+const RELOAD_INTERVAL_SECS: u64 = 5;
+
+#[derive(Debug)]
+struct AcceptAnyServerCertVerifier;
+
+impl ServerCertVerifier for AcceptAnyServerCertVerifier {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &CertificateDer<'_>,
+        _intermediates: &[CertificateDer<'_>],
+        _server_name: &ServerName<'_>,
+        _ocsp_response: &[u8],
+        _now: UnixTime,
+    ) -> Result<ServerCertVerified, RustlsError> {
+        Ok(ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+        rustls::crypto::aws_lc_rs::default_provider()
+            .signature_verification_algorithms
+            .supported_schemes()
+    }
+}
+
+fn tls_client_config() -> Arc<ClientConfig> {
+    Arc::new(
+        ClientConfig::builder()
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
+            .with_no_client_auth(),
+    )
+}
+
+/// A live TLS session over a blocking TCP stream, kept open across reloads.
+struct TlsSession {
+    stream: StreamOwned<ClientConnection, TcpStream>,
+    fingerprint: String,
+    host: String,
+}
+
+/// Completes a TLS handshake with `addr` and returns the session together with
+/// the SHA-256 fingerprint of the served leaf certificate.
+fn tls_connect(addr: &str) -> Result<TlsSession, BoxError> {
+    let host = addr.split(':').next().unwrap_or("127.0.0.1").to_string();
+    let server_name = ServerName::try_from(host.clone())?;
+    let mut conn = ClientConnection::new(tls_client_config(), server_name)?;
+    let mut tcp = TcpStream::connect(addr)?;
+    tcp.set_read_timeout(Some(std::time::Duration::from_secs(10)))?;
+    tcp.set_write_timeout(Some(std::time::Duration::from_secs(10)))?;
+
+    // Drive the handshake to completion so peer_certificates() is populated.
+    while conn.is_handshaking() {
+        conn.complete_io(&mut tcp)?;
+    }
+
+    let leaf = conn
+        .peer_certificates()
+        .and_then(|certs| certs.first())
+        .ok_or("server presented no certificate")?;
+    let fingerprint = Sha256::digest(leaf.as_ref())
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect::<String>();
+
+    Ok(TlsSession {
+        stream: StreamOwned::new(conn, tcp),
+        fingerprint,
+        host,
+    })
+}
+
+/// Sends a keep-alive HTTP request on the session and reads the response head,
+/// proving the TLS session is still fully functional.
+fn http_roundtrip(session: &mut TlsSession) -> Result<String, BoxError> {
+    let request = format!("GET / HTTP/1.1\r\nHost: {}\r\nConnection: keep-alive\r\n\r\n", session.host);
+    session.stream.write_all(request.as_bytes())?;
+    session.stream.flush()?;
+
+    // Read up to the end of the headers plus a body chunk; anonymous ListBuckets
+    // answers a small error payload, which is fine — we only need a valid
+    // HTTP status line over the existing TLS session.
+    let mut buf = vec![0_u8; 8192];
+    let read = session.stream.read(&mut buf)?;
+    if read == 0 {
+        return Err("connection closed by server".into());
+    }
+    let head = String::from_utf8_lossy(&buf[..read]).to_string();
+    if !head.starts_with("HTTP/1.1") {
+        return Err(format!("unexpected response head: {head}").into());
+    }
+    Ok(head)
+}
+
+async fn write_cert_pair(tls_dir: &std::path::Path, sans: Vec<String>) -> Result<(), BoxError> {
+    let cert = generate_simple_self_signed(sans)?;
+    tokio::fs::write(tls_dir.join(CERT_FILE), cert.cert.pem()).await?;
+    tokio::fs::write(tls_dir.join(KEY_FILE), cert.signing_key.serialize_pem()).await?;
+    Ok(())
+}
+
+/// Polls new TLS connections until the served fingerprint changes away from
+/// `old_fingerprint`, returning the new one.
+async fn wait_for_new_fingerprint(addr: &str, old_fingerprint: &str, within: Duration) -> Result<String, BoxError> {
+    let deadline = tokio::time::Instant::now() + within;
+    loop {
+        let addr_owned = addr.to_string();
+        let observed = tokio::task::spawn_blocking(move || tls_connect(&addr_owned).map(|s| s.fingerprint)).await?;
+        if let Ok(fp) = observed
+            && fp != old_fingerprint
+        {
+            return Ok(fp);
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("served certificate never rotated away from {old_fingerprint} within {within:?}").into());
+        }
+        sleep(Duration::from_secs(1)).await;
+    }
+}
+
+/// Starts the rustfs binary with HTTPS + hot reload enabled, capturing its
+/// stdout/stderr to `log_path`. The harness's own start path is unusable here:
+/// its readiness probe drives the AWS SDK over the environment URL, and the SDK
+/// rejects the self-signed test certificate.
+async fn start_https_server_with_reload(
+    env: &mut RustFSTestEnvironment,
+    tls_dir: &std::path::Path,
+    log_path: &str,
+) -> TestResult {
+    let log_file = std::fs::OpenOptions::new().create(true).append(true).open(log_path)?;
+    let log_file_err = log_file.try_clone()?;
+    let process = std::process::Command::new(crate::common::rustfs_binary_path())
+        .env("RUST_LOG", "rustfs=info")
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_TLS_RELOAD_ENABLE", "true")
+        .env("RUSTFS_TLS_RELOAD_INTERVAL", RELOAD_INTERVAL_SECS.to_string())
+        .stdout(std::process::Stdio::from(log_file))
+        .stderr(std::process::Stdio::from(log_file_err))
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+    env.process = Some(process);
+
+    // Readiness = a TLS handshake completes on the listener.
+    let addr = env.address.clone();
+    for attempt in 0..60 {
+        let addr_clone = addr.clone();
+        if tokio::task::spawn_blocking(move || tls_connect(&addr_clone)).await?.is_ok() {
+            return Ok(());
+        }
+        if attempt == 59 {
+            return Err("HTTPS server never completed a TLS handshake within 60s".into());
+        }
+        sleep(Duration::from_secs(1)).await;
+    }
+    Ok(())
+}
+
+/// Runs `http_roundtrip` on a session inside the blocking pool, handing the
+/// session back for later reuse (the point: the same TLS session survives).
+async fn roundtrip_and_return(mut session: TlsSession) -> Result<TlsSession, BoxError> {
+    tokio::task::spawn_blocking(move || {
+        http_roundtrip(&mut session)?;
+        Ok::<_, BoxError>(session)
+    })
+    .await?
+}
+
+#[tokio::test]
+#[serial]
+async fn test_tls_certificate_hot_reload_live_listener() -> TestResult {
+    init_logging();
+    // Install the process-wide rustls crypto provider (idempotent).
+    let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(format!("{}/tls", env.temp_dir));
+    tokio::fs::create_dir_all(&tls_dir).await?;
+    write_cert_pair(&tls_dir, vec!["localhost".into(), "127.0.0.1".into()]).await?;
+
+    let log_path = format!("{}/server.log", env.temp_dir);
+    start_https_server_with_reload(&mut env, &tls_dir, &log_path).await?;
+
+    let addr = env.address.clone();
+
+    // --- certificate A is served, and the session stays usable ---------------
+    let session_a = {
+        let addr = addr.clone();
+        tokio::task::spawn_blocking(move || tls_connect(&addr)).await??
+    };
+    let fingerprint_a = session_a.fingerprint.clone();
+    let session_a = roundtrip_and_return(session_a).await?;
+
+    // --- swap to certificate B: new connections pick it up without restart ---
+    write_cert_pair(&tls_dir, vec!["localhost".into(), "127.0.0.1".into()]).await?;
+    let reload_budget = Duration::from_secs(RELOAD_INTERVAL_SECS * 4 + 10);
+    let fingerprint_b = wait_for_new_fingerprint(&addr, &fingerprint_a, reload_budget).await?;
+    assert_ne!(fingerprint_a, fingerprint_b);
+
+    // The connection opened under certificate A must survive the rotation.
+    let _session_a = roundtrip_and_return(session_a).await?;
+
+    // --- garbage material: fail-safe, keep serving B, log the failure --------
+    tokio::fs::write(tls_dir.join(CERT_FILE), b"not a certificate").await?;
+    tokio::fs::write(tls_dir.join(KEY_FILE), b"not a key").await?;
+
+    // Give the reload loop at least two ticks to observe the bad material.
+    sleep(Duration::from_secs(RELOAD_INTERVAL_SECS * 2 + 2)).await;
+
+    let after_bad = {
+        let addr = addr.clone();
+        tokio::task::spawn_blocking(move || tls_connect(&addr)).await??
+    };
+    assert_eq!(
+        after_bad.fingerprint, fingerprint_b,
+        "bad on-disk material must not change or drop the served certificate"
+    );
+
+    let log = tokio::fs::read_to_string(&log_path).await.unwrap_or_default();
+    assert!(
+        log.contains("tls_reload_failed") || log.contains("TLS reload failed"),
+        "a failed reload must leave an assertable log event; captured log did not contain one"
+    );
+
+    // The process must still be alive and serving.
+    let final_session = tokio::task::spawn_blocking(move || tls_connect(&addr)).await??;
+    let _ = roundtrip_and_return(final_session).await?;
+
+    env.stop_server();
+    Ok(())
+}
@@ -49,7 +49,7 @@ rustfs-signer.workspace = true
 rustfs-storage-api.workspace = true
 rustfs-tls-runtime.workspace = true
 rustfs-checksums.workspace = true
-rustfs-config = { workspace = true, features = ["constants", "notify", "audit", "server-config-model"] }
+rustfs-config = { workspace = true, features = ["notify", "audit", "server-config-model"] }
 rustfs-concurrency.workspace = true
 rustfs-credentials = { workspace = true }
 rustfs-common.workspace = true
@@ -62,9 +62,9 @@ rustfs-s3-types = { workspace = true }
 rustfs-data-usage.workspace = true
 rustfs-object-capacity.workspace = true
 async-trait.workspace = true
-bytes.workspace = true
+bytes = { workspace = true, features = ["serde"] }
 byteorder = { workspace = true }
-chrono.workspace = true
+chrono = { workspace = true, features = ["serde"] }
 glob = { workspace = true }
 thiserror.workspace = true
 flatbuffers.workspace = true
@@ -72,22 +72,22 @@ futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
 tracing-opentelemetry.workspace = true
-serde.workspace = true
-time.workspace = true
+serde = { workspace = true, features = ["derive"] }
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
 bytesize.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
-s3s.workspace = true
+s3s = { workspace = true, features = ["minio"] }
 http.workspace = true
 opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
-uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
-reed-solomon-erasure = { workspace = true }
+uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
+reed-solomon-erasure = { workspace = true, features = ["simd-accel"] }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
-moka = { workspace = true }
+moka = { workspace = true, features = ["future"] }
 rustfs-lock.workspace = true
 rustfs-io-metrics.workspace = true
 regex = { workspace = true }
@@ -101,38 +101,38 @@ sha1 = { workspace = true }
 sha2 = { workspace = true }
 hex-simd = { workspace = true }
 tempfile.workspace = true
-hyper.workspace = true
-hyper-util.workspace = true
-hyper-rustls.workspace = true
-rustls.workspace = true
+hyper = { workspace = true, features = ["http2", "http1", "server"] }
+hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
+rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
-tonic.workspace = true
+tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
+tonic = { workspace = true, features = ["gzip", "deflate"] }
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
-tower.workspace = true
+tower = { workspace = true, features = ["timeout"] }
 async-channel.workspace = true
 enumset = { workspace = true }
 num_cpus = { workspace = true }
-rand.workspace = true
+rand = { workspace = true, features = ["serde"] }
 pin-project-lite.workspace = true
 md-5.workspace = true
 memmap2 = { workspace = true }
 libc.workspace = true
 # "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate
 # (backlog#1178); "fs" comes from the workspace default.
-rustix = { workspace = true, features = ["process"] }
+rustix = { workspace = true, features = ["process", "fs"] }
 rustfs-madmin.workspace = true
-reqwest = { workspace = true }
-aes-gcm.workspace = true
+reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
+aes-gcm = { workspace = true, features = ["rand_core"] }
 chacha20poly1305.workspace = true
-aws-sdk-s3 = { workspace = true }
+aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }
-smallvec = { workspace = true }
-shadow-rs.workspace = true
+smallvec = { workspace = true, features = ["serde"] }
+shadow-rs = { workspace = true, default-features = false }
 async-recursion.workspace = true
 aws-credential-types = { workspace = true }
 aws-smithy-types = { workspace = true }
-aws-smithy-runtime-api = { workspace = true }
+aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
@@ -141,7 +141,7 @@ google-cloud-auth = { workspace = true }
 aws-config = { workspace = true }
 faster-hex = { workspace = true }
 ratelimit = { workspace = true }
-aws-smithy-http-client.workspace = true
+aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }

 # Observability and Metrics
 metrics = { workspace = true }
@@ -153,19 +153,19 @@ metrics = { workspace = true }
 rustfs-uring = "0.2.1"

 [dev-dependencies]
-tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
 criterion = { workspace = true, features = ["html_reports"] }
 temp-env = { workspace = true, features = ["async_closure"] }
-tracing-subscriber = { workspace = true, features = ["json"] }
+tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
 serial_test = { workspace = true }
-opentelemetry_sdk = { workspace = true }
+opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 proptest = "1"
 rcgen.workspace = true
-insta = { workspace = true }
+insta = { workspace = true, features = ["yaml", "json"] }
 rustfs-crypto = { workspace = true }

 [build-dependencies]
-shadow-rs = { workspace = true, features = ["build", "metadata"] }
+shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] }

 [[bench]]
 name = "erasure_benchmark"
@@ -1023,20 +1023,49 @@ fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
    http_client_fn(move |_settings, _components| connector.clone())
 }

-fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
-    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem.as_bytes())
+fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
+    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
        .collect::<Result<Vec<_>, _>>()
-        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))?;
+        .map_err(|err| format!("invalid PEM encoding: {err}"))?;

    if certs.is_empty() {
-        return Err(BucketTargetError::Io(std::io::Error::other(
-            "invalid target CA PEM: no certificates found",
-        )));
+        return Err("no certificates found".to_string());
    }

-    let mut trust_store = smithy_tls::TrustStore::empty();
-    trust_store.add_pem_certificate(ca_cert_pem.as_bytes());
+    // Smithy's rustls adapter defers parsing custom certificates and assumes
+    // they are valid when the HTTPS connector is built. Validate every DER
+    // certificate first so malformed configuration is reported rather than
+    // reaching an `expect` in the dependency.
+    let mut validation_store = rustls::RootCertStore::empty();
+    for cert in certs {
+        validation_store
+            .add(cert)
+            .map_err(|err| format!("invalid X.509 certificate: {err}"))?;
+    }

+    Ok(())
+}
+
+fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), BucketTargetError> {
+    validate_ca_pem_bundle(ca_cert_pem.as_bytes())
+        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))
+}
+
+fn compose_replication_trust_store(certificate_bundles: impl IntoIterator<Item = Vec<u8>>) -> (smithy_tls::TrustStore, usize) {
+    // `TrustStore::default()` keeps the platform-native roots enabled. Target
+    // and RUSTFS_TLS_PATH certificates extend that baseline instead of
+    // replacing it with a target-specific trust island.
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut custom_bundle_count = 0;
+    for pem in certificate_bundles {
+        trust_store.add_pem_certificate(pem);
+        custom_bundle_count += 1;
+    }
+
+    (trust_store, custom_bundle_count)
+}
+
+fn build_aws_s3_http_client_with_trust_store(trust_store: smithy_tls::TrustStore) -> Result<SharedHttpClient, BucketTargetError> {
    let tls_context = smithy_tls::TlsContext::builder()
        .with_trust_store(trust_store)
        .build()
@@ -1048,6 +1077,60 @@ fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<Shar
        .build_https())
 }

+async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
+    let mut certificate_bundles = Vec::new();
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => match validate_ca_pem_bundle(&pem) {
+            Ok(()) => certificate_bundles.push(pem),
+            Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
+        },
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if trust_leaf_cert_as_ca {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => match validate_ca_pem_bundle(&pem) {
+                Ok(()) => certificate_bundles.push(pem),
+                Err(err) => warn!(
+                    "ignoring invalid leaf certificate {:?} for replication client trust store: {}",
+                    leaf_cert_path, err
+                ),
+            },
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    certificate_bundles
+}
+
+async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
+    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
+    if tls_path.is_empty() {
+        return Vec::new();
+    }
+
+    load_tls_path_ca_bundles(
+        Path::new(&tls_path),
+        rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
+    )
+    .await
+}
+
+async fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
+    validate_target_ca_pem(ca_cert_pem)?;
+
+    let mut certificate_bundles = load_configured_tls_ca_bundles().await;
+    certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
+    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
+
+    build_aws_s3_http_client_with_trust_store(trust_store)
+}
+
 async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Option<SharedHttpClient>, BucketTargetError> {
    if !target.secure {
        return Ok(None);
@@ -1058,62 +1141,28 @@ async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Op
    }

    if has_custom_ca_pem(target) {
-        return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem).map(Some);
+        return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem)
+            .await
+            .map(Some);
    }

    Ok(build_aws_s3_http_client_from_tls_path().await)
 }

 async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
-    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
-    if tls_path.is_empty() {
+    let certificate_bundles = load_configured_tls_ca_bundles().await;
+    if certificate_bundles.is_empty() {
        return None;
    }

-    let tls_dir = Path::new(&tls_path);
-    let mut trust_store = smithy_tls::TrustStore::empty();
-    let mut has_custom_certs = false;
-
-    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
-    match tokio::fs::read(&ca_path).await {
-        Ok(pem) => {
-            trust_store.add_pem_certificate(pem);
-            has_custom_certs = true;
-        }
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
-    }
-
-    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
-        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
-        match tokio::fs::read(&leaf_cert_path).await {
-            Ok(pem) => {
-                trust_store.add_pem_certificate(pem);
-                has_custom_certs = true;
-            }
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
-        }
-    }
-
-    if !has_custom_certs {
-        return None;
-    }
-
-    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
-        Ok(ctx) => ctx,
+    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
+    match build_aws_s3_http_client_with_trust_store(trust_store) {
+        Ok(client) => Some(client),
        Err(e) => {
            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
-            return None;
+            None
        }
-    };
-
-    Some(
-        SmithyHttpClientBuilder::new()
-            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
-            .tls_context(tls_context)
-            .build_https(),
-    )
+    }
 }

 fn should_force_path_style(target: &BucketTarget) -> bool {
@@ -1920,6 +1969,63 @@ mod tests {
    use super::*;
    use rcgen::generate_simple_self_signed;

+    fn spawn_single_request_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>) -> (u16, std::thread::JoinHandle<()>) {
+        use std::io::{Read, Write};
+
+        ensure_rustls_crypto_provider();
+        let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test TLS listener should bind");
+        let port = listener
+            .local_addr()
+            .expect("test TLS listener should have an address")
+            .port();
+        let server_config = rustls::ServerConfig::builder()
+            .with_no_client_auth()
+            .with_single_cert(
+                vec![cert.cert.der().clone()],
+                rustls_pki_types::PrivateKeyDer::try_from(cert.signing_key.serialize_der())
+                    .expect("test TLS private key should convert"),
+            )
+            .expect("test TLS server config should build");
+
+        let handle = std::thread::spawn(move || {
+            let (stream, _) = listener.accept().expect("test TLS client should connect");
+            stream
+                .set_read_timeout(Some(Duration::from_secs(10)))
+                .expect("test TLS read timeout should configure");
+            stream
+                .set_write_timeout(Some(Duration::from_secs(10)))
+                .expect("test TLS write timeout should configure");
+            let connection = rustls::ServerConnection::new(Arc::new(server_config)).expect("test TLS connection should build");
+            let mut stream = rustls::StreamOwned::new(connection, stream);
+            let mut request = [0_u8; 8192];
+            let _ = stream.read(&mut request).expect("test TLS request should be readable");
+            stream
+                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("test TLS response should be written");
+            stream.flush().expect("test TLS response should flush");
+        });
+
+        (port, handle)
+    }
+
+    fn s3_client_with_http_client(port: u16, http_client: SharedHttpClient) -> S3Client {
+        let credentials = SdkCredentials::builder()
+            .access_key_id("test-access")
+            .secret_access_key("test-secret")
+            .provider_name("bucket_target_tls_test")
+            .build();
+        let config = S3Config::builder()
+            .endpoint_url(format!("https://localhost:{port}"))
+            .credentials_provider(SharedCredentialsProvider::new(credentials))
+            .region(SdkRegion::new("us-east-1"))
+            .force_path_style(true)
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+            .http_client(http_client)
+            .build();
+
+        S3Client::from_conf(config)
+    }
+
    #[test]
    fn replication_target_versioning_enabled_requires_enabled_status() {
        let enabled = BucketVersioningStatus::Enabled;
@@ -2243,6 +2349,78 @@ mod tests {
        assert_eq!(client.endpoint, "https://192.168.1.10:9000");
    }

+    #[tokio::test]
+    async fn replication_trust_store_composes_system_global_and_target_roots_for_real_tls() {
+        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
+        let global_ca =
+            generate_simple_self_signed(vec!["localhost".to_string()]).expect("global CA certificate should generate");
+        let target_ca =
+            generate_simple_self_signed(vec!["localhost".to_string()]).expect("target CA certificate should generate");
+
+        tokio::fs::write(tls_dir.path().join(RUSTFS_CA_CERT), global_ca.cert.pem())
+            .await
+            .expect("global CA bundle should be written");
+
+        let mut certificate_bundles = load_tls_path_ca_bundles(tls_dir.path(), false).await;
+        certificate_bundles.push(target_ca.cert.pem().into_bytes());
+        let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
+        assert!(
+            format!("{trust_store:?}").contains("enable_native_roots: true"),
+            "per-target trust must retain the SDK's platform-native roots"
+        );
+        let http_client = build_aws_s3_http_client_with_trust_store(trust_store).expect("composed TLS client should build");
+
+        let (global_port, global_server) = spawn_single_request_https_server(&global_ca);
+        s3_client_with_http_client(global_port, http_client.clone())
+            .head_bucket()
+            .bucket("test-bucket")
+            .send()
+            .await
+            .expect("global RUSTFS_TLS_PATH CA should authenticate its TLS server");
+        global_server.join().expect("global CA TLS server should finish");
+
+        let (target_port, target_server) = spawn_single_request_https_server(&target_ca);
+        s3_client_with_http_client(target_port, http_client)
+            .head_bucket()
+            .bucket("test-bucket")
+            .send()
+            .await
+            .expect("per-target CA should authenticate its TLS server alongside global roots");
+        target_server.join().expect("target CA TLS server should finish");
+    }
+
+    #[tokio::test]
+    async fn tls_path_leaf_trust_remains_opt_in() {
+        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
+        let global_ca =
+            generate_simple_self_signed(vec!["global-ca.example".to_string()]).expect("global CA certificate should generate");
+        let trusted_leaf =
+            generate_simple_self_signed(vec!["leaf.example".to_string()]).expect("trusted leaf certificate should generate");
+        tokio::fs::write(tls_dir.path().join(RUSTFS_CA_CERT), global_ca.cert.pem())
+            .await
+            .expect("global CA bundle should be written");
+        tokio::fs::write(tls_dir.path().join(RUSTFS_TLS_CERT), trusted_leaf.cert.pem())
+            .await
+            .expect("trusted leaf certificate should be written");
+
+        assert_eq!(load_tls_path_ca_bundles(tls_dir.path(), true).await.len(), 2);
+        assert_eq!(load_tls_path_ca_bundles(tls_dir.path(), false).await.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn skip_tls_verify_takes_priority_over_invalid_custom_ca_pem() {
+        let client = build_aws_s3_http_client_for_target(&BucketTarget {
+            secure: true,
+            skip_tls_verify: true,
+            ca_cert_pem: "not a pem".to_string(),
+            ..Default::default()
+        })
+        .await
+        .expect("skip verification should bypass custom CA parsing");
+
+        assert!(client.is_some(), "secure targets with skip verification need a custom HTTP client");
+    }
+
    #[tokio::test]
    async fn get_remote_target_client_internal_rejects_invalid_custom_ca_pem() {
        let sys = BucketTargetSys::default();
@@ -2267,6 +2445,31 @@ mod tests {
        assert!(err.to_string().contains("invalid target CA PEM"));
    }

+    #[test]
+    fn target_ca_rejects_pem_wrapped_invalid_der_before_smithy_builds() {
+        let err = validate_target_ca_pem("-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n")
+            .expect_err("PEM-wrapped invalid DER must be rejected");
+
+        assert!(err.to_string().contains("invalid target CA PEM"));
+        assert!(err.to_string().contains("invalid X.509 certificate"));
+    }
+
+    #[tokio::test]
+    async fn invalid_global_ca_is_ignored_without_reaching_smithy() {
+        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
+        tokio::fs::write(
+            tls_dir.path().join(RUSTFS_CA_CERT),
+            b"-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n",
+        )
+        .await
+        .expect("invalid global CA fixture should be written");
+
+        assert!(
+            load_tls_path_ca_bundles(tls_dir.path(), false).await.is_empty(),
+            "invalid global CA must fall back to default roots instead of reaching Smithy's panic path"
+        );
+    }
+
    // backlog#806-16 regression tests for the rolling one-minute latency window.

    #[test]
@@ -836,6 +836,12 @@ pub struct TransitionState {
    compensation_scheduled_tasks: AtomicI64,
    compensation_running_tasks: AtomicI64,
    compensation_buckets: Arc<Mutex<HashSet<String>>>,
+    // (bucket, object, version) currently queued or being transitioned. A single
+    // PUT can enqueue the same object twice (immediate transition + startup
+    // compensation backfill); without this guard the duplicates run concurrently,
+    // and the loser races the winner's source cleanup — reading data the winner
+    // already removed and logging a spurious NotFound failure (rustfs/backlog#1268).
+    in_flight_transitions: Arc<Mutex<HashSet<(String, String, String)>>>,
    last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
 }

@@ -869,10 +875,39 @@ impl TransitionState {
            compensation_scheduled_tasks: AtomicI64::new(0),
            compensation_running_tasks: AtomicI64::new(0),
            compensation_buckets: Arc::new(Mutex::new(HashSet::new())),
+            in_flight_transitions: Arc::new(Mutex::new(HashSet::new())),
            last_day_stats: Arc::new(Mutex::new(HashMap::new())),
        })
    }

+    fn transition_key(oi: &ObjectInfo) -> (String, String, String) {
+        (
+            oi.bucket.clone(),
+            oi.name.clone(),
+            oi.version_id.map(|v| v.to_string()).unwrap_or_default(),
+        )
+    }
+
+    /// Try to claim a transition for this exact (bucket, object, version). Returns
+    /// `false` when one is already queued or running, so the caller can skip the
+    /// duplicate enqueue (rustfs/backlog#1268). The claim is released in the
+    /// worker once the transition finishes, or here if the enqueue fails.
+    fn reserve_transition(&self, oi: &ObjectInfo) -> bool {
+        let key = Self::transition_key(oi);
+        match self.in_flight_transitions.lock() {
+            Ok(mut set) => set.insert(key),
+            Err(poisoned) => poisoned.into_inner().insert(key),
+        }
+    }
+
+    fn release_transition(&self, oi: &ObjectInfo) {
+        let key = Self::transition_key(oi);
+        match self.in_flight_transitions.lock() {
+            Ok(mut set) => set.remove(&key),
+            Err(poisoned) => poisoned.into_inner().remove(&key),
+        };
+    }
+
    fn reserve_bucket_compensation(&self, bucket: &str) -> bool {
        let inserted = match self.compensation_buckets.lock() {
            Ok(mut scheduled) => scheduled.insert(bucket.to_string()),
@@ -1046,6 +1081,15 @@ impl TransitionState {
            return false;
        }

+        // Deduplicate concurrent enqueues of the same object version. The claim is
+        // released by the worker after the transition finishes (rustfs/backlog#1268).
+        // This is a no-op, not a new enqueue, so it must not touch the enqueue
+        // counters (the first enqueue already counted this object).
+        if !self.reserve_transition(oi) {
+            self.record_scanner_transition_state();
+            return true;
+        }
+
        let task = TransitionTask {
            obj_info: oi.clone(),
            src: src.clone(),
@@ -1084,6 +1128,9 @@ impl TransitionState {
                    self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::QueueClosed { timeout_ms: None });
                }
            }
+            if !queued {
+                self.release_transition(oi);
+            }
            record_scanner_transition_enqueue_result(src, 1, queued);
            self.record_scanner_transition_state();
            return queued;
@@ -1113,6 +1160,9 @@ impl TransitionState {
                );
            }
        }
+        if !queued {
+            self.release_transition(oi);
+        }
        record_scanner_transition_enqueue_result(src, 1, queued);
        self.record_scanner_transition_state();
        queued
@@ -1239,6 +1289,9 @@ impl TransitionState {

                            emit_transition_complete_event(obj_info_for_event);
                        }
+                        // Release the dedup claim so a later lifecycle pass can
+                        // re-transition this object if needed (rustfs/backlog#1268).
+                        transition_state.release_transition(&task.obj_info);
                        TransitionState::add_counter(&transition_state.active_tasks, -1);
                        transition_state.record_scanner_transition_state();
                    }
@@ -3244,8 +3297,16 @@ mod tests {
            ..Default::default()
        };

+        // A distinct object fills past the capacity-1 queue and is reported as a
+        // missed enqueue. (Re-enqueuing the same object would instead be deduped;
+        // see transition_reserve_dedupes_same_object_version.)
+        let other = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object-2".to_string(),
+            ..Default::default()
+        };
        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
+        let second = state.queue_transition_task(&other, &event, &LcEventSrc::Scanner).await;

        assert!(first);
        assert!(!second);
@@ -3267,8 +3328,13 @@ mod tests {
            ..Default::default()
        };

+        let other = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object-2".to_string(),
+            ..Default::default()
+        };
        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
+        let second = state.queue_transition_task(&other, &event, &LcEventSrc::Scanner).await;

        assert!(first);
        assert!(!second);
@@ -3769,6 +3835,62 @@ mod tests {
        assert_eq!(state.scanner_transition_state_update().compensation_pending, 1);
    }

+    #[test]
+    fn transition_reserve_dedupes_same_object_version() {
+        // Regression for rustfs/backlog#1268: a single object version can be
+        // enqueued twice (immediate transition + compensation backfill). The
+        // second claim must be rejected until the first is released, so the two
+        // do not run concurrently and race the source cleanup.
+        let state = TransitionState::new_with_capacity(4);
+
+        let oi = ObjectInfo {
+            bucket: "foo".to_string(),
+            name: "payload.bin".to_string(),
+            version_id: None,
+            ..Default::default()
+        };
+
+        assert!(state.reserve_transition(&oi), "first claim must succeed");
+        assert!(!state.reserve_transition(&oi), "duplicate claim must be rejected");
+
+        // A different object is independent.
+        let other = ObjectInfo {
+            bucket: "foo".to_string(),
+            name: "other.bin".to_string(),
+            version_id: None,
+            ..Default::default()
+        };
+        assert!(state.reserve_transition(&other), "distinct object must claim independently");
+
+        // After release, the same object can be re-claimed (later lifecycle pass).
+        state.release_transition(&oi);
+        assert!(state.reserve_transition(&oi), "re-claim after release must succeed");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn queue_transition_task_dedupes_same_object_without_second_enqueue() {
+        // Capacity 4 leaves room, so a rejected second enqueue can only be the
+        // dedup guard, not queue pressure (rustfs/backlog#1268).
+        let state = TransitionState::new_with_capacity(4);
+        let object = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            ..Default::default()
+        };
+        let event = crate::bucket::lifecycle::lifecycle::Event {
+            action: IlmAction::TransitionAction,
+            ..Default::default()
+        };
+
+        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
+        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
+
+        assert!(first, "first enqueue must succeed");
+        assert!(second, "duplicate enqueue is reported handled (already queued)");
+        assert_eq!(state.transition_rx.len(), 1, "only one task must actually be queued");
+    }
+
    #[tokio::test]
    #[serial]
    async fn transition_state_init_honors_runtime_configured_worker_count() {
@@ -260,7 +260,9 @@ pub(crate) fn replication_put_object_header_size(put_options: &PutObjectOptions)

 fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObject<'_> {
    ReplicationSourceObject {
-        mod_time: object_info.mod_time,
+        mod_time: object_info
+            .mod_time
+            .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
        version_id: object_info.version_id.map(|version_id| version_id.to_string()),
        etag: object_info.etag.as_deref(),
        actual_size: object_info.get_actual_size().unwrap_or_default(),
@@ -463,6 +465,35 @@ mod tests {
        );
    }

+    #[test]
+    fn replication_action_for_target_head_compares_http_date_precision() {
+        for (source_nanos, target_secs, expected) in [
+            (10_123_456_789, 10, ReplicationAction::None),
+            (-10_876_543_211, -11, ReplicationAction::None),
+            (10_600_000_000, 11, ReplicationAction::All),
+        ] {
+            let mod_time = OffsetDateTime::from_unix_timestamp_nanos(source_nanos).expect("valid timestamp");
+            let object_info = ObjectInfo {
+                mod_time: Some(mod_time),
+                version_id: Some(Uuid::new_v4()),
+                etag: Some("abc123".to_string()),
+                size: 10,
+                ..Default::default()
+            };
+            let target = HeadObjectOutput::builder()
+                .last_modified(DateTime::from_secs(target_secs))
+                .version_id(object_info.version_id.expect("version ID").to_string())
+                .e_tag("abc123")
+                .content_length(10)
+                .build();
+
+            assert_eq!(
+                replication_action_for_target_head(&object_info, &target, ReplicationType::Object),
+                expected
+            );
+        }
+    }
+
    #[test]
    fn replication_remove_options_mark_replication_requests() {
        let mtime = OffsetDateTime::UNIX_EPOCH + Duration::seconds(10);
@@ -585,4 +616,43 @@ mod tests {

        assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
    }
+
+    // T3 (#1264): the outbound replication path forwards a stored object checksum into
+    // user_metadata via decrypt_checksums, which is algorithm-agnostic. This locks that
+    // the AWS 2026-04 additional algorithms (XXHash3/64/128, SHA-512, MD5) are forwarded
+    // identically to the classic five — i.e. replication treats the new algorithms
+    // consistently, with no new-algorithm-specific gap on the outbound side.
+    #[test]
+    fn replication_put_object_options_forwards_new_algorithm_checksums_like_classic() {
+        use rustfs_rio::{Checksum, ChecksumType};
+
+        let payload = b"replication checksum consistency payload";
+        let cases = [
+            // classic five (baseline)
+            ("CRC32", ChecksumType::CRC32),
+            ("SHA256", ChecksumType::SHA256),
+            // AWS 2026-04 additional algorithms
+            ("XXHASH3", ChecksumType::XXHASH3),
+            ("XXHASH64", ChecksumType::XXHASH64),
+            ("XXHASH128", ChecksumType::XXHASH128),
+            ("SHA512", ChecksumType::SHA512),
+            ("MD5", ChecksumType::MD5),
+        ];
+
+        for (name, ty) in cases {
+            let checksum = Checksum::new_from_data(ty, payload).expect("compute checksum");
+            let object_info = ObjectInfo {
+                checksum: Some(checksum.to_bytes(&[])),
+                ..Default::default()
+            };
+
+            let (opts, _is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
+
+            assert_eq!(
+                opts.user_metadata.get(name),
+                Some(&checksum.encoded),
+                "replication must forward the {name} checksum into user_metadata identically to the classic algorithms"
+            );
+        }
+    }
 }
@@ -120,7 +120,11 @@ impl Default for PutObjectOptions {
            storage_class: "".to_string(),
            website_redirect_location: "".to_string(),
            part_size: 0,
-            legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
+            // Empty, not OFF: `header()` emits x-amz-object-lock-legal-hold for
+            // any non-empty status, and CompleteMultipartUpload rejects requests
+            // that carry object-lock headers, breaking multipart transitions
+            // (rustfs/rustfs#4811). Only send the header when a status is set.
+            legalhold: ObjectLockLegalHoldStatus::from_static(""),
            send_content_md5: false,
            disable_content_sha256: false,
            disable_multipart: false,
@@ -18,6 +18,7 @@
 #![allow(clippy::all)]

 use http::{HeaderMap, HeaderName, StatusCode};
+use http_body_util::BodyExt;
 use hyper::body::Bytes;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
@@ -244,7 +245,23 @@ impl TransitionClient {
            )));
        }
        //}
-        let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
+        // Parse the CreateMultipartUpload response for the UploadId. Returning a
+        // default (empty) result here made every multipart transition fail at the
+        // first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        let initiate_multipart_upload_result =
+            quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
+                .map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
+        if initiate_multipart_upload_result.upload_id.is_empty() {
+            return Err(std::io::Error::other("CreateMultipartUpload response missing UploadId"));
+        }
        Ok(initiate_multipart_upload_result)
    }

@@ -432,3 +449,56 @@ pub struct UploadPartParams {
    pub custom_header: HeaderMap,
    pub trailer: HeaderMap,
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::client::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
+
+    #[test]
+    fn complete_multipart_upload_serializes_s3_part_elements() {
+        // Regression for rustfs/rustfs#4811: without serde renames quick-xml emits
+        // <parts>/<part_num>/<etag>, so the remote parses zero <Part> elements and
+        // completes a 0-byte object (while still returning 200). The body must use
+        // S3 element names, and MD5-only transitions must not emit empty checksum
+        // elements.
+        let complete = CompleteMultipartUpload {
+            parts: vec![
+                CompletePart {
+                    part_num: 1,
+                    etag: "etag-one".to_string(),
+                    ..Default::default()
+                },
+                CompletePart {
+                    part_num: 2,
+                    etag: "etag-two".to_string(),
+                    ..Default::default()
+                },
+            ],
+        };
+        let xml = complete.marshal_msg().expect("marshal");
+        assert!(xml.contains("<Part>"), "missing <Part>: {xml}");
+        assert!(xml.contains("<PartNumber>1</PartNumber>"), "missing PartNumber: {xml}");
+        assert!(xml.contains("<ETag>etag-one</ETag>"), "missing ETag: {xml}");
+        assert!(xml.contains("<PartNumber>2</PartNumber>"), "missing part 2: {xml}");
+        assert!(!xml.contains("part_num"), "leaked rust field name: {xml}");
+        assert!(!xml.contains("<ChecksumCRC32>"), "emitted empty checksum: {xml}");
+    }
+
+    #[test]
+    fn parses_create_multipart_upload_response() {
+        // Regression for rustfs/rustfs#4811: `initiate_multipart_upload` used to
+        // return a default (empty) result, so every multipart transition failed
+        // the first UploadPart with "UploadID cannot be empty". The UploadId must
+        // be parsed out of the S3 XML response.
+        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Bucket>bar</Bucket>
+  <Key>foo/payload.bin</Key>
+  <UploadId>a1b2c3-d4e5-f6</UploadId>
+</InitiateMultipartUploadResult>"#;
+        let parsed: InitiateMultipartUploadResult = quick_xml::de::from_str(xml).expect("parse");
+        assert_eq!(parsed.upload_id, "a1b2c3-d4e5-f6");
+        assert_eq!(parsed.bucket, "bar");
+        assert_eq!(parsed.key, "foo/payload.bin");
+    }
+}
@@ -25,6 +25,7 @@ use std::io::Error;
 use std::sync::RwLock;
 use std::{collections::HashMap, sync::Arc};
 use time::{OffsetDateTime, format_description};
+use tokio::io::AsyncReadExt;
 use tokio::{select, sync::mpsc};
 use tokio_util::sync::CancellationToken;
 use tracing::warn;
@@ -45,6 +46,33 @@ use crate::client::utils::base64_encode;
 use rustfs_utils::path::trim_etag;
 use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};

+/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
+/// reaches EOF first. Advances the reader so the next call returns the following
+/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
+/// the entire source into the first part and left later parts empty
+/// (rustfs/rustfs#4811).
+async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec<u8>, std::io::Error> {
+    match reader {
+        ReaderImpl::Body(content_body) => {
+            let take = content_body.len().min(want);
+            Ok(content_body.split_to(take).to_vec())
+        }
+        ReaderImpl::ObjectBody(content_body) => {
+            let mut buf = vec![0u8; want];
+            let mut filled = 0;
+            while filled < want {
+                let n = content_body.read(&mut buf[filled..]).await?;
+                if n == 0 {
+                    break;
+                }
+                filled += n;
+            }
+            buf.truncate(filled);
+            Ok(buf)
+        }
+    }
+}
+
 pub struct UploadedPartRes {
    pub error: std::io::Error,
    pub part_num: i64,
@@ -133,7 +161,6 @@ impl TransitionClient {
        let mut total_uploaded_size: i64 = 0;

        let mut parts_info = HashMap::<i64, ObjectPart>::new();
-        let mut buf = Vec::<u8>::with_capacity(part_size as usize);

        let mut md5_base64: String = "".to_string();
        for part_number in 1..=total_parts_count {
@@ -141,14 +168,12 @@ impl TransitionClient {
                part_size = lastpart_size;
            }

-            match &mut reader {
-                ReaderImpl::Body(content_body) => {
-                    buf = content_body.to_vec();
-                }
-                ReaderImpl::ObjectBody(content_body) => {
-                    buf = content_body.read_all().await?;
-                }
-            }
+            // Read exactly this part's bytes. Using `read_all()`/`to_vec()` here
+            // drained the whole source into the first part and left every later
+            // part empty, silently corrupting any multipart upload of a streamed
+            // (`ObjectBody`) source — e.g. ILM transitions of >128 MiB objects,
+            // which split into 128 MiB parts (rustfs/rustfs#4811).
+            let buf = read_multipart_part(&mut reader, part_size as usize).await?;
            let length = buf.len();

            if opts.send_content_md5 {
@@ -159,7 +184,7 @@ impl TransitionClient {
                };
                let hash = md5_hash.hash_encode(&buf[..length]);
                md5_base64 = base64_encode(hash.as_ref());
-            } else {
+            } else if opts.auto_checksum.is_set() {
                let mut crc = opts.auto_checksum.hasher()?;
                crc.update(&buf[..length]);
                let csum = crc.finalize();
@@ -174,6 +199,10 @@ impl TransitionClient {
                    warn!("Invalid header name: {}", opts.auto_checksum.key());
                }
            }
+            // else: neither MD5 nor a concrete additional checksum was requested,
+            // so upload the part without a per-part checksum header. Guarding the
+            // branch on `is_set()` avoids calling `hasher()` on `ChecksumNone`,
+            // which errors with "unsupported checksum type" (rustfs/rustfs#4811).

            let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
            let mut p = UploadPartParams {
@@ -183,7 +212,9 @@ impl TransitionClient {
                reader: hooked,
                part_number,
                md5_base64: md5_base64.clone(),
-                size: part_size,
+                // Use the bytes actually read, not the planned part_size, so the
+                // uploaded Content-Length matches the body even on a short read.
+                size: length as i64,
                //sse: opts.server_side_encryption,
                stream_sha256: !opts.disable_content_sha256,
                custom_header: custom_header.clone(),
@@ -194,7 +225,7 @@ impl TransitionClient {

            parts_info.entry(part_number).or_insert(obj_part);

-            total_uploaded_size += part_size as i64;
+            total_uploaded_size += length as i64;
        }

        if size > 0 && total_uploaded_size != size {
@@ -593,9 +624,79 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou

 #[cfg(test)]
 mod tests {
-    use super::{ObjectPart, collect_complete_parts};
+    use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
+    use crate::object_api::GetObjectReader;
+    use bytes::Bytes;
    use std::collections::HashMap;

+    // Drive a reader through the same per-part loop the multipart stream uses and
+    // collect the size of every part. Regression for rustfs/rustfs#4811: the old
+    // `read_all()` per part drained the whole source into part 1.
+    async fn collect_part_sizes(mut reader: ReaderImpl, total: usize, part_size: usize, last_part_size: usize) -> Vec<usize> {
+        let parts = total.div_ceil(part_size);
+        let mut sizes = Vec::new();
+        for part_number in 1..=parts {
+            let want = if part_number == parts { last_part_size } else { part_size };
+            let buf = read_multipart_part(&mut reader, want).await.unwrap();
+            sizes.push(buf.len());
+        }
+        // Nothing must remain after the planned parts are consumed.
+        assert!(read_multipart_part(&mut reader, part_size).await.unwrap().is_empty());
+        sizes
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_splits_streamed_object_body_evenly() {
+        // 250 bytes at part_size 100 -> parts [100, 100, 50], mirroring the
+        // >128 MiB / 128 MiB split from the bug report on a small deterministic
+        // stream.
+        let total = 250usize;
+        let (mut w, r) = tokio::io::duplex(64);
+        tokio::spawn(async move {
+            use tokio::io::AsyncWriteExt;
+            let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
+            w.write_all(&data).await.unwrap();
+        });
+        let reader = ReaderImpl::ObjectBody(GetObjectReader {
+            stream: Box::new(r),
+            object_info: Default::default(),
+            buffered_body: None,
+            body_source: Default::default(),
+        });
+
+        let sizes = collect_part_sizes(reader, total, 100, 50).await;
+        assert_eq!(sizes, vec![100, 100, 50]);
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_splits_in_memory_body_evenly() {
+        let total = 250usize;
+        let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
+        let reader = ReaderImpl::Body(Bytes::from(data));
+
+        let sizes = collect_part_sizes(reader, total, 100, 50).await;
+        assert_eq!(sizes, vec![100, 100, 50]);
+    }
+
+    #[tokio::test]
+    async fn read_multipart_part_stops_at_eof_without_overrun() {
+        // Reader shorter than the requested part size must return only what is
+        // available, not block or pad.
+        let (mut w, r) = tokio::io::duplex(64);
+        tokio::spawn(async move {
+            use tokio::io::AsyncWriteExt;
+            w.write_all(&[1u8; 30]).await.unwrap();
+        });
+        let mut reader = ReaderImpl::ObjectBody(GetObjectReader {
+            stream: Box::new(r),
+            object_info: Default::default(),
+            buffered_body: None,
+            body_source: Default::default(),
+        });
+        let buf = read_multipart_part(&mut reader, 100).await.unwrap();
+        assert_eq!(buf.len(), 30);
+    }
+
    fn parts_map(n: i64) -> HashMap<i64, ObjectPart> {
        let mut m = HashMap::new();
        for i in 1..=n {
@@ -209,7 +209,8 @@ pub struct ListObjectPartsResult {
    pub encoding_type: String,
 }

-#[derive(Debug, Default)]
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
 pub struct InitiateMultipartUploadResult {
    pub bucket: String,
    pub key: String,
@@ -229,15 +230,28 @@ pub struct CompleteMultipartUploadResult {
    pub checksum_crc64nvme: String,
 }

+// Field renames drive the CompleteMultipartUpload XML body sent to the remote.
+// Without them quick-xml emits the Rust field names (<part_num>/<etag>), so the
+// remote parses zero <Part> elements and completes a 0-byte object while still
+// returning 200 — the multipart transition silently produces an empty object
+// (rustfs/rustfs#4811). Empty checksum fields are skipped so an MD5-only
+// transition does not emit blank <ChecksumCRC32/> elements.
 #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
 pub struct CompletePart {
    //api has
-    pub etag: String,
+    #[serde(rename = "PartNumber")]
    pub part_num: i64,
+    #[serde(rename = "ETag")]
+    pub etag: String,
+    #[serde(rename = "ChecksumCRC32", skip_serializing_if = "String::is_empty")]
    pub checksum_crc32: String,
+    #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "String::is_empty")]
    pub checksum_crc32c: String,
+    #[serde(rename = "ChecksumSHA1", skip_serializing_if = "String::is_empty")]
    pub checksum_sha1: String,
+    #[serde(rename = "ChecksumSHA256", skip_serializing_if = "String::is_empty")]
    pub checksum_sha256: String,
+    #[serde(rename = "ChecksumCRC64NVME", skip_serializing_if = "String::is_empty")]
    pub checksum_crc64nvme: String,
 }

@@ -272,7 +286,9 @@ pub struct CopyObjectPartResult {
 }

 #[derive(Debug, Default, serde::Serialize)]
+#[serde(rename = "CompleteMultipartUpload")]
 pub struct CompleteMultipartUpload {
+    #[serde(rename = "Part")]
    pub parts: Vec<CompletePart>,
 }

@@ -71,7 +71,11 @@ impl ChecksumMode {
            8_u8 => ChecksumMode::ChecksumCRC32,
            16_u8 => ChecksumMode::ChecksumCRC32C,
            32_u8 => ChecksumMode::ChecksumCRC64NVME,
-            _ => panic!("enum err."),
+            // Fail closed: any mode without a concrete base algorithm (e.g. a
+            // bare ChecksumFullObject flag) is treated as "no checksum" rather
+            // than panicking. Callers already gate real work behind
+            // is_set()/can_composite()/hasher(), so this only removes a crash.
+            _ => ChecksumMode::ChecksumNone,
        }
    }

@@ -177,6 +181,17 @@ impl ChecksumMode {
    }

    pub fn is_set(&self) -> bool {
+        // `ChecksumNone` is the zeroth enum variant, so it occupies bit 0 of the
+        // `EnumSet` repr and a naive `len() == 1` check reports "no checksum" as a
+        // configured checksum. A checksum is only "set" when a concrete algorithm
+        // (one with a real hasher) is selected; the bare `ChecksumFullObject` flag
+        // has no base algorithm and is likewise not set. Treating `ChecksumNone`
+        // as set made ILM transitions of >128 MiB objects fail with
+        // "unsupported checksum type" (rustfs/rustfs#4811): the multipart put path
+        // took the checksum branch and called `ChecksumNone.hasher()`.
+        if matches!(self, ChecksumMode::ChecksumNone) {
+            return false;
+        }
        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
        s.len() == 1
    }
@@ -272,6 +287,70 @@ impl ChecksumMode {
    }
 }

+#[cfg(test)]
+mod tests {
+    use super::ChecksumMode;
+
+    #[test]
+    fn test_base_is_fail_closed_and_never_panics() {
+        // Every mode must resolve to a concrete base without panicking. The bare
+        // ChecksumFullObject flag has no base algorithm and must fall back to
+        // ChecksumNone instead of crashing (previously `panic!("enum err.")`).
+        assert_eq!(ChecksumMode::ChecksumFullObject.base(), ChecksumMode::ChecksumNone);
+        assert_eq!(ChecksumMode::ChecksumNone.base(), ChecksumMode::ChecksumNone);
+        assert_eq!(ChecksumMode::ChecksumCRC32.base(), ChecksumMode::ChecksumCRC32);
+        assert_eq!(ChecksumMode::ChecksumCRC32C.base(), ChecksumMode::ChecksumCRC32C);
+        assert_eq!(ChecksumMode::ChecksumSHA1.base(), ChecksumMode::ChecksumSHA1);
+        assert_eq!(ChecksumMode::ChecksumSHA256.base(), ChecksumMode::ChecksumSHA256);
+        assert_eq!(ChecksumMode::ChecksumCRC64NVME.base(), ChecksumMode::ChecksumCRC64NVME);
+    }
+
+    #[test]
+    fn test_hasher_fails_closed_for_unsupported_mode() {
+        // Modes without a real hasher must return an error, not panic.
+        assert!(ChecksumMode::ChecksumNone.hasher().is_err());
+        assert!(ChecksumMode::ChecksumFullObject.hasher().is_err());
+        assert!(ChecksumMode::ChecksumCRC32.hasher().is_ok());
+    }
+
+    #[test]
+    fn test_is_set_is_false_for_none_and_bare_full_object() {
+        // Regression for rustfs/rustfs#4811: `ChecksumNone` must NOT be reported as
+        // a configured checksum. It is the zeroth enum variant (bit 0 of the
+        // EnumSet repr), so the old `len() == 1` check treated it as set and drove
+        // the multipart put path into `ChecksumNone.hasher()` → "unsupported
+        // checksum type". Every mode reported as set must also have a real hasher.
+        assert!(!ChecksumMode::ChecksumNone.is_set());
+        assert!(!ChecksumMode::ChecksumFullObject.is_set());
+
+        for mode in [
+            ChecksumMode::ChecksumCRC32,
+            ChecksumMode::ChecksumCRC32C,
+            ChecksumMode::ChecksumSHA1,
+            ChecksumMode::ChecksumSHA256,
+            ChecksumMode::ChecksumCRC64NVME,
+        ] {
+            assert!(mode.is_set(), "{mode:?} should be set");
+            assert!(mode.hasher().is_ok(), "{mode:?} reported set but has no hasher");
+        }
+    }
+
+    #[test]
+    fn test_set_default_upgrades_none() {
+        // With `is_set()` fixed, `set_default` must upgrade an unset mode to the
+        // provided default (previously `ChecksumNone` was seen as set and never
+        // upgraded).
+        let mut mode = ChecksumMode::ChecksumNone;
+        mode.set_default(ChecksumMode::ChecksumCRC32C);
+        assert_eq!(mode, ChecksumMode::ChecksumCRC32C);
+
+        // An already-set mode is left untouched.
+        let mut existing = ChecksumMode::ChecksumSHA256;
+        existing.set_default(ChecksumMode::ChecksumCRC32C);
+        assert_eq!(existing, ChecksumMode::ChecksumSHA256);
+    }
+}
+
 #[derive(Default)]
 pub struct Checksum {
    checksum_type: ChecksumMode,
@@ -121,8 +121,11 @@ pub fn new_getobjectreader<'a>(
    let is_compressed = false; //oi.is_compressed_ok();

    let rs_;
-    if rs.is_none() && opts.part_number.is_some() && opts.part_number.expect("operation should succeed") > 0 {
-        rs_ = part_number_to_rangespec(oi.clone(), opts.part_number.expect("operation should succeed"));
+    if rs.is_none()
+        && let Some(part_number) = opts.part_number
+        && part_number > 0
+    {
+        rs_ = part_number_to_rangespec(oi.clone(), part_number);
    } else {
        rs_ = rs.clone();
    }
@@ -158,6 +161,16 @@ pub fn new_getobjectreader<'a>(

        return Ok((get_fn, off as i64, length as i64));
    }
+    if rs.is_none() && opts.part_number.is_none() && oi.size >= 0 {
+        get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| GetObjectReader {
+            object_info: oi.clone(),
+            stream: Box::new(input_reader),
+            buffered_body: None,
+            body_source: Default::default(),
+        });
+
+        return Ok((get_fn, 0, oi.size));
+    }
    Err(ErrorResponse {
        code: S3ErrorCode::InvalidRange,
        message: "Invalid range".to_string(),
@@ -198,6 +211,57 @@ pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use std::sync::Arc;
+
+    fn multipart_object_info() -> ObjectInfo {
+        ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: 6 * 1024 * 1024,
+            actual_size: 6 * 1024 * 1024,
+            parts: Arc::new(vec![
+                ObjectPartInfo {
+                    number: 1,
+                    size: 5 * 1024 * 1024,
+                    actual_size: 5 * 1024 * 1024,
+                    ..Default::default()
+                },
+                ObjectPartInfo {
+                    number: 2,
+                    size: 1024 * 1024,
+                    actual_size: 1024 * 1024,
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn test_new_getobjectreader_uses_full_range_for_unranged_multipart_object() {
+        let oi = multipart_object_info();
+        let opts = ObjectOptions::default();
+
+        let result = new_getobjectreader(&None, &oi, &opts, &HeaderMap::new())
+            .expect("unranged multipart object should build a full-object reader");
+
+        assert_eq!(result.1, 0);
+        assert_eq!(result.2, oi.size);
+    }
+
+    #[test]
+    fn test_new_getobjectreader_keeps_part_number_range_for_multipart_object() {
+        let oi = multipart_object_info();
+        let opts = ObjectOptions {
+            part_number: Some(2),
+            ..Default::default()
+        };
+
+        let result = new_getobjectreader(&None, &oi, &opts, &HeaderMap::new()).expect("part number should build that part range");
+
+        assert_eq!(result.1, 5 * 1024 * 1024);
+        assert_eq!(result.2, 1024 * 1024);
+    }

    #[test]
    fn test_to_s3s_etag() {
@@ -91,10 +91,41 @@ pub fn is_minio_header(header_key: &str) -> bool {
    header_key.to_lowercase().starts_with("x-minio-")
 }

+/// Standard base64 (with `+`/`/` and `=` padding). Every base64 value this
+/// transition client emits or parses — `Content-MD5`, `x-amz-checksum-*`, and
+/// checksum digests in request/response bodies — is S3 wire format, which is
+/// standard base64. The URL-safe, unpadded alphabet used previously made remotes
+/// reject `Content-MD5` with "Invalid content MD5: Base64Error" and could not
+/// even decode a padded checksum coming back from the peer (rustfs/rustfs#4811).
 pub fn base64_encode(input: &[u8]) -> String {
-    base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
+    base64_simd::STANDARD.encode_to_string(input)
 }

 pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
-    base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
+    base64_simd::STANDARD.decode_to_vec(input)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{base64_decode, base64_encode};
+
+    #[test]
+    fn base64_encode_is_standard_s3_wire_format() {
+        // S3 reads Content-MD5 / checksum values with a standard base64 decoder,
+        // so the encoder must emit '+'/'/' and '=' padding and round-trip through
+        // one. Regression for rustfs/rustfs#4811 ("Invalid content MD5:
+        // Base64Error"). 16-byte MD5-length input chosen to force '=' padding.
+        let digest: [u8; 16] = [
+            0xfb, 0xff, 0xff, 0xef, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0,
+        ];
+        let encoded = base64_encode(&digest);
+        assert!(encoded.ends_with('='), "16-byte input must be padded: {encoded}");
+        assert!(!encoded.contains(['-', '_']), "must use the standard alphabet: {encoded}");
+        let via_standard = base64_simd::STANDARD
+            .decode_to_vec(encoded.as_bytes())
+            .expect("standard decode");
+        assert_eq!(via_standard, digest);
+        // Our own decoder must accept the same wire format it produces.
+        assert_eq!(base64_decode(encoded.as_bytes()).expect("round-trip"), digest);
+    }
 }
@@ -23,17 +23,19 @@ use rustfs_config::{
    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
-    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
-    POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
-    POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
-    PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
-    PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
-    REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
-    REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
-    REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
-    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
+    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
+    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
+    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
+    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
+    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
+    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
+    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
+    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
+    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
+    WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
+    WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;

@@ -350,6 +352,21 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
            value: DEFAULT_LIMIT.to_string(),
            hidden_if_empty: false,
        },
+        KV {
+            key: NATS_JETSTREAM_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
+            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
+            hidden_if_empty: false,
+        },
        KV {
            key: COMMENT_KEY.to_owned(),
            value: "".to_owned(),
@@ -23,16 +23,18 @@ use rustfs_config::{
    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
-    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
-    POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
-    POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
-    PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
-    PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
-    REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
-    REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
-    REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
-    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
+    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
+    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
+    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
+    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
+    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
+    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
+    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
+    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
+    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
+    WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;

@@ -328,6 +330,21 @@ pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
            value: DEFAULT_LIMIT.to_string(),
            hidden_if_empty: false,
        },
+        KV {
+            key: NATS_JETSTREAM_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
+            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
+            hidden_if_empty: false,
+        },
        KV {
            key: COMMENT_KEY.to_owned(),
            value: "".to_owned(),
@@ -0,0 +1,146 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Deterministic crash-point injection for erasure-coded commit sequences.
+//!
+//! Each [`CrashPoint`] names one instant inside a multi-step commit at which a
+//! `#[cfg(test)]` scenario can simulate a hard power loss: the commit stops dead
+//! at the armed step with **no** cleanup, leaving the on-disk state exactly as
+//! the preceding steps left it. The test then reopens the disk (or rebuilds the
+//! erasure set) and asserts the raw state is coherent — the object reads back as
+//! either the whole old version or the whole new version, never a torn mix — and
+//! that any staged leftovers are reclaimable and the operation is safely
+//! retryable. This is the crash-consistency counterpart to the graceful
+//! `should_fail_*` failpoints, which exercise in-process rollback rather than an
+//! abrupt loss.
+//!
+//! Unlike the graceful failpoints, a crash point runs no rollback: it models the
+//! process disappearing, so the assertion is purely over the bytes left on disk.
+//!
+//! Test-only: the arming registry and the real [`should_crash_at`] are
+//! `#[cfg(test)]`. In every other build `should_crash_at` is a const-`false`
+//! `#[inline(always)]` no-op, so the injection points on the real commit paths
+//! (`rename_data`, `write_all_meta`, `complete_multipart_upload`) compile away to
+//! nothing and add no branch to production writes. The [`CrashPoint`] variants
+//! are still constructed at those call sites in every build, so no variant is
+//! dead code.
+//!
+//! Coverage plan: rustfs/backlog#935 / rustfs/backlog#896 (rename_data),
+//! rustfs/backlog#864 (fault-interrupted overwrite/delete must not mutate a
+//! committed object), rustfs/backlog#878 (old-or-new-never-mixed).
+
+/// A named instant inside an erasure-coded commit sequence at which a test can
+/// simulate a hard power loss.
+///
+/// All points share one keyed registry (each arm is matched on `(point, key)`),
+/// so a scenario arms exactly the window under test and every other commit path
+/// runs untouched.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum CrashPoint {
+    /// `rename_data`: after the data dir is renamed into its destination but
+    /// before the old-metadata rollback backup is written. Pre-commit — xl.meta
+    /// is untouched, so a crash here must leave the object readable as the old
+    /// version (or absent when there was none).
+    RenameAfterDataRename,
+    /// `rename_data`: after the rollback backup is durable, immediately before
+    /// the xl.meta commit rename that makes the new version visible. Still
+    /// pre-commit, so a crash here must also leave the old version readable.
+    RenameAfterBackupBeforeMetaCommit,
+    /// `rename_data`: after the xl.meta commit rename has made the new version
+    /// visible, before the durability fsync and with **no** rollback. The commit
+    /// rename already landed, so a crash here must leave the object readable as
+    /// the new version (rustfs/backlog#878, new-version side).
+    RenameAfterMetaCommit,
+    /// `write_all_meta` (the atomic temp+rename behind `update_metadata` and
+    /// `write_metadata`): after the replacement xl.meta is staged in the tmp
+    /// bucket but before the rename that publishes it. Pre-commit — the
+    /// destination xl.meta is untouched, so a crash here must leave the object's
+    /// metadata byte-for-byte the old version (rustfs/backlog#864); the staged
+    /// tmp file is a harmless orphan swept by tmp-bucket GC.
+    MetaWriteAfterTmpBeforeRename,
+    /// `complete_multipart_upload`: after the upload is fully staged and locked
+    /// but before the authoritative `rename_data` commit. Pre-commit — no disk
+    /// has moved staged data, so a crash here must leave any prior committed
+    /// version byte-for-byte intact (rustfs/backlog#864) and the upload fully
+    /// retryable.
+    MultipartBeforeCommitRename,
+    /// `complete_multipart_upload`: after the `rename_data` commit succeeds but
+    /// before the stale `part.N.meta` cleanup. Post-commit — the new version is
+    /// durably committed and visible, so a crash here must leave the object
+    /// readable as the new version; the un-reclaimed staging parts are swept by
+    /// a retried completion or upload GC (rustfs/backlog#946).
+    MultipartAfterCommitBeforePartsCleanup,
+}
+
+#[cfg(test)]
+pub(crate) use armed::{arm, disarm, should_crash_at};
+
+/// Production no-op: the compiler inlines this to `false`, so armed crash points
+/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
+#[cfg(not(test))]
+#[inline(always)]
+pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
+    false
+}
+
+#[cfg(test)]
+mod armed {
+    use super::CrashPoint;
+    use std::sync::{Mutex, MutexGuard};
+
+    // A keyed multiset of pending arms, not a single slot: the rename_data /
+    // xl.meta scenarios serialize via `durability_mode_override` while the
+    // multipart scenarios use `#[serial]`, so the two groups can overlap under
+    // `cargo test`'s shared-process threads. Keying every arm on a unique object
+    // path lets concurrent scenarios coexist without clobbering each other; a
+    // match on `(point, key)` is unambiguous. `#[serial]` still gives the
+    // multipart tests their own serialization for unrelated global state.
+    static ARMED: Mutex<Vec<(CrashPoint, String)>> = Mutex::new(Vec::new());
+
+    fn guard() -> MutexGuard<'static, Vec<(CrashPoint, String)>> {
+        // A poisoned lock only means a prior test panicked mid-scenario; recover
+        // the registry rather than cascade the panic into unrelated tests.
+        ARMED.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+    }
+
+    /// Arm a one-shot crash: the next operation that reaches `point` while
+    /// committing `key` stops there. Consumed on the first match so it never
+    /// leaks into an unrelated commit. Arming the same `(point, key)` twice is
+    /// idempotent.
+    pub(crate) fn arm(point: CrashPoint, key: &str) {
+        let mut armed = guard();
+        if !armed.iter().any(|(p, k)| *p == point && k == key) {
+            armed.push((point, key.to_string()));
+        }
+    }
+
+    /// Clear the pending arm for exactly this `(point, key)`. A scenario retries
+    /// the same key after the crash, so teardown disarms in case the injection
+    /// was somehow never reached and would otherwise fire inside the retry.
+    pub(crate) fn disarm(point: CrashPoint, key: &str) {
+        guard().retain(|(p, k)| !(*p == point && k == key));
+    }
+
+    /// Returns `true` (consuming the arm) iff a crash is armed for exactly this
+    /// `point` and `key`.
+    pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
+        let mut armed = guard();
+        if let Some(idx) = armed.iter().position(|(p, k)| *p == point && k == key) {
+            armed.swap_remove(idx);
+            true
+        } else {
+            false
+        }
+    }
+}
@@ -13,6 +13,7 @@
 // limitations under the License.

 use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
+use crate::crash_inject::{self, CrashPoint};
 use crate::data_usage::local_snapshot::ensure_data_usage_layout;
 use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout};
 use crate::disk::{
@@ -1478,79 +1479,6 @@ fn should_fail_before_old_metadata_backup(_dst_path: &str) -> bool {
    false
 }

-/// Commit-sequence points where the rename_data crash-consistency harness
-/// (rustfs/backlog#935, test plan in rustfs/backlog#896) can simulate an
-/// abrupt power loss.
-///
-/// Unlike [`should_fail_before_old_metadata_backup`], which exercises the
-/// graceful in-process rollback (delete the staged data dir, return an
-/// error), a crash point models a hard power loss: the commit sequence stops
-/// dead at the armed step with **no** cleanup, leaving the on-disk state
-/// exactly as the preceding steps left it. The harness then reopens the disk
-/// and asserts the raw state is coherent — the object reads back as either the
-/// old version or the new version, never a mixed or corrupt one — without any
-/// rollback code having run.
-///
-/// The variants are constructed at the real commit-path call sites in every
-/// build, but the arming static and [`should_crash_rename_data_at`] are
-/// `#[cfg(test)]`; in production the guard is a const-`false` no-op, so the
-/// injection points compile away to nothing.
-// The `After<step>` naming is deliberate: every crash point names the
-// commit-sequence step it fires immediately after, so the shared prefix is the
-// point, not noise.
-#[allow(clippy::enum_variant_names)]
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub(crate) enum RenameDataCrashPoint {
-    /// After the data dir has been renamed into its destination but before the
-    /// old-metadata rollback backup is written. xl.meta has not been committed
-    /// yet, so a crash here must leave the object readable as the old version.
-    AfterDataRename,
-    /// After the rollback backup is persisted, immediately before the xl.meta
-    /// commit rename that makes the new version visible. Still pre-commit, so a
-    /// crash here must also leave the object readable as the old version.
-    AfterBackupBeforeMetaCommit,
-    /// After the xl.meta commit rename has made the new version visible, in the
-    /// same window the graceful [`should_fail_after_metadata_commit`] failpoint
-    /// covers — but as a hard power loss with **no** rollback. The commit rename
-    /// already landed on disk, so a crash here must leave the object readable as
-    /// the *new* version. This is the post-commit counterpart to the two
-    /// pre-commit points above, closing the "old or new, never mixed" invariant
-    /// (rustfs/backlog#878) from the new-version side.
-    AfterMetaCommit,
-}
-
-#[cfg(test)]
-static RENAME_DATA_CRASH_POINT: std::sync::Mutex<Option<(RenameDataCrashPoint, String)>> = std::sync::Mutex::new(None);
-
-/// Arm a one-shot crash injection: the next `rename_data` committing into
-/// `dst_path` stops at `point`. Consumed on the first match so it never leaks
-/// into an unrelated commit.
-#[cfg(test)]
-fn arm_rename_data_crash(point: RenameDataCrashPoint, dst_path: &str) {
-    *RENAME_DATA_CRASH_POINT
-        .lock()
-        .expect("test crash point lock should not be poisoned") = Some((point, dst_path.to_string()));
-}
-
-#[cfg(test)]
-fn should_crash_rename_data_at(point: RenameDataCrashPoint, dst_path: &str) -> bool {
-    let mut armed = RENAME_DATA_CRASH_POINT
-        .lock()
-        .expect("test crash point lock should not be poisoned");
-    if armed.as_ref().is_some_and(|(p, path)| *p == point && path == dst_path) {
-        armed.take();
-        true
-    } else {
-        false
-    }
-}
-
-#[cfg(not(test))]
-#[inline(always)]
-fn should_crash_rename_data_at(_point: RenameDataCrashPoint, _dst_path: &str) -> bool {
-    false
-}
-
 #[cfg(not(test))]
 fn should_fail_after_metadata_commit(_dst_path: &str) -> bool {
    false
@@ -4755,9 +4683,7 @@ impl LocalDisk {
                        .await);
                    }
                    let rollback_data_path = rollback_path.join(dir.to_string());
-                    if let Err(err) = rename_all(&dir_path, &rollback_data_path, &rollback_path).await
-                        && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
-                    {
+                    if let Err(err) = rename_all_ignore_missing_source(&dir_path, &rollback_data_path, &rollback_path).await {
                        return Err(restore_delete_rollback_after_error(
                            object_dir,
                            &xlpath,
@@ -4872,6 +4798,16 @@ impl LocalDisk {
        self.write_all_internal(&tmp_file_path, InternalBuf::Ref(buf), tmp_sync, &tmp_volume_dir)
            .await?;

+        // Crash-consistency injection: hard power loss after the replacement
+        // xl.meta is staged in the tmp bucket but before the atomic rename that
+        // publishes it. The destination xl.meta is untouched, so a crash here
+        // must leave the object's metadata byte-for-byte the old version
+        // (rustfs/backlog#864); the staged tmp file is a harmless orphan swept by
+        // tmp-bucket GC. Compiles to a no-op outside `#[cfg(test)]`.
+        if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, path) {
+            return Err(DiskError::Unexpected);
+        }
+
        rename_all(tmp_file_path, &file_path, volume_dir).await?;

        if sync
@@ -6855,7 +6791,7 @@ impl DiskAPI for LocalDisk {
            // is in place but before xl.meta commits. No cleanup — the harness
            // reopens the disk and asserts the object still reads as the old
            // version (the staged data dir is a harmless orphan for GC).
-            if should_crash_rename_data_at(RenameDataCrashPoint::AfterDataRename, dst_path) {
+            if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -6913,7 +6849,7 @@ impl DiskAPI for LocalDisk {
            // backup is durable but before the xl.meta commit rename. No
            // cleanup — the harness asserts the object still reads as the old
            // version, since the destination xl.meta is untouched here.
-            if should_crash_rename_data_at(RenameDataCrashPoint::AfterBackupBeforeMetaCommit, dst_path) {
+            if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -6946,7 +6882,7 @@ impl DiskAPI for LocalDisk {
            // graceful failpoint above, no rollback runs — the commit rename is
            // already on disk, so the harness asserts the object reads back as
            // the new version.
-            if should_crash_rename_data_at(RenameDataCrashPoint::AfterMetaCommit, dst_path) {
+            if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -7681,10 +7617,7 @@ impl DiskAPI for LocalDisk {
                    .await);
                }
                let rollback_data_path = rollback_path.join(uuid.to_string());
-                if let Err(err) = rename_all(&old_path, &rollback_data_path, &rollback_path).await
-                    && err != DiskError::FileNotFound
-                    && err != DiskError::VolumeNotFound
-                {
+                if let Err(err) = rename_all_ignore_missing_source(&old_path, &rollback_data_path, &rollback_path).await {
                    return Err(restore_delete_rollback_after_error(
                        file_path.as_path(),
                        &xl_path,
@@ -7962,10 +7895,56 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
 mod test {
    use super::*;
    use rustfs_filemeta::ErasureInfo;
-    use std::io;
+    use std::io::{self, Write};
    use std::pin::Pin;
+    use std::sync::{Arc, Mutex};
    use std::task::{Context, Poll};
    use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
+    use tracing_subscriber::fmt::MakeWriter;
+
+    #[derive(Clone, Default)]
+    struct CapturedLogs {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    struct CapturedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl CapturedLogs {
+        fn contents(&self) -> String {
+            let buffer = self
+                .buffer
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .clone();
+            String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
+        }
+    }
+
+    impl Write for CapturedLogWriter {
+        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+            self.buffer
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for CapturedLogs {
+        type Writer = CapturedLogWriter;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            CapturedLogWriter {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }

    fn test_file_info(name: &str, version_id: Uuid, data_dir: Option<Uuid>, data: Option<Bytes>) -> FileInfo {
        let size = data
@@ -8143,13 +8122,14 @@ mod test {
    /// old-or-new invariant, only with a wider (documented) power-loss window.
    mod crash_consistency {
        use super::*;
+        use crate::crash_inject::{self, CrashPoint};
        use tempfile::tempdir;

        const VERSION_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
        const OLD_DATA_DIR: &str = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
        const NEW_DATA_DIR: &str = "cccccccc-cccc-cccc-cccc-cccccccccccc";

-        async fn run_scenario(mode: DurabilityMode, crash: Option<RenameDataCrashPoint>, with_old_version: bool) {
+        async fn run_scenario(mode: DurabilityMode, crash: Option<CrashPoint>, with_old_version: bool) {
            // Serializes with every other durability-sensitive test and pins the
            // resolved tier for the whole scenario (held until dropped).
            let _mode = durability_mode_override::set(mode);
@@ -8200,7 +8180,7 @@ mod test {
                .expect("new tmp data should be written");

            if let Some(point) = crash {
-                arm_rename_data_crash(point, object);
+                crash_inject::arm(point, object);
            }
            let new_fi = test_file_info(object, version_id, Some(new_data_dir), None);
            let result = disk
@@ -8215,7 +8195,7 @@ mod test {
                .await;

            match crash {
-                Some(RenameDataCrashPoint::AfterMetaCommit) => {
+                Some(CrashPoint::RenameAfterMetaCommit) => {
                    // Hard power loss right after the xl.meta commit rename: the
                    // commit landed and no rollback ran, so the object must read
                    // back as the new version — whether or not an old version
@@ -8283,9 +8263,9 @@ mod test {
            }
        }

-        const CRASH_POINTS: [RenameDataCrashPoint; 2] = [
-            RenameDataCrashPoint::AfterDataRename,
-            RenameDataCrashPoint::AfterBackupBeforeMetaCommit,
+        const CRASH_POINTS: [CrashPoint; 2] = [
+            CrashPoint::RenameAfterDataRename,
+            CrashPoint::RenameAfterBackupBeforeMetaCommit,
        ];

        #[tokio::test]
@@ -8331,22 +8311,138 @@ mod test {
        // the same invariant.
        #[tokio::test]
        async fn overwrite_post_commit_crash_keeps_new_version_strict() {
-            run_scenario(DurabilityMode::Strict, Some(RenameDataCrashPoint::AfterMetaCommit), true).await;
+            run_scenario(DurabilityMode::Strict, Some(CrashPoint::RenameAfterMetaCommit), true).await;
        }

        #[tokio::test]
        async fn overwrite_post_commit_crash_keeps_new_version_relaxed() {
-            run_scenario(DurabilityMode::Relaxed, Some(RenameDataCrashPoint::AfterMetaCommit), true).await;
+            run_scenario(DurabilityMode::Relaxed, Some(CrashPoint::RenameAfterMetaCommit), true).await;
        }

        #[tokio::test]
        async fn fresh_post_commit_crash_keeps_new_version_strict() {
-            run_scenario(DurabilityMode::Strict, Some(RenameDataCrashPoint::AfterMetaCommit), false).await;
+            run_scenario(DurabilityMode::Strict, Some(CrashPoint::RenameAfterMetaCommit), false).await;
        }

        #[tokio::test]
        async fn fresh_post_commit_crash_keeps_new_version_relaxed() {
-            run_scenario(DurabilityMode::Relaxed, Some(RenameDataCrashPoint::AfterMetaCommit), false).await;
+            run_scenario(DurabilityMode::Relaxed, Some(CrashPoint::RenameAfterMetaCommit), false).await;
+        }
+    }
+
+    /// Crash-consistency for the in-place xl.meta update path — the atomic
+    /// temp+rename inside [`LocalDisk::write_all_meta`] shared by `update_metadata`
+    /// and `write_metadata` (delete markers, tag/metadata rewrites, decommission).
+    ///
+    /// rustfs/backlog#864: a fault that interrupts an in-place metadata rewrite
+    /// must not mutate the committed object. [`CrashPoint::MetaWriteAfterTmpBeforeRename`]
+    /// models a hard power loss after the replacement xl.meta is staged in the tmp
+    /// bucket but before the publishing rename. Both durability tiers are held to
+    /// the same rule: the destination xl.meta survives byte-for-byte, the object
+    /// still reads as the old version, the staged tmp file is a reclaimable orphan
+    /// confined to the tmp bucket, and a later un-injected rewrite publishes
+    /// cleanly (retryable). This path previously had only parser-level unit tests.
+    mod meta_write_crash_consistency {
+        use super::*;
+        use crate::crash_inject::{self, CrashPoint};
+        use tempfile::tempdir;
+
+        async fn run(mode: DurabilityMode) {
+            // Serialize with every other durability-sensitive test and pin the
+            // resolved tier for the whole scenario.
+            let _mode = durability_mode_override::set(mode);
+
+            let dir = tempdir().expect("temp dir should be created");
+            let endpoint =
+                Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+            let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+
+            let bucket = "bucket";
+            let object = "meta-write-crash-object";
+            ensure_test_volume(&disk, bucket).await;
+            ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+
+            // Seed a committed object (version 1) by writing its xl.meta directly:
+            // read_data=false reads never touch the data dir, so no shard staging
+            // is needed to exercise the metadata commit window.
+            let object_dir = dir.path().join(bucket).join(object);
+            fs::create_dir_all(&object_dir).await.expect("object dir should be created");
+            let meta_path = object_dir.join(STORAGE_FORMAT_FILE);
+            let v1 = Uuid::new_v4();
+            let old_meta = test_meta(test_file_info(object, v1, Some(Uuid::new_v4()), None));
+            fs::write(&meta_path, &old_meta)
+                .await
+                .expect("committed xl.meta should be written");
+
+            // Arm the crash, then attempt an in-place rewrite that adds version 2.
+            let meta_key = format!("{object}/{STORAGE_FORMAT_FILE}");
+            crash_inject::arm(CrashPoint::MetaWriteAfterTmpBeforeRename, &meta_key);
+            let v2 = Uuid::new_v4();
+            let result = disk
+                .write_metadata("", bucket, object, test_file_info(object, v2, Some(Uuid::new_v4()), None))
+                .await;
+            assert!(
+                matches!(result, Err(DiskError::Unexpected)),
+                "{mode:?}: the armed crash point must be the failure that surfaced, got {result:?}"
+            );
+
+            // Reopen the disk to model a process restart after the crash.
+            drop(disk);
+            let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should reopen");
+
+            // rustfs/backlog#864: the committed xl.meta is byte-for-byte the old
+            // version — the interrupted rewrite never published.
+            let after = fs::read(&meta_path).await.expect("xl.meta must survive the crash");
+            assert_eq!(&after, &old_meta, "{mode:?}: xl.meta must remain the old version byte-for-byte");
+
+            // The old version still reads; the un-published new version is absent.
+            disk.read_version("", bucket, object, &v1.to_string(), &ReadOptions::default())
+                .await
+                .expect("the old version must remain readable after the crash");
+            let new_read = disk
+                .read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
+                .await;
+            assert!(
+                matches!(new_read, Err(DiskError::FileVersionNotFound)),
+                "{mode:?}: the interrupted update must not publish the new version, got {new_read:?}"
+            );
+
+            // The crash leaves no staging debris in the object directory; the
+            // staged replacement is a reclaimable orphan under the tmp bucket.
+            let mut object_entries = fs::read_dir(&object_dir).await.expect("object dir should list");
+            let mut object_files = Vec::new();
+            while let Some(entry) = object_entries
+                .next_entry()
+                .await
+                .expect("object dir entry should be readable")
+            {
+                object_files.push(entry.file_name().to_string_lossy().to_string());
+            }
+            assert_eq!(
+                object_files,
+                vec![STORAGE_FORMAT_FILE.to_string()],
+                "{mode:?}: the object directory must hold only its committed xl.meta"
+            );
+
+            // A retried rewrite (un-injected) publishes cleanly: the path is safely
+            // retryable after the crash.
+            crash_inject::disarm(CrashPoint::MetaWriteAfterTmpBeforeRename, &meta_key);
+            disk.write_metadata("", bucket, object, test_file_info(object, v2, Some(Uuid::new_v4()), None))
+                .await
+                .expect("a retried metadata rewrite must succeed after the crash");
+            disk.read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
+                .await
+                .expect("the retried version must be readable");
+        }
+
+        #[tokio::test]
+        async fn meta_write_crash_before_rename_keeps_old_version_strict() {
+            run(DurabilityMode::Strict).await;
+        }
+
+        #[tokio::test]
+        async fn meta_write_crash_before_rename_keeps_old_version_relaxed() {
+            run(DurabilityMode::Relaxed).await;
        }
    }

@@ -10376,6 +10472,149 @@ mod test {
        );
    }

+    #[tokio::test(flavor = "current_thread")]
+    async fn test_delete_version_missing_inline_data_dir_does_not_warn() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+
+        let bucket = "bucket";
+        let object = "dir/inline-object";
+        let version_id = Uuid::parse_str("31313131-1111-2222-3333-444444444444").expect("version id should parse");
+        let data_dir = Uuid::parse_str("32323232-1111-2222-3333-444444444444").expect("data dir should parse");
+        let rollback_dir = Uuid::parse_str("33333333-1111-2222-3333-444444444444").expect("rollback dir should parse");
+
+        ensure_test_volume(&disk, bucket).await;
+
+        let object_dir = dir.path().join(bucket).join(object);
+        fs::create_dir_all(&object_dir).await.expect("object dir should be created");
+        let fi = test_file_info(object, version_id, Some(data_dir), Some(Bytes::from_static(b"inline")));
+        fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(fi.clone()))
+            .await
+            .expect("inline metadata should be written");
+        assert!(!object_dir.join(data_dir.to_string()).exists());
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        disk.delete_version(
+            bucket,
+            object,
+            fi,
+            false,
+            DeleteOptions {
+                old_data_dir: Some(rollback_dir),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("missing inline data dir should not fail deletion");
+
+        assert!(!object_dir.join(STORAGE_FORMAT_FILE).exists());
+        assert!(
+            object_dir
+                .join(rollback_dir.to_string())
+                .join(STORAGE_FORMAT_FILE_BACKUP)
+                .exists()
+        );
+        assert!(!logs.contents().contains("reliable_rename failed"));
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_delete_versions_missing_inline_data_dir_does_not_warn() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+
+        let bucket = "bucket";
+        let object = "dir/inline-object-batch";
+        let version_id = Uuid::parse_str("34343434-1111-2222-3333-444444444444").expect("version id should parse");
+        let data_dir = Uuid::parse_str("35353535-1111-2222-3333-444444444444").expect("data dir should parse");
+        let rollback_dir = Uuid::parse_str("36363636-1111-2222-3333-444444444444").expect("rollback dir should parse");
+
+        ensure_test_volume(&disk, bucket).await;
+
+        let object_dir = dir.path().join(bucket).join(object);
+        fs::create_dir_all(&object_dir).await.expect("object dir should be created");
+        let fi = test_file_info(object, version_id, Some(data_dir), Some(Bytes::from_static(b"inline")));
+        fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(fi.clone()))
+            .await
+            .expect("inline metadata should be written");
+        assert!(!object_dir.join(data_dir.to_string()).exists());
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        disk.delete_versions_internal(
+            bucket,
+            object,
+            &[fi],
+            &DeleteOptions {
+                old_data_dir: Some(rollback_dir),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("missing inline data dir should not fail batch deletion");
+
+        assert!(!object_dir.join(STORAGE_FORMAT_FILE).exists());
+        assert!(
+            object_dir
+                .join(rollback_dir.to_string())
+                .join(STORAGE_FORMAT_FILE_BACKUP)
+                .exists()
+        );
+        assert!(!logs.contents().contains("reliable_rename failed"));
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_ignore_missing_source_helper_still_warns_on_real_error() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("temp dir should be created");
+        let src = dir.path().join("src");
+        let dst = dir.path().join("dst");
+        fs::create_dir_all(&src).await.expect("source dir should be created");
+        fs::write(src.join("part.1"), b"live-data")
+            .await
+            .expect("source data should be written");
+        fs::create_dir_all(&dst).await.expect("destination dir should be created");
+        fs::write(dst.join("sentinel"), b"conflict")
+            .await
+            .expect("destination conflict should be written");
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        let err = rename_all_ignore_missing_source(&src, &dst, dir.path())
+            .await
+            .expect_err("a non-empty rollback destination must reject rename");
+
+        assert_ne!(err, DiskError::FileNotFound);
+        assert!(src.exists());
+        assert!(dst.join("sentinel").exists());
+        assert!(logs.contents().contains("reliable_rename failed"));
+    }
+
    #[tokio::test]
    async fn test_delete_version_rollback_restores_staged_data_dir() {
        use tempfile::tempdir;
@@ -26,7 +26,7 @@ use std::{
 };
 use tokio::fs;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
-use tracing::{debug, warn};
+use tracing::warn;

 /// Check path length according to OS limits.
 pub fn check_path_length(path_name: &str) -> Result<()> {
@@ -527,7 +527,7 @@ async fn reliable_rename_inner(
    src_file_path: impl AsRef<Path>,
    dst_file_path: impl AsRef<Path>,
    base_dir: impl AsRef<Path>,
-    warn_on_failure: bool,
+    warn_on_missing_source: bool,
 ) -> io::Result<()> {
    if let Some(parent) = dst_file_path.as_ref().parent()
        && !file_exists(parent)
@@ -542,32 +542,8 @@ async fn reliable_rename_inner(
                i += 1;
                continue;
            }
-            if warn_on_failure {
-                // NotFound is an expected outcome for speculative renames (staging
-                // a dataDir that inline objects never had, trashing an already-
-                // removed tmp path, restoring an absent rollback backup): those
-                // callers treat FileNotFound as acceptable, and on per-request
-                // delete paths a WARN becomes a log storm (one line per disk per
-                // DELETE). Keep it at debug so quorum-masked cases (e.g. a tmp
-                // file lost before a commit rename) stay diagnosable; genuine
-                // failures keep the WARN. The error is returned either way.
-                if e.kind() == io::ErrorKind::NotFound {
-                    debug!(
-                        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
-                        src_file_path.as_ref(),
-                        dst_file_path.as_ref(),
-                        base_dir.as_ref(),
-                        e
-                    );
-                } else {
-                    warn!(
-                        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
-                        src_file_path.as_ref(),
-                        dst_file_path.as_ref(),
-                        base_dir.as_ref(),
-                        e
-                    );
-                }
+            if warn_on_missing_source || e.kind() != io::ErrorKind::NotFound {
+                warn_reliable_rename_failure(src_file_path.as_ref(), dst_file_path.as_ref(), base_dir.as_ref(), &e);
            }
            return Err(e);
        }
@@ -578,6 +554,13 @@ async fn reliable_rename_inner(
    Ok(())
 }

+fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
+    warn!(
+        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
+        src_file_path, dst_file_path, base_dir, err
+    );
+}
+
 /// Whether a failed `rename` in [`reliable_rename_inner`] should be retried.
 ///
 /// Only the first failure is retried, and `NotFound` is never retried: the
@@ -755,24 +738,22 @@ mod tests {
    }

    #[tokio::test]
-    async fn rename_all_ignore_missing_source_returns_ok() {
+    async fn rename_all_ignore_missing_source_returns_ok_without_warn() {
        let temp_dir = tempdir().expect("create temp dir");
        let src = temp_dir.path().join("missing");
        let dst = temp_dir.path().join("dst");

+        let (logs, _guard) = warn_capture();
        rename_all_ignore_missing_source(&src, &dst, temp_dir.path())
            .await
            .expect("missing cleanup source must be ignored");

        assert!(!dst.exists());
+        assert!(!logs.contents().contains("reliable_rename failed"));
    }

    #[tokio::test]
-    async fn rename_all_missing_source_skips_rename_warn() {
-        // Callers like delete_version's rollback staging accept FileNotFound
-        // (inline objects have no dataDir), so a missing source must not emit
-        // the per-disk "reliable_rename failed" WARN — under delete storms it
-        // was pure log noise. The error itself is still returned.
+    async fn rename_all_missing_source_still_warns() {
        let temp_dir = tempdir().expect("create temp dir");
        let src = temp_dir.path().join("missing");
        let dst = temp_dir.path().join("dst");
@@ -785,8 +766,8 @@ mod tests {
        assert!(matches!(err, DiskError::FileNotFound));
        let captured = logs.contents();
        assert!(
-            !captured.contains("reliable_rename failed"),
-            "NotFound must not emit the reliable_rename WARN, got: {captured}"
+            captured.contains("reliable_rename failed"),
+            "ordinary missing-source failures must keep the WARN, got: {captured}"
        );
    }

@@ -1091,6 +1091,7 @@ where
        let mut completed = 0usize;
        let mut failed = 0usize;
        let mut first_shard_recorded = false;
+        let mut hedged = false;
        // Scope the reader borrow (held by `reader_iter`/`sets`) so it is released
        // before the retirement pass mutates `self.readers` below.
        {
@@ -1115,7 +1116,53 @@ where
                ));
            }

-            while let Some((i, _read_cost, result, _should_retire)) = sets.next().await {
+            // Hedge the straggler. Once every healthy shard has had `hedge_delay`
+            // to return, stop waiting out the full per-shard `read_timeout` for a
+            // slow shard: cancel the still-pending reads and let the stripe-aligned
+            // parity substitution below cover their slots — but only when enough
+            // unengaged parity remains to reconstruct them, otherwise keep waiting.
+            // A slow shard here otherwise stalls the whole stripe (and the GET's
+            // first byte) until the 10s read timeout fires: the large-object GET
+            // first-byte tail seen in backlog#1156.
+            let hedge_delay = shard_read_hedge_delay(read_timeout);
+            let hedge_sleep = hedge_delay.map(tokio::time::sleep);
+            tokio::pin!(hedge_sleep);
+            loop {
+                let item = match hedge_sleep.as_mut().as_pin_mut() {
+                    Some(sleep) if !hedged => {
+                        tokio::select! {
+                            biased;
+                            item = sets.next() => item,
+                            _ = sleep => {
+                                hedged = true;
+                                // Stop waiting for the straggler(s) only once the stripe
+                                // can be finished without them: either every data shard
+                                // is already in (no reconstruction needed), or there are
+                                // at least `data_shards + 1` shards — decode quorum plus
+                                // a reconstruction-verification source, so a missing data
+                                // shard is reconstructed *and* verified, never settled at
+                                // exactly `data_shards` (which would silently skip
+                                // verification, erasure.rs). In the read-all default every
+                                // slot is engaged, so `sets` already holds data + parity
+                                // and this stops one slow shard from stalling the stripe
+                                // (and the GET's first byte) for the full read timeout
+                                // (backlog#1156). Otherwise keep waiting — the straggler
+                                // may be the verification source, or the only route to
+                                // quorum on a degraded stripe.
+                                let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
+                                let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
+                                if !data_missing || succeeded > data_shards {
+                                    break;
+                                }
+                                continue;
+                            }
+                        }
+                    }
+                    _ => sets.next().await,
+                };
+                let Some((i, _read_cost, result, _should_retire)) = item else {
+                    break;
+                };
                completed += 1;
                if !first_shard_recorded {
                    if let Some(path) = metrics_path {
@@ -1134,14 +1181,46 @@ where
                        failed += 1;
                    }
                }
+                // Once the hedge has fired, stop as soon as the stripe can finish
+                // (all data present, or the decode+verification quorum) rather than
+                // draining every straggler — a shard that resolves just after the
+                // hedge must not pull the first byte back out to the read timeout.
+                // Gated on `hedged` so a healthy stripe still reads every shard
+                // (the read-all default invariant, backlog#923 gate off).
+                if hedged {
+                    let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
+                    let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
+                    if !data_missing || succeeded > data_shards {
+                        break;
+                    }
+                }
            }
        }

-        // A data shard may have died during this stripe's reads. The unengaged
+        // Retire any shard the hedge left in flight: dropping `sets` above
+        // cancelled its read, so the reader is now at an indeterminate stream
+        // position and can never be block-aligned again — the same reason a
+        // mid-block read error retires a reader. Its slot stays missing and is
+        // covered by the stripe-aligned parity substitution below.
+        if hedged {
+            for i in 0..num_readers {
+                if participating[i] && shards[i].is_none() && errs[i].is_none() {
+                    errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read hedged after a slow shard")));
+                    retire_readers.push(i);
+                }
+            }
+        }
+
+        // A data shard may have died or been hedged this stripe. The unengaged
        // parity readers are still unopened, so they can be aligned to *this*
        // stripe and read now: engage them one at a time until the stripe has
        // `data_shards + 1` successful shards (decode quorum plus one
-        // reconstruction-verification source) or parity runs out.
+        // reconstruction-verification source) or parity runs out. In the read-all
+        // default (`RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE` off) every parity
+        // slot is already engaged, so `!self.engaged[idx]` finds nothing and this
+        // loop is a no-op — the hedged data shard is reconstructed from the parity
+        // already read above. It only substitutes deferred parity in the
+        // data-shards-only path (backlog#923).
        loop {
            let success_now = shards.iter().filter(|shard| shard.is_some()).count();
            let data_shard_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
@@ -3632,6 +3711,152 @@ mod tests {
        assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
    }

+    /// Lockstep GET regression for the large-object first-byte tail (backlog#1156).
+    /// A single slow data shard must not stall the stripe for the full per-shard
+    /// `read_timeout`: once the hedge delay elapses and the stripe already holds a
+    /// decode-plus-verification quorum, the lockstep read retires the straggler and
+    /// returns in ~hedge_delay even though `read_timeout` is 60s. Before the hedge
+    /// the lockstep path waited for every launched read, so this completed in
+    /// ~read_timeout (the 10s stall observed on large-object GETs) instead of ~100ms.
+    #[tokio::test]
+    async fn test_lockstep_hedges_slow_shard_instead_of_waiting_read_timeout() {
+        const NUM_SHARDS: usize = 1;
+        const BLOCK_SIZE: usize = 64;
+        const DATA_SHARDS: usize = 2;
+        // Two parity so data_shards + 1 shards are available without the slow one.
+        const PARITY_SHARDS: usize = 2;
+        const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+
+        let hash_algo = HashAlgorithm::None;
+        let readers = vec![
+            // Data shard 0 never resolves; without the hedge the stripe waits out
+            // the full 60s read timeout for it.
+            Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
+            Some(BitrotReader::new(
+                TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
+                SHARD_SIZE,
+                hash_algo.clone(),
+                false,
+            )),
+            Some(BitrotReader::new(
+                TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
+                SHARD_SIZE,
+                hash_algo.clone(),
+                false,
+            )),
+            Some(BitrotReader::new(
+                TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
+                SHARD_SIZE,
+                hash_algo,
+                false,
+            )),
+        ];
+
+        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+        // Reconstruction verification selects the lockstep read path.
+        let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
+            readers,
+            erasure,
+            0,
+            NUM_SHARDS * BLOCK_SIZE,
+            None,
+            vec![ShardReadCost::Unknown; 4],
+            Duration::from_secs(60),
+            true,
+        );
+        assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
+
+        let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
+            .await
+            .expect("lockstep read must hedge the slow shard instead of waiting out the 60s read timeout");
+
+        // The slow data shard was hedged: retired with a TimedOut error; the stripe
+        // reached data_shards + 1 shards from the healthy data + parity.
+        assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
+        assert!(parallel_reader.readers[0].is_none());
+        assert!(bufs[0].is_none());
+        assert!(bufs[1].is_some());
+        assert!(bufs[2].is_some());
+        assert!(bufs[3].is_some());
+        assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
+    }
+
+    /// Lockstep verification-quorum regression (backlog#1156). When a data shard is
+    /// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus
+    /// a reconstruction-verification source), never at exactly `data_shards` — that
+    /// would silently skip reconstruction verification (erasure.rs verifies only
+    /// when `available_shards > data_shards`) and could stream a corrupt
+    /// reconstruction. Here `data1` and `parity2` are instant, `data0` is dead-slow,
+    /// and the verification-source `parity3` returns *after* the hedge delay. The
+    /// read must wait for `parity3` (reaching `data_shards + 1`) rather than settle
+    /// at `data_shards` and abandon it.
+    #[tokio::test]
+    async fn test_lockstep_hedge_waits_for_verification_quorum() {
+        const NUM_SHARDS: usize = 1;
+        const BLOCK_SIZE: usize = 64;
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+
+        let hash_algo = HashAlgorithm::None;
+        // parity3 resolves 200ms in — after the 100ms hedge, but well under the
+        // 500ms test bound and the 60s read timeout.
+        let parity3_ready_at = TokioInstant::now() + Duration::from_millis(200);
+        let readers = vec![
+            Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
+            Some(BitrotReader::new(
+                TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
+                SHARD_SIZE,
+                hash_algo.clone(),
+                false,
+            )),
+            Some(BitrotReader::new(
+                TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
+                SHARD_SIZE,
+                hash_algo.clone(),
+                false,
+            )),
+            Some(BitrotReader::new(
+                TestShardReader::ReadyAt {
+                    cursor: Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS]),
+                    ready_at: parity3_ready_at,
+                    sleep: None,
+                },
+                SHARD_SIZE,
+                hash_algo,
+                false,
+            )),
+        ];
+
+        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+        let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
+            readers,
+            erasure,
+            0,
+            NUM_SHARDS * BLOCK_SIZE,
+            None,
+            vec![ShardReadCost::Unknown; 4],
+            Duration::from_secs(60),
+            true,
+        );
+        assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
+
+        let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
+            .await
+            .expect("hedge must reach data_shards + 1, not stall to the 60s read timeout");
+
+        // data0 stays missing/retired; the verification-source parity3 was NOT
+        // abandoned at the hedge — the read waited for it to reach data_shards + 1.
+        assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
+        assert!(parallel_reader.readers[0].is_none());
+        assert!(bufs[0].is_none());
+        assert!(bufs[1].is_some());
+        assert!(bufs[2].is_some());
+        assert!(bufs[3].is_some(), "verification-source parity must be used, not settled away");
+        assert!(parallel_reader.readers[3].is_some(), "verification-source parity must not be retired");
+        assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
+    }
+
    #[tokio::test]
    async fn test_parallel_reader_drains_completed_shards_after_quorum() {
        const NUM_SHARDS: usize = 1;
@@ -36,6 +36,7 @@ mod cache_value;
 mod cluster;
 mod config;
 mod core;
+mod crash_inject;
 mod data_movement;
 mod data_usage;
 mod diagnostics;
@@ -429,4 +429,19 @@ mod tests {
        assert!(!opts.user_metadata.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()));
        assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
    }
+
+    #[test]
+    fn build_transition_put_options_requests_no_checksum_and_content_md5() {
+        // Regression for rustfs/rustfs#4811: transition uploads must leave the
+        // additional-checksum modes unset and rely on Content-MD5. If `checksum`
+        // were (incorrectly) reported as set, the >128 MiB multipart put path
+        // would call `ChecksumNone.hasher()` and fail with "unsupported checksum
+        // type". Objects <=128 MiB take the single-part path and only worked by
+        // silently dropping the checksum, so pin both invariants here.
+        let opts = build_transition_put_options("COLD".to_string(), HashMap::new());
+
+        assert!(!opts.checksum.is_set(), "transition put must not request an additional checksum");
+        assert!(!opts.auto_checksum.is_set(), "transition put must not preset auto_checksum");
+        assert!(opts.send_content_md5, "transition put must send Content-MD5");
+    }
 }
@@ -1275,6 +1275,10 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
            continue;
        }

+        if files[idx].data.is_none() && disks[idx].is_none() {
+            continue;
+        }
+
        let inline_data = files[idx].data.clone();
        let disk = disks[idx].clone();
        let data_dir = files[idx].data_dir.unwrap_or_default();
@@ -3518,6 +3518,13 @@ fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::Checks
        rustfs_rio::ChecksumType::CRC32 => Some(part.checksum_crc32.clone()),
        rustfs_rio::ChecksumType::CRC32C => Some(part.checksum_crc32c.clone()),
        rustfs_rio::ChecksumType::CRC64_NVME => Some(part.checksum_crc64nvme.clone()),
+        // XXHash3/64/128 and SHA-512 (AWS 2026-04): s3s CompletePart has no typed
+        // field to carry a client-supplied per-part value in the
+        // CompleteMultipartUpload request, so accept the type with no double-check
+        // value (the part was already verified server-side at UploadPart). This
+        // mirrors the missing-value path of the five typed algorithms. Reject only
+        // genuinely unset/invalid types.
+        base if base.is_set() => Some(None),
        _ => None,
    }
 }
@@ -6718,6 +6725,23 @@ mod tests {
            ..Default::default()
        };
        assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
+
+        // AWS 2026-04 additional algorithms have no CompletePart field, so they are
+        // accepted with no double-check value (verified at UploadPart) — Some(None),
+        // NOT the outer None that would fail the multipart completion (#1261).
+        for ct in [
+            rustfs_rio::ChecksumType::XXHASH3,
+            rustfs_rio::ChecksumType::XXHASH64,
+            rustfs_rio::ChecksumType::XXHASH128,
+            rustfs_rio::ChecksumType::SHA512,
+            rustfs_rio::ChecksumType::MD5,
+        ] {
+            assert_eq!(complete_part_checksum(&CompletePart::default(), ct), Some(None), "{ct:?}");
+        }
+
+        // Genuinely unset / invalid types must still be rejected (outer None).
+        assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::NONE), None);
+        assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::INVALID), None);
    }

    fn direct_memory_test_metadata(size: i64) -> (ObjectInfo, FileInfo, ObjectOptions) {
@@ -21,10 +21,10 @@
 //! unchanged; method bodies are moved verbatim and runtime behavior is the same.

 use super::super::*;
+use crate::crash_inject::{self, CrashPoint};
 use std::future::Future;
 use std::time::Duration;
 use tokio::task::JoinSet;
-use tracing::Instrument as _;

 fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err: DiskError) -> Error {
    if err == DiskError::FileNotFound {
@@ -354,17 +354,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
        let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
        let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));

-        let ec_write_span = tracing::info_span!(
-            "rustfs.ec.write",
-            event = "rustfs_ec_write",
-            component = "ecstore",
-            "rustfs.ec.data_shards" = fi.erasure.data_blocks,
-            "rustfs.ec.parity_shards" = fi.erasure.parity_blocks,
-            "rustfs.ec.write_quorum" = write_quorum,
-            "rustfs.ec.block_bytes" = fi.erasure.block_size,
-            "rustfs.distribution.targets" = shuffle_disks.len(),
-            "rustfs.distribution.transport" = "erasure"
-        );
        let result: Result<PartInfo> = async {
            let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
            let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
@@ -448,29 +437,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
            rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
            let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);

-            let (reader, w_size) = async {
-                match write_path {
-                    SmallWritePath::SingleBlockNonInline => {
-                        Arc::new(erasure)
-                            .encode_single_block_non_inline(stream, &mut writers, write_quorum)
-                            .await
-                    }
-                    SmallWritePath::PipelineBatchedLarge => {
-                        Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await
-                    }
-                    SmallWritePath::Inline | SmallWritePath::Pipeline => {
-                        Arc::new(erasure).encode(stream, &mut writers, write_quorum).await
-                    }
+            let (reader, w_size) = match write_path {
+                SmallWritePath::SingleBlockNonInline => {
+                    Arc::new(erasure)
+                        .encode_single_block_non_inline(stream, &mut writers, write_quorum)
+                        .await?
                }
-            }
-            .instrument(tracing::info_span!(
-                "rustfs.ec.distribute",
-                event = "rustfs_ec_distribute",
-                component = "ecstore",
-                "rustfs.distribution.targets" = shuffle_disks.len(),
-                "rustfs.distribution.transport" = "erasure"
-            ))
-            .await?;
+                SmallWritePath::PipelineBatchedLarge => {
+                    Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?
+                }
+                SmallWritePath::Inline | SmallWritePath::Pipeline => {
+                    Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
+                }
+            };

            if let Some(stage_start) = encode_stage_start {
                rustfs_io_metrics::record_put_object_stage_duration(
@@ -598,7 +577,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {

            Ok(ret)
        }
-        .instrument(ec_write_span)
        .await;

        if result.is_err()
@@ -1443,6 +1421,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {

        let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);

+        // Crash-consistency injection: hard power loss after the upload is fully
+        // staged and locked but before the authoritative rename_data commit. No
+        // disk has moved the staged data, so a crash here must leave any prior
+        // committed version byte-for-byte intact (rustfs/backlog#864) and the
+        // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
+        if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, object) {
+            return Err(StorageError::Unexpected);
+        }
+
        // The trailing `_` drops the rename_data old-size backfill
        // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
        // `get_object_info` lookup, so the backfill has no consumer here yet.
@@ -1457,6 +1444,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
        )
        .await?;

+        // Crash-consistency injection: hard power loss after the authoritative
+        // rename_data commit succeeded but before the stale part.N.meta cleanup.
+        // The new version is durably committed and visible, so a crash here must
+        // leave the object readable as the new version; the un-reclaimed staging
+        // parts are swept by a retried completion or upload GC (rustfs/backlog#946).
+        // Compiles to a no-op outside `#[cfg(test)]`.
+        if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object) {
+            return Err(StorageError::Unexpected);
+        }
+
        // backlog#946: reclaim the stale per-part metadata (and any superfluous
        // part.N data files no longer in the completed set) only *after* the
        // authoritative rename_data commit above has succeeded. If rename_data
@@ -1535,37 +1532,8 @@ mod tests {
    use crate::storage_api_contracts::namespace::NamespaceLocking as _;
    use rustfs_lock::{LockClient, client::local::LocalClient};
    use serial_test::serial;
-    use std::sync::atomic::{AtomicU8, Ordering};
    use tempfile::TempDir;
    use tokio::sync::RwLock;
-    use tracing_subscriber::{Layer, Registry, layer::SubscriberExt};
-
-    const EC_WRITE_SPAN_SEEN: u8 = 1;
-    const EC_DISTRIBUTE_SPAN_SEEN: u8 = 2;
-
-    #[derive(Clone)]
-    struct EcWriteSpanLayer {
-        seen: Arc<AtomicU8>,
-    }
-
-    impl<S> Layer<S> for EcWriteSpanLayer
-    where
-        S: tracing::Subscriber,
-    {
-        fn on_new_span(
-            &self,
-            attrs: &tracing::span::Attributes<'_>,
-            _: &tracing::Id,
-            _: tracing_subscriber::layer::Context<'_, S>,
-        ) {
-            let flag = match attrs.metadata().name() {
-                "rustfs.ec.write" => EC_WRITE_SPAN_SEEN,
-                "rustfs.ec.distribute" => EC_DISTRIBUTE_SPAN_SEEN,
-                _ => return,
-            };
-            self.seen.fetch_or(flag, Ordering::Relaxed);
-        }
-    }

    async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
        let mut leftovers = Vec::new();
@@ -1794,9 +1762,6 @@ mod tests {

    #[tokio::test]
    async fn put_object_part_failure_cleans_tmp_workspace_inline() {
-        let seen = Arc::new(AtomicU8::new(0));
-        let dispatch = tracing::Dispatch::new(Registry::default().with(EcWriteSpanLayer { seen: Arc::clone(&seen) }));
-        let _guard = tracing::dispatcher::set_default(&dispatch);
        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
        let bucket = "multipart-tmp-clean-bucket";
        let object = "object";
@@ -1820,12 +1785,6 @@ mod tests {
            .await
            .expect_err("short multipart stream should fail");

-        assert_eq!(
-            seen.load(Ordering::Relaxed),
-            EC_WRITE_SPAN_SEEN | EC_DISTRIBUTE_SPAN_SEEN,
-            "multipart write must create EC write and distribution spans"
-        );
-
        let leftovers = non_trash_tmp_entries(&temp_dirs).await;
        assert!(
            leftovers.is_empty(),
@@ -2186,4 +2145,222 @@ mod tests {
        let paged: Vec<&str> = first.iter().chain(second.iter()).map(|u| u.upload_id.as_str()).collect();
        assert_eq!(paged, vec!["u0", "u1", "u2", "u3"]);
    }
+
+    /// Crash-consistency for the two `complete_multipart_upload` commit windows.
+    ///
+    /// rustfs/backlog#864: a fault that interrupts a completion must never mutate
+    /// an already-committed object; the object must read back as the whole old
+    /// version or the whole new version, never a torn mix, and the upload must
+    /// stay recoverable.
+    ///
+    /// [`CrashPoint::MultipartBeforeCommitRename`] fires after the upload is fully
+    /// staged and locked but before the authoritative `rename_data` commit: a
+    /// prior committed version survives byte-for-byte and the upload is retryable.
+    /// [`CrashPoint::MultipartAfterCommitBeforePartsCleanup`] fires after the
+    /// commit lands but before the stale `part.N.meta` cleanup: the new version is
+    /// readable and the un-reclaimed staging is harmless and reclaimable.
+    mod crash_consistency {
+        use super::*;
+        use crate::crash_inject::{self, CrashPoint};
+        use crate::storage_api_contracts::object::ObjectIO as _;
+        use http::HeaderMap;
+        use tokio::io::AsyncReadExt as _;
+
+        /// 1 MiB keeps every object off the 128 KiB inline fast path, so the
+        /// commit moves real erasure shards through `rename_data`.
+        const PART_SIZE: usize = 1 << 20;
+
+        fn payload(fill: u8) -> Vec<u8> {
+            vec![fill; PART_SIZE]
+        }
+
+        async fn make_bucket_on_all(disks: &[DiskStore], bucket: &str) {
+            for disk in disks {
+                disk.make_volume(bucket).await.expect("bucket volume should be created");
+            }
+        }
+
+        /// Stage a single-part multipart upload without completing it. A single
+        /// part is the last part, so the 5 MiB minimum-part-size gate does not
+        /// apply. Returns the upload id and the `CompletePart` list.
+        async fn stage_upload(
+            set_disks: &Arc<SetDisks>,
+            bucket: &str,
+            object: &str,
+            content: &[u8],
+        ) -> (String, Vec<CompletePart>) {
+            let upload = set_disks
+                .new_multipart_upload(bucket, object, &ObjectOptions::default())
+                .await
+                .expect("multipart upload should be created");
+            let mut reader = PutObjReader::new(
+                HashReader::from_stream(
+                    Cursor::new(content.to_vec()),
+                    content.len() as i64,
+                    content.len() as i64,
+                    None,
+                    None,
+                    false,
+                )
+                .expect("hash reader should be constructed"),
+            );
+            let part = set_disks
+                .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
+                .await
+                .expect("uploading the part should succeed");
+            (
+                upload.upload_id,
+                vec![CompletePart {
+                    part_num: part.part_num,
+                    etag: part.etag,
+                    ..Default::default()
+                }],
+            )
+        }
+
+        async fn complete(
+            set_disks: &Arc<SetDisks>,
+            bucket: &str,
+            object: &str,
+            upload_id: &str,
+            parts: Vec<CompletePart>,
+        ) -> Result<ObjectInfo> {
+            set_disks
+                .clone()
+                .complete_multipart_upload(bucket, object, upload_id, parts, &ObjectOptions::default())
+                .await
+        }
+
+        /// Read the whole committed object back; returns `(body, etag)`. The full
+        /// body read also proves the served length matches (never a torn/short body).
+        async fn read_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> (Vec<u8>, Option<String>) {
+            let mut reader = set_disks
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("object reader should open");
+            let etag = reader.object_info.etag.clone();
+            let mut body = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut body)
+                .await
+                .expect("object should stream fully");
+            (body, etag)
+        }
+
+        async fn upload_is_listed(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, upload_id: &str) -> bool {
+            let page = set_disks
+                .list_multipart_uploads(bucket, object, None, None, None, 1000)
+                .await
+                .expect("listing multipart uploads should succeed");
+            page.uploads.iter().any(|u| u.upload_id == upload_id)
+        }
+
+        #[tokio::test]
+        #[serial]
+        async fn pre_commit_crash_keeps_committed_old_version() {
+            let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+            let bucket = "multipart-crash-pre";
+            let object = "crash-pre-object";
+            make_bucket_on_all(&disk_stores, bucket).await;
+
+            // Commit an OLD version through the multipart path.
+            let old = payload(0xA1);
+            let (u_old, parts_old) = stage_upload(&set_disks, bucket, object, &old).await;
+            complete(&set_disks, bucket, object, &u_old, parts_old)
+                .await
+                .expect("the old version should commit");
+            let (old_body, old_etag) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(old_body, old, "the old version must round-trip before the crash");
+
+            // Stage a replacement upload with NEW content, then crash before commit.
+            let new = payload(0xB2);
+            let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
+            crash_inject::arm(CrashPoint::MultipartBeforeCommitRename, object);
+            let crashed = complete(&set_disks, bucket, object, &u_new, parts_new.clone()).await;
+            assert!(
+                matches!(crashed, Err(StorageError::Unexpected)),
+                "the armed pre-commit crash point must be the failure that surfaced, got {crashed:?}"
+            );
+
+            // rustfs/backlog#864: the committed old version is untouched, byte-for-byte.
+            let (after_body, after_etag) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(after_body, old, "the interrupted completion must not mutate the committed old version");
+            assert_eq!(after_etag, old_etag, "the old version's ETag must be unchanged");
+
+            // The staged upload survived intact (part.N.meta present on quorum), so
+            // a retried completion commits the new version cleanly (retryable).
+            assert!(
+                total_part1_meta(&temp_dirs).await > 0,
+                "the un-committed upload must keep its staging parts for retry"
+            );
+            crash_inject::disarm(CrashPoint::MultipartBeforeCommitRename, object);
+            complete(&set_disks, bucket, object, &u_new, parts_new)
+                .await
+                .expect("a retried completion must succeed");
+            let (retry_body, retry_etag) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(retry_body, new, "the retried completion must publish the whole new version");
+            assert_ne!(retry_etag, old_etag, "the new version must carry a distinct ETag");
+        }
+
+        #[tokio::test]
+        #[serial]
+        async fn post_commit_crash_publishes_new_version_and_leaves_reclaimable_upload() {
+            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+            let bucket = "multipart-crash-post";
+            let object = "crash-post-object";
+            make_bucket_on_all(&disk_stores, bucket).await;
+
+            // Stage an upload and crash AFTER the commit rename but BEFORE cleanup.
+            let new = payload(0xC3);
+            let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
+            let parts_retry = parts_new.clone();
+            crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
+            let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
+            assert!(
+                matches!(crashed, Err(StorageError::Unexpected)),
+                "the armed post-commit crash point must be the failure that surfaced, got {crashed:?}"
+            );
+            crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
+
+            // The commit landed: the new version reads back whole and correct.
+            let (body, _etag) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(body, new, "a post-commit crash must leave the whole new version readable");
+
+            // The completion never reached its cleanup, so the upload's staging
+            // directory is still listable — leftover that must be reclaimable.
+            assert!(
+                upload_is_listed(&set_disks, bucket, object, &u_new).await,
+                "the post-commit crash must leave the upload listable for reclamation"
+            );
+
+            // A retried CompleteMultipartUpload is answered deterministically: the
+            // commit rename already consumed the upload's metadata, so the retry
+            // resolves to InvalidUploadID (NoSuchUpload to the S3 client, the
+            // standard answer for a completed-then-retried upload) — never a torn
+            // state, and the committed object is untouched by the retry.
+            let retried = complete(&set_disks, bucket, object, &u_new, parts_retry).await;
+            assert!(
+                matches!(retried, Err(StorageError::InvalidUploadID(..))),
+                "a retried complete after the commit landed must resolve to InvalidUploadID, got {retried:?}"
+            );
+            let (body_after_retry, _) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(body_after_retry, new, "the failed retry must not disturb the committed object");
+
+            // Reclaim the leftover exactly as the production tail does: delete_all
+            // on the upload path (abort_multipart_upload cannot — the upload's
+            // metadata is gone, so it too answers InvalidUploadID).
+            let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
+            set_disks
+                .delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
+                .await
+                .expect("sweeping the leftover upload staging should succeed");
+            assert!(
+                !upload_is_listed(&set_disks, bucket, object, &u_new).await,
+                "reclaiming the upload must drop it from the listing"
+            );
+            let (body_after, _) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
+        }
+    }
 }
@@ -23,7 +23,6 @@ use super::super::*;

 use crate::disk::OldCurrentSize;
 use crate::object_api::GetObjectBodySource;
-use tracing::Instrument as _;

 /// Length of the full plaintext body when — and only when — this read's output
 /// is exactly the object's complete plaintext, so the app-layer body cache may
@@ -741,17 +740,6 @@ impl SetDisks {

        let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());

-        let ec_write_span = tracing::info_span!(
-            "rustfs.ec.write",
-            event = "rustfs_ec_write",
-            component = "ecstore",
-            "rustfs.ec.data_shards" = data_drives,
-            "rustfs.ec.parity_shards" = parity_drives,
-            "rustfs.ec.write_quorum" = write_quorum,
-            "rustfs.ec.block_bytes" = fi.erasure.block_size,
-            "rustfs.distribution.targets" = shuffle_disks.len(),
-            "rustfs.distribution.transport" = "erasure"
-        );
        let result: Result<(ObjectInfo, Option<OldCurrentSize>)> = async {
            let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);

@@ -801,15 +789,7 @@ impl SetDisks {
                    }
                })
                .collect();
-            let writer_results = join_all(writer_futs)
-                .instrument(tracing::info_span!(
-                    "rustfs.ec.distribute",
-                    event = "rustfs_ec_distribute",
-                    component = "ecstore",
-                    "rustfs.distribution.targets" = shuffle_disks.len(),
-                    "rustfs.distribution.transport" = "erasure"
-                ))
-                .await;
+            let writer_results = join_all(writer_futs).await;
            let mut writers = Vec::with_capacity(writer_results.len());
            let mut errors = Vec::with_capacity(writer_results.len());
            for (w, e) in writer_results {
@@ -1159,7 +1139,6 @@ impl SetDisks {
                old_current_size,
            ))
        }
-        .instrument(ec_write_span)
        .await;

        if issue3031_diag_enabled()
@@ -4095,6 +4095,27 @@ mod tests {
        assert_eq!(&out[..n], [b"aaaa", b"bbbb", b"cccc", b"dddd"][fallback_index]);
    }

+    #[tokio::test]
+    async fn bitrot_reader_setup_skips_deferred_slots_without_a_source() {
+        let setup = setup_inline_bitrot_readers_with_env(
+            vec![Some(b"aaaa"), Some(b"bbbb"), None, Some(b"dddd")],
+            2,
+            2,
+            BitrotReaderSetupMode::ReadQuorum,
+            true,
+        )
+        .await;
+
+        assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum));
+        assert_eq!(setup.available_shards(), 2);
+        assert_eq!(setup.deferred_shards(), 1);
+        assert!(setup.readers[2].is_none(), "a slot without inline data or a disk has no usable source");
+        assert!(matches!(setup.errors[2], Some(DiskError::DiskNotFound)));
+        assert!(setup.deferred_stripe_handles[2].is_none());
+        assert!(setup.readers[3].is_some(), "an inline shard remains available as a deferred fallback");
+        assert!(setup.deferred_stripe_handles[3].is_some());
+    }
+
    #[tokio::test]
    async fn bitrot_reader_setup_preference_uses_data_blocks_first_without_env() {
        let setup = setup_inline_bitrot_readers_with_preference(
@@ -1308,11 +1308,19 @@ impl ECStore {
        opts: &ObjectOptions,
    ) -> Result<()> {
        let object = encode_dir_object(object);
+        let mut opts = opts.clone();
+        let _object_lock_guard = self
+            .acquire_object_write_lock_if_needed("restore_transitioned_object", bucket, &object, &mut opts)
+            .await?;
+
        if self.single_pool() {
-            return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await;
+            return self.pools[0]
+                .clone()
+                .restore_transitioned_object(bucket, &object, &opts)
+                .await;
        }

-        let opts = transition_restore_pool_opts(opts);
+        let opts = transition_restore_pool_opts(&opts);
        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
            .await?;
@@ -2171,6 +2179,31 @@ mod tests {
        );
    }

+    #[tokio::test]
+    #[serial_test::serial]
+    async fn restore_transitioned_object_waits_for_existing_reader() {
+        let store = Arc::new(new_read_lock_test_store().await);
+        let ns_lock = store
+            .handle_new_ns_lock("bucket", "object")
+            .await
+            .expect("namespace lock should be created");
+        let _reader = ns_lock
+            .get_read_lock(Duration::from_secs(1))
+            .await
+            .expect("reader should acquire the object lock");
+
+        let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
+            store
+                .clone()
+                .handle_restore_transitioned_object("bucket", "object", &ObjectOptions::default())
+                .await
+                .expect_err("restore must wait behind an existing reader")
+        })
+        .await;
+
+        assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
+    }
+
    #[tokio::test]
    #[serial_test::serial]
    async fn reader_lock_is_held_when_optimization_is_disabled() {
@@ -280,14 +280,20 @@ async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {

 async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
    let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
-    let object_size = object_info.size;
+    // `ObjectInfo.size` is the on-disk size. For SSE objects that is the
+    // DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
+    // deliberately larger than the logical object size. The size a client sees
+    // (and what MinIO records via `x-*-internal-actual-size`) comes from
+    // `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
+    // `size` field would never equal the plaintext length for encrypted objects.
+    let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
    let kms_key_b64 = minio_static_kms_key_b64();

    let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
        .await
        .expect("fixture must restore with the configured KMS key");

-    assert_eq!(object_size, expected_size);
+    assert_eq!(decrypted_size, expected_size);
    assert_eq!(plaintext.len(), expected_size as usize);
    assert_eq!(sha256_hex(&plaintext), expected_sha256);
 }
@@ -28,11 +28,11 @@ categories = ["web-programming", "development-tools"]
 doctest = false

 [dependencies]
-serde.workspace = true
+serde = { workspace = true, features = ["derive"] }
 thiserror.workspace = true

 [dev-dependencies]
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }

 [lints]
 workspace = true
@@ -34,22 +34,22 @@ hotpath = { workspace = true, optional = true }
 crc-fast = { workspace = true }
 rmp.workspace = true
 rmp-serde.workspace = true
-serde.workspace = true
-time.workspace = true
-uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
-tokio = { workspace = true, features = ["io-util", "macros", "sync"] }
-xxhash-rust = { workspace = true, features = ["xxh64"] }
-bytes.workspace = true
+serde = { workspace = true, features = ["derive"] }
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
+uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
+tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] }
+xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
+bytes = { workspace = true, features = ["serde"] }
 rustfs-utils = { workspace = true, features = ["hash", "http"] }
 byteorder = { workspace = true }
 tracing.workspace = true
 thiserror.workspace = true
-s3s.workspace = true
+s3s = { workspace = true, features = ["minio"] }
 regex.workspace = true
 arc-swap.workspace = true

 [dev-dependencies]
-criterion = { workspace = true }
+criterion = { workspace = true, features = ["html_reports"] }
 tempfile = { workspace = true }
 proptest = "1"

@@ -0,0 +1,209 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Structured version-graph marshal -> load roundtrip property (backlog#1151
+//! sec-10).
+//!
+//! xl.meta roundtrip coverage was a fixed example test plus byte-level no-panic
+//! fuzz properties; nothing generated STRUCTURED version graphs, so a lossy
+//! encoding bug in the version header, ordering, delete-marker, or
+//! dual-internal-metadata-key handling would only be caught if the fixed
+//! example happened to hit it. These properties generate multi-version graphs
+//! (objects + delete markers, distinct and colliding mod_times, user metadata,
+//! and the AGENTS.md dual `x-rustfs-internal-*` / `x-minio-internal-*` keys)
+//! and assert `FileMeta::marshal_msg` -> `FileMeta::load` preserves them
+//! exactly: version count, order, headers, and fully decoded per-version
+//! content.
+
+use proptest::collection::vec;
+use proptest::prelude::*;
+use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMeta, FileMetaVersion, MetaDeleteMarker, MetaObject, VersionType};
+use rustfs_utils::http::insert_bytes;
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+/// Internal metadata suffixes to write under BOTH internal prefixes (the
+/// cross-cutting dual-key invariant). Realistic suffixes used by production
+/// code paths.
+const SYS_SUFFIXES: &[&str] = &["replication-status", "transition-status", "tier-name"];
+
+#[derive(Debug, Clone)]
+struct GenVersion {
+    is_delete_marker: bool,
+    version_id: u128,
+    /// Seconds offset added to the base timestamp. Small range on purpose so
+    /// some versions share a mod_time and the tie-break ordering is exercised.
+    mod_offset: i64,
+    size: i64,
+    user_meta: Vec<(String, String)>,
+    sys_suffix_idx: usize,
+    sys_value: Vec<u8>,
+}
+
+fn gen_version_strategy() -> impl Strategy<Value = GenVersion> {
+    (
+        proptest::bool::weighted(0.3),
+        1u128..u128::MAX,
+        0i64..8,
+        1i64..(1 << 30),
+        vec(("[a-z]{1,8}", "[a-zA-Z0-9 ]{0,16}"), 0..3),
+        0usize..SYS_SUFFIXES.len(),
+        vec(any::<u8>(), 1..24),
+    )
+        .prop_map(
+            |(is_delete_marker, version_id, mod_offset, size, user_meta, sys_suffix_idx, sys_value)| GenVersion {
+                is_delete_marker,
+                version_id,
+                mod_offset,
+                size,
+                user_meta,
+                sys_suffix_idx,
+                sys_value,
+            },
+        )
+}
+
+const BASE_TS: i64 = 1_700_000_000;
+
+fn build_version(g: &GenVersion) -> FileMetaVersion {
+    let mod_time = OffsetDateTime::from_unix_timestamp(BASE_TS + g.mod_offset).expect("valid timestamp");
+    let mut meta_sys = HashMap::new();
+    // Dual internal metadata keys: insert_bytes writes BOTH the
+    // x-rustfs-internal-* and x-minio-internal-* forms.
+    insert_bytes(&mut meta_sys, SYS_SUFFIXES[g.sys_suffix_idx], g.sys_value.clone());
+
+    if g.is_delete_marker {
+        FileMetaVersion {
+            version_type: VersionType::Delete,
+            delete_marker: Some(MetaDeleteMarker {
+                version_id: Some(Uuid::from_u128(g.version_id)),
+                mod_time: Some(mod_time),
+                meta_sys,
+            }),
+            write_version: 0,
+            ..Default::default()
+        }
+    } else {
+        FileMetaVersion {
+            version_type: VersionType::Object,
+            object: Some(MetaObject {
+                version_id: Some(Uuid::from_u128(g.version_id)),
+                data_dir: Some(Uuid::from_u128(g.version_id.wrapping_add(1).max(1))),
+                erasure_algorithm: ErasureAlgo::ReedSolomon,
+                erasure_m: 2,
+                erasure_n: 2,
+                erasure_block_size: 1 << 20,
+                erasure_index: 1,
+                erasure_dist: vec![1, 2, 3, 4],
+                bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
+                part_numbers: vec![1],
+                part_etags: vec![],
+                part_sizes: vec![g.size as usize],
+                part_actual_sizes: vec![g.size],
+                part_indices: vec![],
+                size: g.size,
+                mod_time: Some(mod_time),
+                meta_sys,
+                meta_user: g.user_meta.iter().cloned().collect(),
+            }),
+            write_version: 0,
+            ..Default::default()
+        }
+    }
+}
+
+proptest! {
+    /// A generated version graph survives marshal -> load exactly: count,
+    /// order, headers, and fully decoded version content (objects, delete
+    /// markers, user metadata, and both internal metadata keys).
+    #[test]
+    fn version_graph_marshal_load_roundtrip(gens in vec(gen_version_strategy(), 1..8)) {
+        let mut fm = FileMeta::new();
+        for g in &gens {
+            fm.add_version_filemata(build_version(g)).expect("add generated version");
+        }
+
+        let encoded = fm.marshal_msg().expect("marshal generated version graph");
+        let loaded = FileMeta::load(&encoded).expect("load marshaled version graph");
+
+        // Count and metadata version survive.
+        prop_assert_eq!(loaded.versions.len(), fm.versions.len(), "version count must survive the roundtrip");
+        prop_assert_eq!(loaded.meta_ver, fm.meta_ver, "meta_ver must survive the roundtrip");
+
+        // Version ORDER survives exactly: add_version_filemata sorted the
+        // in-memory graph (mod_time desc with documented tie-breaks, delete
+        // markers interleaved); load must reproduce that same sequence, not
+        // re-derive a different one.
+        let pre: Vec<_> = fm.versions.iter().map(|v| (v.header.version_id, v.header.version_type.clone())).collect();
+        let post: Vec<_> = loaded.versions.iter().map(|v| (v.header.version_id, v.header.version_type.clone())).collect();
+        prop_assert_eq!(pre, post, "version (id, type) sequence must survive the roundtrip");
+
+        for (orig, back) in fm.versions.iter().zip(loaded.versions.iter()) {
+            // Full header equality (mod_time, flags, signature, ec fields).
+            prop_assert_eq!(&orig.header, &back.header, "version header must survive the roundtrip");
+
+            // Fully decoded version content equality.
+            let orig_v = orig.parse_version_meta().expect("decode original version");
+            let back_v = back.parse_version_meta().expect("decode roundtripped version");
+            prop_assert_eq!(&orig_v, &back_v, "decoded version content must survive the roundtrip");
+
+            // Delete-marker semantics preserved.
+            if back.header.version_type == VersionType::Delete {
+                prop_assert!(back_v.delete_marker.is_some(), "delete marker payload must survive");
+                prop_assert!(back_v.object.is_none(), "a delete marker must not grow an object payload");
+            } else {
+                prop_assert!(back_v.object.is_some(), "object payload must survive");
+            }
+
+            // Dual internal metadata keys: BOTH prefixed forms survive with the
+            // same value (losing either breaks MinIO interop; losing both loses
+            // replication/tier state).
+            let meta_sys = match (&back_v.object, &back_v.delete_marker) {
+                (Some(o), _) => &o.meta_sys,
+                (_, Some(d)) => &d.meta_sys,
+                _ => unreachable!("version decoded above"),
+            };
+            let orig_sys = match (&orig_v.object, &orig_v.delete_marker) {
+                (Some(o), _) => &o.meta_sys,
+                (_, Some(d)) => &d.meta_sys,
+                _ => unreachable!("version decoded above"),
+            };
+            for (k, v) in orig_sys {
+                prop_assert_eq!(
+                    meta_sys.get(k).map(|b| b.as_slice()),
+                    Some(v.as_slice()),
+                    "internal metadata key {} must survive the roundtrip with an identical value",
+                    k
+                );
+            }
+        }
+    }
+
+    /// Idempotence: a second marshal of the loaded graph is byte-identical.
+    /// Catches encoders that mutate or reorder state on the way out (a lossy
+    /// first pass shows up as a divergent second pass).
+    #[test]
+    fn version_graph_marshal_is_idempotent(gens in vec(gen_version_strategy(), 1..6)) {
+        let mut fm = FileMeta::new();
+        for g in &gens {
+            fm.add_version_filemata(build_version(g)).expect("add generated version");
+        }
+
+        let first = fm.marshal_msg().expect("first marshal");
+        let loaded = FileMeta::load(&first).expect("load first marshal");
+        let second = loaded.marshal_msg().expect("second marshal");
+        prop_assert_eq!(first, second, "marshal(load(marshal(x))) must be byte-identical to marshal(x)");
+    }
+}
@@ -34,27 +34,27 @@ rustfs-storage-api = { workspace = true }
 rustfs-common = { workspace = true }
 rustfs-madmin = { workspace = true }
 rustfs-utils = { workspace = true }
-tokio = { workspace = true, features = ["sync","io-util","time","macros"] }
-tokio-util = { workspace = true }
+tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
+tokio-util = { workspace = true, features = ["io", "compat"] }
 tracing = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
-serde_json = { workspace = true }
+serde_json = { workspace = true, features = ["raw_value"] }
 thiserror = { workspace = true }
-uuid = { workspace = true, features = ["v4", "serde"] }
+uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
 async-trait = { workspace = true }
 futures = { workspace = true }
 metrics = { workspace = true }
 base64 = { workspace = true }

 [dev-dependencies]
-serde_json = { workspace = true }
+serde_json = { workspace = true, features = ["raw_value"] }
+rustfs-test-utils = { workspace = true }
 serial_test = { workspace = true }
-tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 http = { workspace = true }
 temp-env = { workspace = true }
-tokio = { workspace = true, features = ["test-util","fs"] }
+tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }

 [lib]
 doctest = false
@@ -99,6 +99,54 @@ impl Error {
    pub fn transient_skip(message: impl Into<String>) -> Self {
        Error::TransientSkip { message: message.into() }
    }
+
+    /// Whether a heal operation can be retried without changing its inputs.
+    pub(crate) fn is_recoverable_heal(&self) -> bool {
+        match self {
+            Error::TaskCancelled => false,
+            Error::TaskTimeout | Error::TransientSkip { .. } => true,
+            Error::Storage(err) => {
+                err.is_quorum_error()
+                    || matches!(err, EcstoreError::SlowDown | EcstoreError::OperationCanceled | EcstoreError::Lock(_))
+                    || is_recoverable_heal_error_message(&err.to_string())
+            }
+            Error::Disk(err) => {
+                matches!(
+                    err,
+                    DiskError::ErasureReadQuorum
+                        | DiskError::ErasureWriteQuorum
+                        | DiskError::Timeout
+                        | DiskError::SourceStalled
+                        | DiskError::FaultyRemoteDisk
+                        | DiskError::FaultyDisk
+                ) || is_recoverable_heal_error_message(&err.to_string())
+            }
+            Error::TaskExecutionFailed { message } | Error::IO(message) | Error::Other(message) => {
+                is_recoverable_heal_error_message(message)
+            }
+            Error::Io(err) => is_recoverable_heal_error_message(&err.to_string()),
+            _ => false,
+        }
+    }
+}
+
+fn is_recoverable_heal_error_message(error: &str) -> bool {
+    let error = error.to_ascii_lowercase();
+    [
+        "failed to acquire read lock",
+        "lock acquisition failed",
+        "lock acquisition timeout",
+        "remote lock rpc timed out",
+        "deadline has elapsed",
+        "timed out",
+        "transport error",
+        "network error",
+        "connection refused",
+        "operation canceled",
+        "quorum not reached",
+    ]
+    .iter()
+    .any(|pattern| error.contains(pattern))
 }

 impl From<Error> for std::io::Error {
@@ -34,7 +34,7 @@ use tokio::{
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};

-use super::{DiskError, EcstoreError, HealDiskExt as _, local_disk_map_read};
+use super::{DiskError, HealDiskExt as _, local_disk_map_read};

 const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
 const LOG_COMPONENT_HEAL: &str = "heal";
@@ -565,9 +565,12 @@ fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(Hea
    if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
        return None;
    }
+    if task.has_batch_failure() {
+        return None;
+    }

    let error = err.to_string();
-    if !is_recoverable_heal_error(err, &error) {
+    if !err.is_recoverable_heal() {
        return None;
    }

@@ -582,54 +585,6 @@ fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
    delay.min(MAX_RECOVERABLE_HEAL_RETRY_DELAY)
 }

-fn is_recoverable_heal_error(err: &Error, error: &str) -> bool {
-    match err {
-        Error::TaskCancelled => false,
-        Error::TaskTimeout | Error::TransientSkip { .. } => true,
-        Error::Storage(err) => is_recoverable_storage_heal_error(err) || is_recoverable_heal_error_message(error),
-        Error::Disk(err) => is_recoverable_disk_heal_error(err) || is_recoverable_heal_error_message(error),
-        Error::TaskExecutionFailed { .. } | Error::Io(_) | Error::IO(_) | Error::Other(_) => {
-            is_recoverable_heal_error_message(error)
-        }
-        _ => false,
-    }
-}
-
-fn is_recoverable_storage_heal_error(err: &EcstoreError) -> bool {
-    err.is_quorum_error() || matches!(err, EcstoreError::SlowDown | EcstoreError::OperationCanceled | EcstoreError::Lock(_))
-}
-
-fn is_recoverable_disk_heal_error(err: &DiskError) -> bool {
-    matches!(
-        err,
-        DiskError::ErasureReadQuorum
-            | DiskError::ErasureWriteQuorum
-            | DiskError::Timeout
-            | DiskError::SourceStalled
-            | DiskError::FaultyRemoteDisk
-            | DiskError::FaultyDisk
-    )
-}
-
-fn is_recoverable_heal_error_message(error: &str) -> bool {
-    let error = error.to_ascii_lowercase();
-    [
-        "failed to acquire read lock",
-        "lock acquisition failed",
-        "lock acquisition timeout",
-        "remote lock rpc timed out",
-        "deadline has elapsed",
-        "timed out",
-        "transport error",
-        "network error",
-        "connection refused",
-        "operation canceled",
-        "quorum not reached",
-    ]
-    .iter()
-    .any(|pattern| error.contains(pattern))
-}
-
 /// Heal config
 #[derive(Debug, Clone)]
 pub struct HealConfig {
@@ -2917,8 +2872,9 @@ fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String,
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::heal::EcstoreError;
    use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
-    use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
+    use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
    use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
    use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
    use rustfs_madmin::heal_commands::HealResultItem;
@@ -3678,6 +3634,24 @@ mod tests {
        assert!(retry_request_for_result(&task, &result).is_none());
    }

+    #[tokio::test]
+    async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
+        let result = Err(task
+            .record_batch_failure(BatchHealFailure {
+                scope: "bucket:bucket".to_string(),
+                failed: 1,
+                retryable: 1,
+                permanent: 0,
+                first_object: "object".to_string(),
+                first_error: "Lock acquisition timeout".to_string(),
+            })
+            .await);
+
+        assert!(retry_request_for_result(&task, &result).is_none());
+    }
+
    #[test]
    fn test_heal_type_matches_path_normalizes_prefix_trailing_slash() {
        let heal_type = HealType::Prefix {
@@ -25,7 +25,10 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use serde::{Deserialize, Serialize};
 use std::{
    future::Future,
-    sync::Arc,
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    },
    time::{Duration, Instant, SystemTime},
 };
 use tokio::sync::RwLock;
@@ -41,6 +44,8 @@ const EVENT_HEAL_TASK_STATE: &str = "heal_task_state";
 const EVENT_HEAL_OBJECT_STAGE: &str = "heal_object_stage";
 const EVENT_HEAL_OBJECT_MISSING: &str = "heal_object_missing";
 const EVENT_HEAL_OBJECT_RESULT: &str = "heal_object_result";
+const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3;
+const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5;
 const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage";
 const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result";
 const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage";
@@ -181,6 +186,26 @@ pub enum HealTaskStatus {
    Timeout,
 }

+#[derive(Debug)]
+pub(crate) struct BatchHealFailure {
+    pub(crate) scope: String,
+    pub(crate) failed: u64,
+    pub(crate) retryable: u64,
+    pub(crate) permanent: u64,
+    pub(crate) first_object: String,
+    pub(crate) first_error: String,
+}
+
+impl std::fmt::Display for BatchHealFailure {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Heal batch failed for {}: {} failed ({} retryable, {} permanent); first failure at {}: {}",
+            self.scope, self.failed, self.retryable, self.permanent, self.first_object, self.first_error
+        )
+    }
+}
+
 /// Heal request
 #[derive(Debug, Clone)]
 pub struct HealRequest {
@@ -281,6 +306,8 @@ pub struct HealTask {
    pub progress: Arc<RwLock<HealProgress>>,
    /// Result items collected from storage heal calls.
    pub result_items: Arc<RwLock<Vec<HealResultItem>>>,
+    batch_failure: Arc<RwLock<Option<BatchHealFailure>>>,
+    batch_failure_recorded: Arc<AtomicBool>,
    /// Created time
    pub created_at: SystemTime,
    /// Queue admission time
@@ -310,6 +337,8 @@ impl HealTask {
            status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
            progress: Arc::new(RwLock::new(HealProgress::new())),
            result_items: Arc::new(RwLock::new(Vec::new())),
+            batch_failure: Arc::new(RwLock::new(None)),
+            batch_failure_recorded: Arc::new(AtomicBool::new(false)),
            created_at: request.created_at,
            enqueued_at: request.enqueued_at,
            started_at: Arc::new(RwLock::new(None)),
@@ -348,6 +377,21 @@ impl HealTask {
        }
    }

+    pub(crate) fn has_batch_failure(&self) -> bool {
+        self.batch_failure_recorded.load(Ordering::Acquire)
+    }
+
+    pub(crate) async fn record_batch_failure(&self, failure: BatchHealFailure) -> Error {
+        self.batch_failure_recorded.store(true, Ordering::Release);
+        let message = failure.to_string();
+        *self.batch_failure.write().await = Some(failure);
+        Error::TaskExecutionFailed { message }
+    }
+
+    async fn take_batch_failure(&self) -> Option<BatchHealFailure> {
+        self.batch_failure.write().await.take()
+    }
+
    pub fn metric_set_label(&self) -> String {
        match &self.heal_type {
            HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
@@ -477,6 +521,15 @@ impl HealTask {
        }
    }

+    fn bucket_object_retry_delay(&self, retry_attempt: u32) -> Duration {
+        let base = Duration::from_secs(2_u64.saturating_pow(retry_attempt.clamp(1, MAX_BUCKET_OBJECT_HEAL_RETRIES)));
+        let jitter_seed = self
+            .id
+            .bytes()
+            .fold(0_u64, |acc, byte| acc.wrapping_mul(31).wrapping_add(u64::from(byte)));
+        Duration::from_millis(jitter_seed % 500).saturating_add(base)
+    }
+
    fn should_return_typed_heal_error(err: &Error) -> bool {
        matches!(err, Error::Storage(_) | Error::Disk(_))
    }
@@ -1263,9 +1316,61 @@ impl HealTask {
        );

        let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
+        let mut failed = 0_u64;
+        let mut retryable = 0_u64;
+        let mut permanent = 0_u64;
+        let mut first_object = None;
+        let mut first_error = None;
        for bucket_info in bucket_infos {
            self.check_control_flags().await?;
-            self.heal_bucket(&bucket_info.name).await?;
+            let mut retry_attempt = 0_u32;
+            loop {
+                match self.heal_bucket(&bucket_info.name).await {
+                    Ok(()) => break,
+                    Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
+                    Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
+                    Err(err) => {
+                        if let Some(failure) = self.take_batch_failure().await {
+                            failed = failed.saturating_add(failure.failed);
+                            retryable = retryable.saturating_add(failure.retryable);
+                            permanent = permanent.saturating_add(failure.permanent);
+                            first_object.get_or_insert(failure.first_object);
+                            first_error.get_or_insert(failure.first_error);
+                            break;
+                        }
+                        if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                            retry_attempt = retry_attempt.saturating_add(1);
+                            self.await_with_control(async {
+                                tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
+                                Ok(())
+                            })
+                            .await?;
+                            continue;
+                        }
+                        failed = failed.saturating_add(1);
+                        if err.is_recoverable_heal() {
+                            retryable = retryable.saturating_add(1);
+                        } else {
+                            permanent = permanent.saturating_add(1);
+                        }
+                        first_object.get_or_insert(bucket_info.name.clone());
+                        first_error.get_or_insert_with(|| err.to_string());
+                        break;
+                    }
+                }
+            }
+        }
+
+        if failed > 0 {
+            let failure = BatchHealFailure {
+                scope: "cluster".to_string(),
+                failed,
+                retryable,
+                permanent,
+                first_object: first_object.unwrap_or_default(),
+                first_error: first_error.unwrap_or_default(),
+            };
+            return Err(self.record_batch_failure(failure).await);
        }

        Ok(())
@@ -1292,7 +1397,12 @@ impl HealTask {
        let mut scanned = 0u64;
        let mut healed = 0u64;
        let mut failed = 0u64;
+        let mut retryable_failed = 0u64;
+        let mut permanent_failed = 0u64;
        let mut bytes = 0u64;
+        let mut first_failed_object = None;
+        let mut first_error = None;
+        let mut failure_samples_logged = 0_u64;

        let heal_opts = HealOpts {
            recursive: false,
@@ -1315,34 +1425,44 @@ impl HealTask {
                )
                .await?;

-            for item in objects {
-                self.check_control_flags().await?;
-                let object = item.name.as_str();
-                scanned += 1;
-                {
-                    let mut progress = self.progress.write().await;
-                    progress.set_current_object(Some(format!("{bucket}/{object}")));
-                    progress.update_progress(scanned, healed, failed, bytes);
+            let mut pending = objects;
+            let mut retry_attempt = 0_u32;
+            while !pending.is_empty() {
+                if retry_attempt > 0 {
+                    self.await_with_control(async {
+                        tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
+                        Ok(())
+                    })
+                    .await?;
                }
-
-                // Heal each enumerated version directly. There is no latest-only
-                // existence gate: genuine absence surfaces through heal_object's
-                // not-found result and is handled below.
-                match self
-                    .await_with_control(
-                        self.storage
-                            .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
-                    )
-                    .await
-                {
-                    Ok((result, None)) => {
-                        healed += 1;
-                        bytes = bytes.saturating_add(result.object_size as u64);
-                        self.record_result_item(result).await;
+                let mut retry = Vec::with_capacity(pending.len());
+                for item in pending {
+                    self.check_control_flags().await?;
+                    let object = item.name.as_str();
+                    if retry_attempt == 0 {
+                        scanned = scanned.saturating_add(1);
                    }
-                    Ok((_, Some(err))) => {
-                        if is_missing_object_dir_heal_result(object, &err) {
-                            healed += 1;
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.set_current_object(Some(format!("{bucket}/{object}")));
+                        progress.update_progress(scanned, healed, failed, bytes);
+                    }
+
+                    let error = match self
+                        .await_with_control(
+                            self.storage
+                                .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
+                        )
+                        .await
+                    {
+                        Ok((result, None)) => {
+                            healed = healed.saturating_add(1);
+                            bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
+                            self.record_result_item(result).await;
+                            None
+                        }
+                        Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
+                            healed = healed.saturating_add(1);
                            debug!(
                                target: "rustfs::heal::task",
                                event = EVENT_HEAL_BUCKET_RESULT,
@@ -1350,77 +1470,77 @@ impl HealTask {
                                subsystem = LOG_SUBSYSTEM_TASK,
                                task_id = %self.id,
                                bucket,
-                                object = %object,
+                                object,
                                result = "object_dir_not_found_skipped",
                                "Heal bucket object-dir candidate skipped after not-found result"
                            );
-                            continue;
+                            None
                        }
-                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
-                            warn!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object = %object,
-                                result = "transient_skip",
-                                error = %err,
-                                "Heal bucket object repair skipped due to transient error"
-                            );
-                        } else {
-                            failed += 1;
-                            warn!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object = %object,
-                                result = "object_failed",
-                                error = %err,
-                                "Heal bucket object repair failed"
-                            );
-                        }
-                    }
-                    Err(err) => {
-                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
-                            warn!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object = %object,
-                                result = "transient_skip",
-                                error = %err,
-                                "Heal bucket object repair skipped due to transient error"
-                            );
-                        } else {
-                            failed += 1;
-                            warn!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object = %object,
-                                result = "object_failed",
-                                error = %err,
-                                "Heal bucket object repair failed"
-                            );
-                        }
-                    }
-                }
+                        Ok((_, Some(err))) | Err(err) => Some(err),
+                    };
+
+                    if let Some(err) = error {
+                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                            warn!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object,
+                                result = "transient_skip",
+                                error = %err,
+                                "Heal bucket object repair skipped due to transient metadata error"
+                            );
+                        } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                            debug!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object,
+                                retry_attempt = retry_attempt.saturating_add(1),
+                                error = %err,
+                                result = "object_retry_scheduled",
+                                "Heal bucket object retry scheduled"
+                            );
+                            retry.push(item);
+                        } else {
+                            failed = failed.saturating_add(1);
+                            if err.is_recoverable_heal() {
+                                retryable_failed = retryable_failed.saturating_add(1);
+                            } else {
+                                permanent_failed = permanent_failed.saturating_add(1);
+                            }
+                            first_failed_object.get_or_insert_with(|| object.to_string());
+                            first_error.get_or_insert_with(|| err.to_string());
+                            if failure_samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
+                                failure_samples_logged = failure_samples_logged.saturating_add(1);
+                                warn!(
+                                    target: "rustfs::heal::task",
+                                    event = EVENT_HEAL_BUCKET_RESULT,
+                                    component = LOG_COMPONENT_HEAL,
+                                    subsystem = LOG_SUBSYSTEM_TASK,
+                                    task_id = %self.id,
+                                    bucket,
+                                    object,
+                                    retry_attempt,
+                                    error = %err,
+                                    result = "object_failed",
+                                    "Heal bucket object repair failed"
+                                );
+                            }
+                        }
+                    }

-                {
                    let mut progress = self.progress.write().await;
                    progress.update_progress(scanned, healed, failed, bytes);
                }
+                pending = retry;
+                retry_attempt = retry_attempt.saturating_add(1);
            }

            if !is_truncated {
@@ -1435,9 +1555,15 @@ impl HealTask {
        }

        if failed > 0 {
-            return Err(Error::TaskExecutionFailed {
-                message: format!("Failed to heal {failed} object(s) in bucket {bucket}"),
-            });
+            let failure = BatchHealFailure {
+                scope: format!("bucket:{bucket}"),
+                failed,
+                retryable: retryable_failed,
+                permanent: permanent_failed,
+                first_object: first_failed_object.unwrap_or_default(),
+                first_error: first_error.unwrap_or_default(),
+            };
+            return Err(self.record_batch_failure(failure).await);
        }

        debug!(
@@ -2223,7 +2349,7 @@ mod tests {
    use super::*;
    use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo};
    use rustfs_madmin::heal_commands::HealResultItem;
-    use std::collections::HashMap;
+    use std::collections::{HashMap, VecDeque};
    use std::sync::Mutex;

    use super::super::storage_api::status::BucketInfo;
@@ -2238,11 +2364,15 @@ mod tests {
        object_exists: Mutex<Option<bool>>,
        object_exists_by_name: Mutex<HashMap<String, MockObjectExists>>,
        heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
+        heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
        deleted_objects: Mutex<Vec<String>>,
        format_no_heal_required: Mutex<bool>,
        listed_prefixes: Mutex<Vec<String>>,
        truncate_without_token: Mutex<bool>,
        include_object_dir_candidate: Mutex<bool>,
+        listed_buckets: Mutex<Option<Vec<String>>>,
+        bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
+        bucket_heal_calls: Mutex<Vec<String>>,
    }

    /// Build a latest, non-delete-marker heal list item with no version id.
@@ -2257,6 +2387,8 @@ mod tests {
    enum MockHealObjectOutcome {
        OkWithOtherError(&'static str),
        ErrOther(&'static str),
+        RetryableReadQuorum,
+        PermanentOther(&'static str),
    }

    #[derive(Clone, Copy)]
@@ -2326,10 +2458,19 @@ mod tests {
        }

        async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
-            Ok(vec![BucketInfo {
-                name: "bucket-a".to_string(),
-                ..Default::default()
-            }])
+            let buckets = self
+                .listed_buckets
+                .lock()
+                .unwrap()
+                .clone()
+                .unwrap_or_else(|| vec!["bucket-a".to_string()]);
+            Ok(buckets
+                .into_iter()
+                .map(|name| BucketInfo {
+                    name,
+                    ..Default::default()
+                })
+                .collect())
        }

        async fn object_exists(&self, _bucket: &str, object: &str) -> Result<bool> {
@@ -2364,12 +2505,37 @@ mod tests {
                .unwrap()
                .push(version_id.map(ToString::to_string));
            self.object_heal_opts.lock().unwrap().push(*opts);
+            if let Some(outcome) = self
+                .heal_object_outcomes
+                .lock()
+                .unwrap()
+                .get_mut(object)
+                .and_then(VecDeque::pop_front)
+            {
+                return match outcome {
+                    MockHealObjectOutcome::RetryableReadQuorum => Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
+                        bucket.to_string(),
+                        object.to_string(),
+                    ))),
+                    MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
+                    MockHealObjectOutcome::OkWithOtherError(message) => {
+                        Ok((HealResultItem::default(), Some(Error::other(message))))
+                    }
+                    MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
+                };
+            }
            if let Some(outcome) = self.heal_object_outcome.lock().unwrap().take() {
                return match outcome {
                    MockHealObjectOutcome::OkWithOtherError(message) => {
                        Ok((HealResultItem::default(), Some(Error::other(message))))
                    }
-                    MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
+                    MockHealObjectOutcome::ErrOther(message) | MockHealObjectOutcome::PermanentOther(message) => {
+                        Err(Error::other(message))
+                    }
+                    MockHealObjectOutcome::RetryableReadQuorum => Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
+                        bucket.to_string(),
+                        object.to_string(),
+                    ))),
                };
            }
            if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
@@ -2393,8 +2559,18 @@ mod tests {
            ))
        }

-        async fn heal_bucket(&self, _bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+        async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+            self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
            self.bucket_heal_opts.lock().unwrap().push(*opts);
+            if let Some(message) = self
+                .bucket_heal_errors
+                .lock()
+                .unwrap()
+                .get_mut(bucket)
+                .and_then(VecDeque::pop_front)
+            {
+                return Err(Error::other(message));
+            }
            Ok(HealResultItem::default())
        }

@@ -2570,6 +2746,152 @@ mod tests {
        assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
    }

+    #[tokio::test(start_paused = true)]
+    async fn test_recursive_bucket_heal_retries_only_retryable_objects() {
+        let storage = Arc::new(MockStorage::default());
+        storage
+            .heal_object_outcomes
+            .lock()
+            .unwrap()
+            .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
+        let request = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        task.heal_bucket("bucket-a")
+            .await
+            .expect("retryable object failure should be retried within the listing page");
+
+        assert_eq!(
+            storage.heal_object_calls.lock().unwrap().as_slice(),
+            ["object-a".to_string(), "object-b".to_string(), "object-a".to_string()]
+        );
+        let progress = task.get_progress().await;
+        assert_eq!(progress.objects_scanned, 2);
+        assert_eq!(progress.objects_healed, 2);
+        assert_eq!(progress.objects_failed, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failures() {
+        let storage = Arc::new(MockStorage::default());
+        storage.heal_object_outcomes.lock().unwrap().insert(
+            "object-a".to_string(),
+            VecDeque::from([
+                MockHealObjectOutcome::RetryableReadQuorum,
+                MockHealObjectOutcome::RetryableReadQuorum,
+                MockHealObjectOutcome::RetryableReadQuorum,
+                MockHealObjectOutcome::RetryableReadQuorum,
+            ]),
+        );
+        storage.heal_object_outcomes.lock().unwrap().insert(
+            "object-b".to_string(),
+            VecDeque::from([MockHealObjectOutcome::PermanentOther("invalid metadata")]),
+        );
+        let request = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        let err = task
+            .heal_bucket("bucket-a")
+            .await
+            .expect_err("exhausted retryable and permanent failures must fail the task");
+
+        assert!(matches!(err, Error::TaskExecutionFailed { .. }));
+        let failure = task
+            .take_batch_failure()
+            .await
+            .expect("batch failure details should be retained on the task");
+        assert_eq!(failure.failed, 2);
+        assert_eq!(failure.retryable, 1);
+        assert_eq!(failure.permanent, 1);
+        assert_eq!(failure.first_object, "object-b");
+        let calls = storage.heal_object_calls.lock().unwrap();
+        assert_eq!(calls.iter().filter(|object| object.as_str() == "object-a").count(), 4);
+        assert_eq!(calls.iter().filter(|object| object.as_str() == "object-b").count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_cluster_heal_continues_after_bucket_failure() {
+        let storage = Arc::new(MockStorage {
+            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
+            ..Default::default()
+        });
+        let request = HealRequest::new(
+            HealType::Cluster,
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        let err = task.execute().await.expect_err("cluster task must report the failed bucket");
+
+        assert!(matches!(err, Error::TaskExecutionFailed { .. }));
+        let failure = task
+            .take_batch_failure()
+            .await
+            .expect("cluster failure details should be retained on the task");
+        assert_eq!(failure.failed, 1);
+        assert_eq!(
+            storage.bucket_heal_calls.lock().unwrap().as_slice(),
+            ["bucket-a".to_string(), "bucket-b".to_string()]
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_cluster_heal_retries_only_recoverable_bucket() {
+        let storage = Arc::new(MockStorage {
+            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+            bucket_heal_errors: Mutex::new(HashMap::from([(
+                "bucket-a".to_string(),
+                VecDeque::from(["lock acquisition timeout"]),
+            )])),
+            ..Default::default()
+        });
+        let request = HealRequest::new(
+            HealType::Cluster,
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        task.execute()
+            .await
+            .expect("cluster task should retry a recoverable bucket failure");
+
+        assert_eq!(
+            storage.bucket_heal_calls.lock().unwrap().as_slice(),
+            ["bucket-a".to_string(), "bucket-a".to_string(), "bucket-b".to_string()]
+        );
+    }
+
    #[tokio::test]
    async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
        let storage = Arc::new(MockStorage::default());
@@ -18,7 +18,7 @@
 //!
 //! These drive the REAL `ECStoreHealStorage` (not a mock) against a real 4-disk
 //! `ECStore`, mirroring `heal_integration_test.rs`. Every test is `#[serial]`
-//! and re-runs the full `setup_test_env` init (which sets the process-global
+//! and re-runs the full `heal_env` init (which sets the process-global
 //! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs
 //! in its own process so the OnceCell never collides.

@@ -34,20 +34,15 @@ use rustfs_heal::heal::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::{Arc, Once},
+    sync::Arc,
    time::Duration,
 };
-use tokio::fs;
-use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{
-    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
-    ObjectOperations as _, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
-};
+use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};

 /// 256 KiB + change: large enough to be stored as non-inline erasure shards
 /// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -62,78 +57,16 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
        .collect()
 }

-static INIT: Once = Once::new();
-
-fn init_tracing() {
-    INIT.call_once(|| {
-        let _ = tracing_subscriber::fmt()
-            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-            .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
-            .with_thread_names(true)
-            .try_init();
-    });
-}
-
-/// Build a real 4-disk `ECStore` + `ECStoreHealStorage`. Mirrors
-/// `heal_integration_test::setup_test_env`.
-async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    init_tracing();
-
-    let test_base_dir = format!("/tmp/rustfs_heal_b5_test_{}", uuid::Uuid::new_v4());
-    let temp_dir = PathBuf::from(&test_base_dir);
-    if temp_dir.exists() {
-        fs::remove_dir_all(&temp_dir).await.ok();
-    }
-    fs::create_dir_all(&temp_dir).await.unwrap();
-
-    let disk_paths = vec![
-        temp_dir.join("disk1"),
-        temp_dir.join("disk2"),
-        temp_dir.join("disk3"),
-        temp_dir.join("disk4"),
-    ];
-    for disk_path in &disk_paths {
-        fs::create_dir_all(disk_path).await.unwrap();
-    }
-
-    let mut endpoints = Vec::new();
-    for (i, disk_path) in disk_paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(i);
-        endpoints.push(endpoint);
-    }
-
-    let pool_endpoints = PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: 4,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "test".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    };
-    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
-
-    init_local_disks(endpoint_pools.clone()).await.unwrap();
-
-    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
-        .await
-        .unwrap();
-
-    let buckets_list = ecstore
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .unwrap();
-    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
-
-    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
-    (disk_paths, ecstore, heal_storage)
+/// Build a real 4-disk `ECStore` + `ECStoreHealStorage` via the shared
+/// rustfs-test-utils environment (backlog#1153 infra-1). Mirrors
+/// `heal_integration_test::heal_env`.
+async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    let env = rustfs_test_utils::TestECStoreEnv::builder()
+        .prefix("rustfs_heal_b5_test")
+        .build()
+        .await;
+    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+    (env.disk_paths, env.ecstore, heal_storage)
 }

 /// Create a bucket with S3 versioning ENABLED at creation time. Without this the
@@ -308,7 +241,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_enumeration_includes_all_versions_and_delete_marker_real_fixture() {
-        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (_disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-enum-versions";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -345,7 +278,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_old_nonlatest_version_after_disk_wipe() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-old-version-heal";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -397,7 +330,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_delete_marker_latest_enumerated_and_healed() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-dm-latest-heal";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -463,7 +396,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_version_id_normalization_null_and_unversioned_real_fixture() {
-        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (_disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-unversioned-normalize";
        create_unversioned_bucket(&ecstore, bucket).await;

@@ -499,7 +432,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_unversioned_bucket_e2e() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-unversioned-e2e";
        create_unversioned_bucket(&ecstore, bucket).await;

@@ -584,7 +517,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_resume_across_page_boundary_e2e() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;
        let bucket = "b5-resume-e2e";
        create_versioned_bucket(&ecstore, bucket).await;

@@ -617,28 +550,37 @@ mod serial_tests {
        };
        let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
        heal_manager.start().await.unwrap();
-        let request = HealRequest::new(
-            HealType::ErasureSet {
-                buckets: vec![bucket.to_string()],
-                set_disk_id: SET_DISK_ID.to_string(),
-            },
-            HealOptions {
-                recursive: true,
-                recreate_missing: true,
-                scan_mode: HealScanMode::Normal,
-                timeout: Some(Duration::from_secs(300)),
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        let task_id = request.id.clone();
-        let admission = heal_manager
-            .submit_heal_request(request)
-            .await
-            .expect("failed to submit erasure-set heal");
-        assert!(admission.is_admitted(), "erasure-set heal must be admitted");

-        wait_for_task(&heal_manager, &task_id, Duration::from_secs(120)).await;
+        // The erasure-set healer defers any per-version heal that hits a
+        // transient error (unmet quorum, DiskNotFound, a slow-disk read under
+        // load) to a later heal cycle: it persists its resume/checkpoint state
+        // and returns a terminal `Failed { .. "retry scheduled" }` so a *fresh*
+        // heal run re-drives the still-unhealed versions (see the finalize step
+        // in `ErasureSetHealer::heal_bucket_with_resume`). In production the
+        // background scanner is that next run; here we supply it ourselves by
+        // re-submitting the idempotent heal. This keeps the e2e faithful to the
+        // resume design without making it hostage to a rare single-version
+        // transient hiccup — the strict data-restoration assertions below still
+        // run only after a genuine `Completed`.
+        let build_request = |force_start: bool| {
+            let mut request = HealRequest::new(
+                HealType::ErasureSet {
+                    buckets: vec![bucket.to_string()],
+                    set_disk_id: SET_DISK_ID.to_string(),
+                },
+                HealOptions {
+                    recursive: true,
+                    recreate_missing: true,
+                    scan_mode: HealScanMode::Normal,
+                    timeout: Some(Duration::from_secs(300)),
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            );
+            request.force_start = force_start;
+            request
+        };
+        drive_heal_to_completion(&heal_manager, build_request, Duration::from_secs(120), 3).await;

        // Every data version is readable end-to-end and physically restored on the
        // wiped disk. (Resume machinery drove the full per-version heal.)
@@ -664,20 +606,74 @@ mod serial_tests {

    /// Poll a heal task to a terminal state, panicking on failure/timeout.
    async fn wait_for_task(heal_manager: &HealManager, task_id: &str, timeout: Duration) {
+        match await_terminal_status(heal_manager, task_id, timeout).await {
+            HealTaskStatus::Completed => {}
+            HealTaskStatus::Failed { error } => panic!("heal task failed: {error}"),
+            HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
+            other => panic!("heal task reached unexpected terminal state: {other:?}"),
+        }
+    }
+
+    /// Poll a heal task until it reaches a terminal state and return that state.
+    /// Panics only on timeout — callers decide how to treat each terminal state.
+    async fn await_terminal_status(heal_manager: &HealManager, task_id: &str, timeout: Duration) -> HealTaskStatus {
        let deadline = tokio::time::Instant::now() + timeout;
        loop {
-            if let Ok(status) = heal_manager.get_task_status(task_id).await {
-                match status {
-                    HealTaskStatus::Completed => return,
-                    HealTaskStatus::Failed { ref error } => panic!("heal task failed: {error}"),
-                    HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
-                    _ => {}
-                }
+            if let Ok(status) = heal_manager.get_task_status(task_id).await
+                && matches!(
+                    status,
+                    HealTaskStatus::Completed
+                        | HealTaskStatus::Failed { .. }
+                        | HealTaskStatus::Cancelled
+                        | HealTaskStatus::Timeout
+                )
+            {
+                return status;
            }
            if tokio::time::Instant::now() >= deadline {
-                panic!("heal task {task_id} did not complete within {timeout:?}");
+                panic!("heal task {task_id} did not reach a terminal state within {timeout:?}");
            }
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
    }
+
+    /// Submit an erasure-set heal and drive it to a genuine `Completed`,
+    /// tolerating the healer's by-design "retry scheduled" deferral.
+    ///
+    /// When a per-version heal hits a transient error the erasure-set healer
+    /// persists its resume state and returns a terminal `Failed` carrying
+    /// `retry scheduled` (retry budget still remaining) — expecting a later heal
+    /// run to finish the job. We re-submit ourselves (an idempotent re-heal),
+    /// mirroring the production background scanner. A `Failed` WITHOUT that
+    /// marker (e.g. `exhausted retries`) or any other non-`Completed` terminal
+    /// state is a real failure and panics.
+    async fn drive_heal_to_completion(
+        heal_manager: &HealManager,
+        build_request: impl Fn(bool) -> HealRequest,
+        per_attempt_timeout: Duration,
+        max_redrives: usize,
+    ) {
+        for attempt in 0..=max_redrives {
+            let request = build_request(attempt > 0);
+            let task_id = request.id.clone();
+            let admission = heal_manager
+                .submit_heal_request(request)
+                .await
+                .expect("failed to submit erasure-set heal");
+            assert!(admission.is_admitted(), "erasure-set heal must be admitted");
+
+            match await_terminal_status(heal_manager, &task_id, per_attempt_timeout).await {
+                HealTaskStatus::Completed => return,
+                HealTaskStatus::Failed { error } if error.contains("retry scheduled") => {
+                    info!(attempt, error, "erasure-set heal deferred a transient version; re-driving to completion");
+                    // Brief settle before the next idempotent re-heal.
+                    tokio::time::sleep(Duration::from_millis(200)).await;
+                }
+                HealTaskStatus::Failed { error } => panic!("heal task failed (non-retryable): {error}"),
+                HealTaskStatus::Cancelled => panic!("heal task was cancelled"),
+                other => panic!("heal task reached unexpected terminal state: {other:?}"),
+            }
+        }
+        panic!("erasure-set heal did not reach Completed within {} attempt(s)", max_redrives + 1);
+    }
 }
@@ -29,18 +29,13 @@ use rustfs_heal::heal::storage::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::{Arc, Once},
+    sync::Arc,
 };
-use tokio::fs;
-use tokio_util::sync::CancellationToken;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{
-    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
-    PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
-};
+use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _};

 /// 256 KiB + change: large enough to be stored as non-inline erasure shards, so
 /// deleting the `xl.meta` file does NOT delete the data (the `part.*` shards live
@@ -57,16 +52,6 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
        .collect()
 }

-static INIT: Once = Once::new();
-
-fn init_tracing() {
-    INIT.call_once(|| {
-        let _ = tracing_subscriber::fmt()
-            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-            .try_init();
-    });
-}
-
 /// Disable the dangling-delete grace window so the destructive path is genuinely
 /// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
 /// be dangling-deleted here. With grace at its 1h default the delete path would be
@@ -78,60 +63,16 @@ fn disable_dangling_grace() {
    }
 }

-/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage`.
-async fn setup_test_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    init_tracing();
-
-    let test_base_dir = format!("/tmp/rustfs_heal_b920_test_{}", uuid::Uuid::new_v4());
-    let temp_dir = PathBuf::from(&test_base_dir);
-    if temp_dir.exists() {
-        fs::remove_dir_all(&temp_dir).await.ok();
-    }
-    fs::create_dir_all(&temp_dir).await.unwrap();
-
-    let disk_paths: Vec<PathBuf> = (0..n_disks).map(|i| temp_dir.join(format!("disk{}", i + 1))).collect();
-    for disk_path in &disk_paths {
-        fs::create_dir_all(disk_path).await.unwrap();
-    }
-
-    let mut endpoints = Vec::new();
-    for (i, disk_path) in disk_paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(i);
-        endpoints.push(endpoint);
-    }
-
-    let pool_endpoints = PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: n_disks,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "test".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    };
-    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
-
-    init_local_disks(endpoint_pools.clone()).await.unwrap();
-
-    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
-        .await
-        .unwrap();
-
-    let buckets_list = ecstore
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .unwrap();
-    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
-
-    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
-    (disk_paths, ecstore, heal_storage)
+/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
+/// shared rustfs-test-utils environment (backlog#1153 infra-1).
+async fn heal_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    let env = rustfs_test_utils::TestECStoreEnv::builder()
+        .prefix("rustfs_heal_b920_test")
+        .disk_count(n_disks)
+        .build()
+        .await;
+    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+    (env.disk_paths, env.ecstore, heal_storage)
 }

 async fn create_versioned_bucket(ecstore: &Arc<ECStore>, bucket: &str) {
@@ -273,7 +214,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn disk_walk_page_enumerates_subquorum_version_omitted_by_list_object_versions() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
        let bucket = "b920-enum-gap";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -310,7 +251,7 @@ mod serial_tests {
    #[serial]
    async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
        let bucket = "b920-meta-lost";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -360,7 +301,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
        let bucket = "b920-torn";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -409,7 +350,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
        let bucket = "b920-reconstruct";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -463,7 +404,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn disk_walk_multipage_resume_heals_each_version_once() {
-        let (_disk_paths, ecstore, _heal_storage) = setup_test_env_n(4).await;
+        let (_disk_paths, ecstore, _heal_storage) = heal_env_n(4).await;
        let bucket = "b920-multipage";
        create_versioned_bucket(&ecstore, bucket).await;

@@ -523,7 +464,7 @@ mod serial_tests {
    #[serial]
    async fn offline_disk_during_walk_does_not_dangling_delete() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
        let bucket = "b920-offline";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -570,7 +511,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
        let bucket = "b1044-deep-verify";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -22,20 +22,16 @@ use rustfs_heal::heal::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::{Arc, Once},
+    sync::Arc,
    time::Duration,
 };
 use tokio::fs;
-use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{
-    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
-    PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
-};
+use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};

 const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
 const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
@@ -58,89 +54,16 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration
    }
 }

-static INIT: Once = Once::new();
-
-pub fn init_tracing() {
-    INIT.call_once(|| {
-        let _ = tracing_subscriber::fmt()
-            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-            .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
-            .with_thread_names(true)
-            .try_init();
-    });
-}
-
-/// Test helper: Create test environment with ECStore
-async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    init_tracing();
-
-    // create temp dir as 4 disks with unique base dir
-    let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4());
-    let temp_dir = std::path::PathBuf::from(&test_base_dir);
-    if temp_dir.exists() {
-        fs::remove_dir_all(&temp_dir).await.ok();
-    }
-    fs::create_dir_all(&temp_dir).await.unwrap();
-
-    // create 4 disk dirs
-    let disk_paths = vec![
-        temp_dir.join("disk1"),
-        temp_dir.join("disk2"),
-        temp_dir.join("disk3"),
-        temp_dir.join("disk4"),
-    ];
-
-    for disk_path in &disk_paths {
-        fs::create_dir_all(disk_path).await.unwrap();
-    }
-
-    // create EndpointServerPools
-    let mut endpoints = Vec::new();
-    for (i, disk_path) in disk_paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
-        // set correct index
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(i);
-        endpoints.push(endpoint);
-    }
-
-    let pool_endpoints = PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: 4,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "test".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    };
-
-    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
-
-    // format disks (only first time)
-    init_local_disks(endpoint_pools.clone()).await.unwrap();
-
-    // Use port 0 so nextest can run this integration binary in parallel
-    // with other ECStore-backed tests without sharing a fixed peer port.
-    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
-        .await
-        .unwrap();
-
-    // init bucket metadata system
-    let buckets_list = ecstore
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .unwrap();
-    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
-    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
-
-    // Create heal storage layer
-    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
-
-    (disk_paths, ecstore, heal_storage)
+/// Test helper: build the shared 4-disk temp-dir ECStore environment
+/// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage
+/// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest.
+async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    let env = rustfs_test_utils::TestECStoreEnv::builder()
+        .prefix("rustfs_heal_heal_test")
+        .build()
+        .await;
+    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+    (env.disk_paths, env.ecstore, heal_storage)
 }

 /// Test helper: Create a test bucket
@@ -169,7 +92,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_object_basic() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;

        // Create test bucket and object
        let bucket_name = "test-heal-object-basic";
@@ -238,7 +161,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_object_reclaims_orphan_data_dir_when_healthy() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;

        let bucket_name = "test-heal-reclaim-orphan";
        let object_name = "healthy-object.bin";
@@ -350,7 +273,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_bucket_basic() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;

        // Create test bucket
        let bucket_name = "test-heal-bucket-basic";
@@ -420,7 +343,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_format_basic() {
-        let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, _ecstore, heal_storage) = heal_env().await;

        // ─── 1️⃣ delete format.json on one disk ──────────────
        let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
@@ -446,7 +369,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_format_with_data() {
-        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (disk_paths, ecstore, heal_storage) = heal_env().await;

        // Create test bucket and object
        let bucket_name = "test-heal-format-with-data";
@@ -535,7 +458,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_storage_api_direct() {
-        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
+        let (_disk_paths, ecstore, heal_storage) = heal_env().await;

        // Test direct heal storage API calls

@@ -15,17 +15,13 @@ pub(crate) mod endpoint_index {
    pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
 }

+// The temp-disk ECStore bootstrap itself lives in rustfs-test-utils
+// (backlog#1153 infra-1); this surface keeps only what the heal test bodies
+// still touch directly.
 #[allow(unused_imports)]
 pub(crate) mod integration {
-    pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
-    pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
-    pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools;
-    pub(crate) use rustfs_ecstore::api::layout::Endpoints;
-    pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints;
    pub(crate) use rustfs_ecstore::api::storage::ECStore;
-    pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
    pub(crate) use rustfs_storage_api::BucketOperations;
-    pub(crate) use rustfs_storage_api::BucketOptions;
    pub(crate) use rustfs_storage_api::MakeBucketOptions;
    pub(crate) use rustfs_storage_api::ObjectIO;
    pub(crate) use rustfs_storage_api::ObjectOperations;
@@ -30,35 +30,36 @@ workspace = true

 [dependencies]
 rustfs-credentials = { workspace = true }
-rustfs-config = { workspace = true, features = ["constants", "server-config-model"] }
-tokio.workspace = true
-time = { workspace = true, features = ["serde-human-readable"] }
+rustfs-config = { workspace = true, features = ["server-config-model"] }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
+time = { workspace = true, features = ["serde-human-readable", "macros"] }
 serde = { workspace = true, features = ["derive", "rc"] }
 rustfs-ecstore = { workspace = true }
 rustfs-storage-api = { workspace = true }
 rustfs-policy.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }
 async-trait.workspace = true
 thiserror.workspace = true
 arc-swap = { workspace = true }
 rustfs-crypto = { workspace = true }
 futures.workspace = true
 base64-simd = { workspace = true }
-jsonwebtoken = { workspace = true }
+jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
 tracing.workspace = true
 rustfs-madmin.workspace = true
 rustfs-utils = { workspace = true, features = ["path"] }
 rustfs-io-metrics.workspace = true
-tokio-util.workspace = true
+tokio-util = { workspace = true, features = ["io", "compat"] }
 pollster.workspace = true
-reqwest = { workspace = true }
-moka = { workspace = true }
-openidconnect = { workspace = true }
+reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
+moka = { workspace = true, features = ["future"] }
+openidconnect = { workspace = true, default-features = false, features = ["accept-rfc3339-timestamps"] }
 http = { workspace = true }
 url = { workspace = true }

 [dev-dependencies]
 pollster.workspace = true
+rustfs-test-utils = { workspace = true }
 serial_test = { workspace = true }
 temp-env = { workspace = true, features = ["async_closure"] }
 tempfile = { workspace = true }
@@ -22,6 +22,15 @@ pub(crate) fn credentials_or_default() -> Credentials {
    credentials().unwrap_or_default()
 }

+/// The signing key for STS session tokens.
+///
+/// GHSA-m77q-r63m-pj89 (intentionally UNFIXED): this returns the root secret
+/// key, so STS JWTs are signed with the shared root secret and anyone holding
+/// it can forge session tokens. The behavior is pinned by
+/// `test_ghsa_m77q_sts_session_token_signed_with_root_secret` in `sys.rs`;
+/// fixing the advisory (a dedicated STS signing key distinct from the root
+/// secret) must update that test red -> green. See
+/// docs/testing/security-regressions.md.
 pub(crate) fn token_signing_key() -> Option<String> {
    credentials().map(|cred| cred.secret_key)
 }
@@ -1952,6 +1952,18 @@ mod tests {
        );
    }

+    /// GHSA-m77q-r63m-pj89 (STS JWTs signed with the shared root secret —
+    /// intentionally UNFIXED). This test characterizes the STS credential
+    /// lifecycle: a session token signed with the active token-signing key
+    /// decodes and authorizes through the STS path. It pins that the SAME key
+    /// both signs and verifies STS tokens.
+    ///
+    /// MUST UPDATE WHEN GHSA-m77q IS FIXED: the fix introduces a dedicated STS
+    /// signing key distinct from the root secret. The narrower pin that captures
+    /// the advisory's exact signature (signing key == root secret) is
+    /// `test_ghsa_m77q_sts_session_token_signed_with_root_secret`; keep both in
+    /// lockstep and flip them red -> green together.
+    /// Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89>
    #[tokio::test]
    async fn test_created_sts_credentials_authorize_with_session_token_claims() {
        ensure_test_global_credentials();
@@ -2053,6 +2065,95 @@ mod tests {
        );
    }

+    /// GHSA-m77q-r63m-pj89 flow-level pin (STS session tokens are signed with the
+    /// shared root secret — intentionally UNFIXED). Anyone holding the root secret
+    /// can forge STS session tokens because RustFS reuses the root secret as the
+    /// STS token-signing key. This test PINS that vulnerable-by-design behavior end
+    /// to end: (1) the token-signing key IS the root secret, (2) an AssumeRole-style
+    /// session token issued with that key decodes with the ROOT secret and NOT with
+    /// any other secret, and (3) it authorizes through the STS path.
+    ///
+    /// MUST UPDATE WHEN GHSA-m77q IS FIXED: the fix introduces a dedicated STS
+    /// signing key distinct from the root secret. That makes assertion (1) and the
+    /// "root secret decodes the token" assertion fail (red). Whoever fixes m77q must
+    /// replace this pin with a regression asserting the NEW behavior — the root
+    /// secret can no longer decode STS session tokens — i.e. red -> green.
+    /// Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89>
+    #[tokio::test]
+    async fn test_ghsa_m77q_sts_session_token_signed_with_root_secret() {
+        ensure_test_global_credentials();
+
+        let root = crate::root_credentials::credentials().expect("global action (root) credentials should be initialized");
+        assert!(!root.secret_key.is_empty(), "root secret must be present for the STS signing-key pin");
+
+        // (1) m77q signature: the STS token-signing key IS the root secret. A fix
+        // that introduces a dedicated STS key makes this fail -> update red->green.
+        assert_eq!(
+            crate::root_credentials::token_signing_key().as_deref(),
+            Some(root.secret_key.as_str()),
+            "GHSA-m77q pin: STS tokens are signed with the root secret. If this fails, \
+             m77q may have been fixed (dedicated STS key) — update this test red->green."
+        );
+
+        // AssumeRole-style issuance: mint a session token with the active signing key.
+        // Use the mock store's registered STS parent (group `testgroup` -> readwrite)
+        // so authorization resolves through the STS path; the m77q pin is about the
+        // signing key, not the parent identity.
+        let parent_user = "sts-fallback-test-parent";
+        let signing_key = crate::root_credentials::token_signing_key().expect("STS token-signing key should be present");
+        let mut claims = HashMap::new();
+        claims.insert("parent".to_string(), Value::String(parent_user.to_string()));
+        claims.insert(
+            "exp".to_string(),
+            Value::Number(serde_json::Number::from(
+                (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(),
+            )),
+        );
+        let mut cred = get_new_credentials_with_metadata(&claims, &signing_key).expect("STS credentials should be created");
+        cred.parent_user = parent_user.to_string();
+
+        // (2) The session token decodes with the ROOT secret specifically — the crux
+        // of m77q — and rejects any non-root secret (HMAC verification fails).
+        let decoded = get_claims_from_token_with_secret(&cred.session_token, &root.secret_key)
+            .expect("GHSA-m77q pin: STS session token must decode with the ROOT secret");
+        assert_eq!(decoded.get("parent").and_then(Value::as_str), Some(parent_user));
+        assert!(
+            get_claims_from_token_with_secret(&cred.session_token, "not-the-root-secret").is_err(),
+            "GHSA-m77q pin: STS session token must NOT decode with a non-root secret"
+        );
+
+        // (3) The root-secret-signed STS credentials authorize end to end.
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.unwrap();
+        let iam_sys = IamSys::new(cache_manager);
+        iam_sys
+            .set_temp_user(&cred.access_key, &cred, None)
+            .await
+            .expect("STS credentials should be persisted in the temp-user cache");
+
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: &cred.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListBucketAction),
+            bucket: "mybucket",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &decoded,
+            deny_only: false,
+        };
+        let prepared = iam_sys.prepare_auth(&args).await;
+        assert!(
+            matches!(prepared.mode, PreparedIamMode::Sts { .. }),
+            "root-secret-signed STS credentials must use the STS authorization path"
+        );
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "root-secret-signed STS credentials should authorize through the parent group policy (m77q)"
+        );
+    }
+
    /// Regression test: temp credentials without groups in args still receive group-attached
    /// policies via the parent user (groups fallback). Without the fallback, policy_db_get
    /// would get None for groups and the user would have no group policies, so the action
@@ -22,9 +22,7 @@

 #[allow(unused_imports)]
 pub(crate) mod fixture {
-    pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
-    pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType};
-    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
+    pub(crate) use rustfs_ecstore::api::layout::SetupType;

    // `update_erasure_type` is a write-side global facade entry. Its use is
    // restricted to reviewed storage_api boundaries; this test-only module is
@@ -31,55 +31,16 @@

 mod ecstore_test_compat;

-use ecstore_test_compat::fixture::{
-    ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, SetupType, init_local_disks, update_erasure_type,
-};
+use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
 use rustfs_iam::cache::Cache;
 use rustfs_iam::store::object::ObjectStore;
 use rustfs_iam::store::{GroupInfo, Store};
 use serial_test::serial;
 use std::collections::HashMap;
-use std::sync::Arc;
-use tokio_util::sync::CancellationToken;

 const TEST_GROUP: &str = "seq-restart-group";
 const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];

-async fn build_local_ecstore(temp_dir: &std::path::Path) -> Arc<ECStore> {
-    let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.join(format!("disk{i}"))).collect();
-    for disk_path in &disk_paths {
-        tokio::fs::create_dir_all(disk_path).await.unwrap();
-    }
-
-    let mut endpoints = Vec::new();
-    for (i, disk_path) in disk_paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(i);
-        endpoints.push(endpoint);
-    }
-
-    let pool_endpoints = PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: 4,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "test".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    };
-    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
-
-    init_local_disks(endpoint_pools.clone()).await.unwrap();
-
-    // Port 0 keeps this integration binary parallel-safe alongside other
-    // ECStore-backed tests.
-    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
-    ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
-        .await
-        .unwrap()
-}
-
 /// Restores single-node erasure mode even when an assertion panics, so a
 /// failing run cannot poison later `#[serial]` tests in this process.
 struct ErasureModeGuard;
@@ -97,7 +58,15 @@ async fn load_all_bypasses_namespace_lock_quorum() {
    // must be shortened before the first locked operation of this process.
    temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
        let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_no_lock_test_").unwrap();
-        let ecstore = build_local_ecstore(temp_dir.path()).await;
+        // Shared temp-disk ECStore env (rustfs-test-utils, backlog#1153 infra-1).
+        // base_dir keeps cleanup ownership with this TempDir; the historical
+        // bootstrap never initialized the bucket-metadata system, so opt out.
+        let ecstore = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(temp_dir.path())
+            .init_bucket_metadata(false)
+            .build()
+            .await
+            .ecstore;
        let store = ObjectStore::new(ecstore);

        // Seed IAM data while namespace locks still work (single-node mode).
@@ -28,15 +28,15 @@ categories = ["development-tools", "filesystem"]
 workspace = true

 [dependencies]
-bytes = { workspace = true }
+bytes = { workspace = true, features = ["serde"] }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] }
+tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
 memmap2 = { workspace = true }
 rustfs-io-metrics = { workspace = true }
 tracing = { workspace = true }

 [dev-dependencies]
-tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }

 [lib]
 doctest = false
@@ -33,14 +33,14 @@ metrics = { workspace = true }
 rustfs-s3-ops = { workspace = true }
 num_cpus = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["sync","rt"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
 tracing = { workspace = true }
 sysinfo = { workspace = true }

 [dev-dependencies]
-criterion = { workspace = true }
+criterion = { workspace = true, features = ["html_reports"] }
 metrics-util = { version = "0.20", features = ["debugging"] }
-tokio = { workspace = true, features = ["test-util","rt","macros"] }
+tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }

 [lints]
 workspace = true
@@ -26,32 +26,32 @@ categories = ["authentication", "web-programming"]
 authors.workspace = true

 [dependencies]
-tokio = { workspace = true, features = ["full"] }
-reqwest = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt", "sync"] }
+reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "json"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = ["raw_value"] }
 thiserror = { workspace = true }
 tracing = { workspace = true }
-time = { workspace = true }
-moka = { workspace = true }
+time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
+moka = { workspace = true, features = ["future"] }
 rustfs-credentials = { workspace = true }
 rustfs-policy = { workspace = true }
 rustfs-utils = { workspace = true, features = ["egress"] }
 url = { workspace = true }
 # Middleware dependencies
-tower = { workspace = true }
+tower = { workspace = true, features = ["timeout"] }
 http = { workspace = true }
-hyper = { workspace = true, features = ["server"] }
+hyper = { workspace = true, features = ["server", "http2", "http1"] }
 http-body = { workspace = true }
 http-body-util = { workspace = true }
-bytes = { workspace = true }
+bytes = { workspace = true, features = ["serde"] }
 futures = { workspace = true }

 [dev-dependencies]
-tokio = { workspace = true, features = ["test-util"] }
-tower = { workspace = true, features = ["util"] }
-hyper = { workspace = true, features = ["server"] }
-serde_json = { workspace = true }
+tokio = { workspace = true, features = ["test-util", "fs", "macros", "rt", "time"] }
+tower = { workspace = true, features = ["util", "timeout"] }
+hyper = { workspace = true, features = ["server", "http2", "http1"] }
+serde_json = { workspace = true, features = ["raw_value"] }
 temp-env = { workspace = true }

 [[test]]
--- a/Show More
+++ b/Show More