feat(obs): enhance observability with tracing spans and metrics integration

Add observability identities and operation spans
2026-07-29 17:48:58 +00:00 · 2026-07-19 22:03:03 +08:00 · 2026-07-14 15:22:20 +08:00
202 changed files with 2687 additions and 19803 deletions
@@ -1,26 +0,0 @@
-## —— Coverage --------------------------------------------------------------------------------------
-
-# Local equivalent of the weekly coverage workflow (.github/workflows/coverage.yml,
-# backlog#1153 infra-5): same measurement scope (--workspace --exclude e2e_test,
-# nextest `ci` profile) and the same per-crate table. Slow — the instrumented
-# build cannot reuse your normal target cache and then runs the whole suite.
-# Doctests are not measured (needs nightly). Outputs land in target/llvm-cov/.
-.PHONY: coverage
-coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, writes target/llvm-cov/)
-	@if ! command -v cargo-llvm-cov >/dev/null 2>&1; then \
-		echo >&2 "❌ cargo-llvm-cov is required for 'make coverage' but was not found."; \
-		echo >&2 "   Install it with:"; \
-		echo >&2 "     cargo install cargo-llvm-cov --locked"; \
-		echo >&2 "     rustup component add llvm-tools-preview"; \
-		exit 1; \
-	fi
-	@if ! command -v cargo-nextest >/dev/null 2>&1; then \
-		echo >&2 "❌ cargo-nextest is required for 'make coverage' (see 'make test')."; \
-		echo >&2 "   Install it with: cargo install cargo-nextest --locked"; \
-		exit 1; \
-	fi
-	NEXTEST_PROFILE=ci cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
-	@mkdir -p target/llvm-cov
-	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
-	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
@@ -45,15 +45,6 @@ e2e-reliability = { max-threads = 1 }
 filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/))'
 test-group = 'ecstore-serial-flaky'

-# Serialize the multipart crash-consistency scenarios (dist-2, backlog#1150):
-# each spawns a 4-disk hermetic erasure set and drives full staged-upload +
-# commit + GET cycles — the same cross-disk-commit IO shape that made
-# concurrent_resend load-sensitive. Preventive serialization only, no retries.
-# The matching ci-profile override is after [profile.ci].
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
-test-group = 'ecstore-serial-flaky'
-
 # Serialize the 4-disk reliability / degraded-read e2e tests (see the
 # e2e-reliability test-group note above). The matching ci-profile override is at
 # the end of the file, after [profile.ci] is declared.
@@ -118,13 +109,6 @@ retries = 2
 filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
 test-group = 'e2e-reliability'

-# Serialize the multipart crash-consistency scenarios under the ci profile too
-# (see the matching default-profile override near the top). Not a quarantine:
-# no retries, just serialized 4-disk cross-disk-commit IO.
-[[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
-test-group = 'ecstore-serial-flaky'
-
 # ---------------------------------------------------------------------------
 # e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
 # ---------------------------------------------------------------------------
@@ -152,7 +136,7 @@ test-group = 'ecstore-serial-flaky'
 # the nightly profile derives its set as "the replication module MINUS this
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
-# regexes byte-identical. Count invariant: 20 here + 23 nightly = 43 total
+# regexes byte-identical. Count invariant: 20 here + 18 nightly = 38 total
 # (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
@@ -160,38 +144,12 @@ test-group = 'ecstore-serial-flaky'
 # the guard now honours an off-by-default opt-in and this suite's source servers
 # set it (RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET) — so the allowlist below is
 # restored.
-#
-# Security negative-auth subset (backlog#1151 sec-5): the three attacker-facing
-# S3 auth-rejection suites join the first clause above by module name —
-# presigned_negative (sec-2), negative_sigv4 (sec-1, header SigV4), and
-# admin_auth (sec-4, admin gate + root-credential lifecycle). All three use
-# RustFSTestEnvironment on a random port and are parallel-safe, so they meet the
-# smoke admission criteria unchanged. This is the wiring step that makes those
-# merged suites actually execute on every PR (they were dead until listed here).
-# A rename that drops any of them out of this filter would silently thin the
-# security gate with no CI signal, so scripts/check_security_smoke_count.sh owns
-# a count-floor guard over exactly this subset (infra-12 mechanism, floor in
-# .config/security-smoke-floor.txt), invoked from the e2e-tests job in ci.yml.
-# NOT here by topology: the GHSA-3p3x FTPS/WebDAV constant-time e2e
-# (protocols::test_protocol_core_suite) binds fixed ports and needs the
-# ftps,webdav features, so it cannot join this random-port, default-feature
-# profile; its GHSA-r5qv sibling is a unit test that already runs in the
-# test-and-lint `--all --exclude e2e_test` pass. See
-# docs/testing/security-regressions.md for the full CI-execution map.
-#
-# ILM tiering main path (backlog#1148 ilm-7): the `reliant::tiering::` clause
-# admits the hermetic transition e2e. Like the fast replication pair checks it
-# spawns a second independent single-node server (the cold RustFS tier), not a
-# cluster, so it keeps the lane's parallel-safe / no-external-dependency
-# properties. The RustFS warm backend has no loopback guard (that guard is
-# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
 [profile.e2e-smoke]
 default-filter = """
    package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud)_test::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative)_test::/)
        | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
        | test(/^reliant::lifecycle::/)
-        | test(/^reliant::tiering::/)
    )
 """
 fail-fast = false
@@ -204,7 +162,7 @@ fail-fast = false
 #
 #   * 8 bucket-replication data-plane tests — they PUT/delete objects and poll
 #     until source and target converge; two replicate over HTTPS.
-#   * 11 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
+#   * 9 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
 #   * 1 `_real_single_node` service-account round-trip test.
 #
@@ -241,79 +199,3 @@ fail-fast = false
 # Emitted to target/nextest/e2e-repl-nightly/junit.xml; uploaded by the nightly
 # workflow as the failure-triage artifact.
 path = "junit.xml"
-
-# ---------------------------------------------------------------------------
-# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
-# ---------------------------------------------------------------------------
-# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
-# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
-# object_lock (33), multipart_auth (109), quota, checksum, encryption,
-# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
-# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
-# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
-#
-# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
-#   * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
-#     ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
-#   * the 6 cluster suites that spin up a RustFSTestClusterEnvironment
-#     (cluster_concurrency, stale_multipart_cleanup_cluster,
-#     namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
-#     object_lambda) — too heavy for the merge budget; they run in ci-7's
-#     nightly 4-node lane.
-#   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (20 slow) lanes and reserves
-#     it for those, so e2e-full does not double-run it.
-#   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
-#     manual-localhost:9000 reliant/policy tests are ci-13's migration.
-#
-# Each e2e test spawns its own single-node rustfs server on a random port with
-# an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
-# parallel-safe — the same property e2e-smoke relies on. The exception is the
-# 4-disk reliability / degraded-read fault-injection tests, serialized below
-# (identical to the ci profile) so several 4-disk servers never run at once.
-# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
-# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
-# product failures cannot be quarantined away with retries, so each family is
-# excluded here with its tracking issue, under the same discipline as the
-# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
-# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
-# negative-path siblings of each family stay in as regression guards.
-#   * rustfs#4842 — extract/snowball expand pipeline 500s (mtime=0
-#     OffsetDateTime deserialization + same-path failures).
-#   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
-#     archive even under ignore-errors semantics.
-#   * rustfs#4844 — anonymous POST-object with SSE-S3 / bucket-default SSE
-#     returns 500.
-#   * rustfs#4845 — 403 on allowed anonymous POST object-lock fields and on
-#     the list metadata=true extension.
-#   * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
-#     under parallel load (multi-node in-process clusters; natural home is
-#     ci-7's nightly cluster lane).
-[profile.e2e-full]
-default-filter = """
-    package(e2e_test)
-    & !test(/^protocols::/)
-    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
-    & !test(/^replication_extension_test::/)
-    & !test(/^multipart_auth_test::test_signed_put_object_extract_(accepts_compat_header|expands_tar_entries_with_prefix_headers|expands_tar_gz_archive|expands_tbz2_archive|expands_tgz_archive|expands_txz_archive|expands_tzst_archive|normalizes_prefix_header_value|preserves_directory_markers_by_default|preserves_object_lock_legal_hold|preserves_object_lock_retention|preserves_pax_metadata_and_version_id|preserves_request_metadata_on_extracted_objects|preserves_sse_c|preserves_sse_s3_and_redirect|preserves_storage_class|uses_bucket_default_sse_s3)$/)
-    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(prefers_exact_minio_prefix_over_suffix_fallback|supports_minio_prefix_and_directory_markers)$/)
-    & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
-    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
-    & !test(/^multipart_auth_test::test_anonymous_post_object_(accepts_sse_s3|rejects_sse_s3_missing_from_policy_conditions|uses_bucket_default_sse_kms|uses_bucket_default_sse_s3)$/)
-    & !test(/^multipart_auth_test::test_anonymous_post_object_(accepts_object_lock_legal_hold_field|accepts_object_lock_retention_fields)$/)
-    & !test(/^list_object(s_v2|_versions)_metadata_extension_test::/)
-    & !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
-"""
-fail-fast = false
-
-[profile.e2e-full.junit]
-# Emitted to target/nextest/e2e-full/junit.xml; uploaded by the e2e-full job.
-path = "junit.xml"
-
-# Serialize the 4-disk reliability / degraded-read e2e tests under e2e-full too
-# (see the e2e-reliability test-group note near the top of this file). Not a
-# quarantine: no retries, just single-threaded so several 4-disk servers never
-# run concurrently.
-[[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
-test-group = 'e2e-reliability'
@@ -1,12 +0,0 @@
-# Committed floor for the number of security negative-auth tests selected by the
-# e2e-smoke PR profile (see scripts/check_security_smoke_count.sh, backlog#1151
-# sec-5).
-#
-# The floor equals the exact count of e2e_test cases whose name starts with a
-# security module prefix (negative_sigv4_test, presigned_negative_test,
-# admin_auth_test) that the [profile.e2e-smoke] default-filter in
-# .config/nextest.toml selects, at the time this file was last updated. CI fails
-# if the selected count drops below this number, so a rename or removal that
-# thins the security smoke gate must update this file in the same PR.
-# Adding tests does not require a bump, but bumping keeps the guard tight.
-16
@@ -136,13 +136,11 @@ jobs:
            echo "⚡ Manual/scheduled build detected"
          fi

-          {
-            echo "should_build=$should_build"
-            echo "build_type=$build_type"
-            echo "version=$version"
-            echo "short_sha=$short_sha"
-            echo "is_prerelease=$is_prerelease"
-          } >> "$GITHUB_OUTPUT"
+          echo "should_build=$should_build" >> $GITHUB_OUTPUT
+          echo "build_type=$build_type" >> $GITHUB_OUTPUT
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "short_sha=$short_sha" >> $GITHUB_OUTPUT
+          echo "is_prerelease=$is_prerelease" >> $GITHUB_OUTPUT

          echo "📊 Build Summary:"
          echo "  - Should build: $should_build"
@@ -222,7 +220,7 @@ jobs:
    name: Build RustFS
    needs: [ build-check, prepare-platform-matrix ]
    if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.platform == 'linux' && fromJSON('["self-hosted","linux","sm-standard-2"]') || matrix.os }}
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
@@ -495,7 +493,7 @@ jobs:
            if [[ "${{ matrix.platform }}" == "windows" ]]; then
              dir
            else
-              ls -lh "${PACKAGE_NAME}.zip"
+              ls -lh ${PACKAGE_NAME}.zip
            fi
          else
            echo "❌ Failed to create package: ${PACKAGE_NAME}.zip"
@@ -544,13 +542,11 @@ jobs:
            fi
          fi

-          {
-            echo "package_name=${PACKAGE_NAME}"
-            echo "package_file=${PACKAGE_NAME}.zip"
-            echo "latest_files=${LATEST_FILES}"
-            echo "build_type=${BUILD_TYPE}"
-            echo "version=${VERSION}"
-          } >> "$GITHUB_OUTPUT"
+          echo "package_name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT
+          echo "package_file=${PACKAGE_NAME}.zip" >> $GITHUB_OUTPUT
+          echo "latest_files=${LATEST_FILES}" >> $GITHUB_OUTPUT
+          echo "build_type=${BUILD_TYPE}" >> $GITHUB_OUTPUT
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT

          echo "📦 Package created: ${PACKAGE_NAME}.zip"
          if [[ -n "$LATEST_FILES" ]]; then
@@ -747,8 +743,8 @@ jobs:
            RELEASE_URL=$(gh release view "$TAG" --json url --jq '.url')
          fi

-          echo "release_id=$RELEASE_ID" >> "$GITHUB_OUTPUT"
-          echo "release_url=$RELEASE_URL" >> "$GITHUB_OUTPUT"
+          echo "release_id=$RELEASE_ID" >> $GITHUB_OUTPUT
+          echo "release_url=$RELEASE_URL" >> $GITHUB_OUTPUT
          echo "Created release: $RELEASE_URL"

  # Prepare and upload release assets
@@ -797,9 +793,9 @@ jobs:
          cd ./release-assets

          # Generate checksums for all files (including latest versions)
-          if compgen -G "*.zip" >/dev/null; then
-            sha256sum -- *.zip > SHA256SUMS
-            sha512sum -- *.zip > SHA512SUMS
+          if ls *.zip >/dev/null 2>&1; then
+            sha256sum *.zip > SHA256SUMS
+            sha512sum *.zip > SHA512SUMS
          fi

          cd ..
@@ -435,16 +435,6 @@ jobs:
      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

-      # Guard the security negative-auth smoke subset (backlog#1151 sec-5)
-      # against a rename or deletion silently dropping it out of the e2e-smoke
-      # filter. The script lists what the profile selects and fails if the count
-      # of security auth-rejection tests falls below the committed floor in
-      # .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
-      # before the smoke suite so a thinned gate fails fast; the `nextest list`
-      # here compiles the e2e_test binaries the run below reuses.
-      - name: Check security smoke subset count floor
-        run: ./scripts/check_security_smoke_count.sh check
-
      # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
      # profile.e2e-smoke default-filter in .config/nextest.toml is the single
      # wiring mechanism for e2e tests in CI — extend that filter instead of
@@ -478,59 +468,6 @@ jobs:
          path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
          retention-days: 3

-  e2e-full:
-    name: End-to-End Tests (full merge gate)
-    # Merge gate only (backlog#1149 ci-5): the never-automated user-visible
-    # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
-    # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
-    # every PR, so it is gated to main pushes, the merge queue, and manual
-    # dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
-    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
-    if: >-
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'merge_group' ||
-      (github.event_name == 'push' && github.ref == 'refs/heads/main')
-    needs: [ build-rustfs-debug-binary ]
-    runs-on: sm-standard-2
-    timeout-minutes: 55
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-e2e
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      # Download after the cache restore so the freshly built binary from the
-      # build job always wins over anything restored into target/debug.
-      - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: rustfs-debug-binary
-          path: target/debug
-
-      - name: Make binary executable
-        run: chmod +x ./target/debug/rustfs
-
-      # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
-      # default-filter in .config/nextest.toml is the single wiring mechanism —
-      # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
-      # debug binary; each test spawns its own rustfs server on a random port.
-      - name: Run e2e full suite
-        run: cargo nextest run --profile e2e-full -p e2e_test
-
-      - name: Upload junit
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: e2e-full-junit-${{ github.run_number }}
-          path: target/nextest/e2e-full/junit.xml
-          retention-days: 7
-
  e2e-tests-rio-v2:
    name: End-to-End Tests (rio-v2)
    needs: [ build-rustfs-debug-binary-rio-v2 ]
@@ -1,122 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
-#
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
-#
-# Measurement scope matches the PR test gate (ci.yml "Run tests"):
-# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
-# NOT measured (ci.yml runs them uninstrumented; `cargo llvm-cov` needs a
-# nightly toolchain to cover doctests). Trend-comparison workflow:
-# docs/testing/README.md "Coverage" section. `make coverage` is the local
-# equivalent. Scheduled failures alert via the ci-8 composite action.
-
-name: coverage
-
-on:
-  workflow_dispatch:
-  schedule:
-    # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
-    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
-    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
-    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
-
-# Only alert-on-failure needs more than read access; it declares its own
-# job-level `issues: write`.
-permissions:
-  contents: read
-
-jobs:
-  coverage:
-    name: Workspace coverage (weekly)
-    runs-on: sm-standard-4
-    # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
-      # retries=0 plus the quarantine list and the ecstore-serial-flaky
-      # serialization. Set via env because `cargo llvm-cov`'s own --profile
-      # flag selects the *cargo build* profile, not the nextest profile.
-      NEXTEST_PROFILE: ci
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-coverage
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-        with:
-          tool: cargo-llvm-cov
-
-      - name: Install llvm-tools component
-        run: rustup component add llvm-tools-preview
-
-      - name: Run instrumented test suite
-        run: cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
-
-      - name: Generate lcov and JSON reports
-        run: |
-          mkdir -p target/llvm-cov
-          cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
-          cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-
-      - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Upload coverage artifact
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: coverage-lcov-${{ github.run_number }}
-          path: |
-            target/llvm-cov/lcov.info
-            target/llvm-cov/coverage.json
-          retention-days: 90
-          if-no-files-found: ignore
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [coverage]
-    # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
-    # manual workflow_dispatch runs stay quiet so a debugging run never files a
-    # spurious alert.
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -247,15 +247,13 @@ jobs:
            esac
          fi

-          {
-            echo "should_build=$should_build"
-            echo "should_push=$should_push"
-            echo "build_type=$build_type"
-            echo "version=$version"
-            echo "short_sha=$short_sha"
-            echo "is_prerelease=$is_prerelease"
-            echo "create_latest=$create_latest"
-          } >> "$GITHUB_OUTPUT"
+          echo "should_build=$should_build" >> $GITHUB_OUTPUT
+          echo "should_push=$should_push" >> $GITHUB_OUTPUT
+          echo "build_type=$build_type" >> $GITHUB_OUTPUT
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "short_sha=$short_sha" >> $GITHUB_OUTPUT
+          echo "is_prerelease=$is_prerelease" >> $GITHUB_OUTPUT
+          echo "create_latest=$create_latest" >> $GITHUB_OUTPUT

          echo "🐳 Docker Build Summary:"
          echo "  - Should build: $should_build"
@@ -322,6 +320,7 @@ jobs:
        run: |
          BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
          VERSION="${{ needs.build-check.outputs.version }}"
+          SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
          CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
          VARIANT_SUFFIX="${{ matrix.suffix }}"

@@ -344,8 +343,8 @@ jobs:
              ;;
          esac

-          echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
-          echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT"
+          echo "docker_release=$DOCKER_RELEASE" >> $GITHUB_OUTPUT
+          echo "docker_channel=$DOCKER_CHANNEL" >> $GITHUB_OUTPUT

          echo "🐳 Docker build parameters:"
          echo "  - Original version: $VERSION"
@@ -377,7 +376,7 @@ jobs:
          fi

          # Output tags
-          echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
+          echo "tags=$TAGS" >> $GITHUB_OUTPUT

          # Generate labels
          LABELS="org.opencontainers.image.title=RustFS"
@@ -388,7 +387,7 @@ jobs:
          LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
          LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"

-          echo "labels=$LABELS" >> "$GITHUB_OUTPUT"
+          echo "labels=$LABELS" >> $GITHUB_OUTPUT

          echo "🐳 Generated Docker tags:"
          echo "$TAGS" | tr ',' '\n' | sed 's/^/  - /'
@@ -1,70 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# MinIO on-disk interop: prove RustFS reads MinIO-written erasure-coded SSE
-# objects with byte-identical data and correct logical size.
-#
-# This is NOT a PR gate. The fixtures are real MinIO backend trees generated on
-# the fly (they are gitignored, never committed), so the job regenerates them
-# each run with Docker and then runs the `#[ignore]` reader tests in
-# crates/ecstore/tests/minio_generated_read_test.rs.
-#
-# Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
-# unlike the self-hosted fleet, whose pods drift in Docker/pip availability
-# (see the infra note in e2e-s3tests.yml). Nightly + manual only.
-name: minio-interop
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Nightly at 03:17 UTC (offset from other nightly jobs).
-    - cron: "17 3 * * *"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  minio-interop:
-    name: MinIO interop (EC + SSE read parity)
-    # Skip on forks: needs the repo's runners and is not a contributor gate.
-    if: github.repository == 'rustfs/rustfs'
-    runs-on: ubuntu-latest
-    timeout-minutes: 40
-    env:
-      # Fixed 32-byte test KMS key baked into the fixture lab; not a secret.
-      RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-minio-interop
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Generate real MinIO fixtures via Docker
-        run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh
-
-      - name: Run MinIO interop reader tests
-        run: |
-          cargo nextest run --run-ignored ignored-only \
-            -p rustfs-ecstore --features rio-v2 \
-            -E 'binary(minio_generated_read_test)'
@@ -135,10 +135,6 @@ jobs:
        run: |
          docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
          docker rm -f rustfs-mint >/dev/null 2>&1 || true
-          # The four disks share one physical device on the runner (a single
-          # loopback filesystem), so the local physical-disk-independence guard
-          # would refuse to start. Bypass it — this is the CI use case the guard
-          # explicitly sanctions via RUSTFS_UNSAFE_BYPASS_DISK_CHECK.
          docker run -d --name rustfs-mint \
            --network rustfs-net \
            -p 9000:9000 \
@@ -146,7 +142,6 @@ jobs:
            -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
            -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
            -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
-            -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
            -v /tmp/rustfs-mint:/data \
            rustfs-ci

@@ -41,10 +41,6 @@ on:
        type: boolean
  pull_request:
    types: [labeled, synchronize, reopened]
-  push:
-    # Every main commit pre-builds and caches its release binary (perf-3) so the
-    # nightly A/B restores a ready baseline instead of paying the double build.
-    branches: [main]

 permissions:
  contents: read
@@ -62,81 +58,22 @@ env:
  RUST_BACKTRACE: 1

 jobs:
-  # perf-3: on every push to main, build the release binary once and cache it
-  # keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
-  # perf-7 PR gate) restore this instead of paying the ~32min-per-side source
-  # build. That double build is what pushed the expanded 24-cell nightly past its
-  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
-  # builds off the shared cargo cache keep each push cheap, and building on the
-  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
-  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
-  # are cancelled ~7/8 of the time and are not a reliable baseline.
-  build-baseline-cache:
-    name: Build + cache baseline binary
-    if: github.event_name == 'push'
-    runs-on: sm-standard-2
-    # Latest-wins: consumers only ever restore the binary for the *current*
-    # origin/main tip, so when pushes land faster than the ~65min build, a
-    # superseded build's output is dead weight — cancel it instead of stacking
-    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
-    # most costs one same-commit self-heal in the A/B job.
-    concurrency:
-      group: perf-baseline-build-main
-      cancel-in-progress: true
-    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
-    # single release build past 60min on this runner — every cache build on
-    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
-    # time of 1h0m0s") and the cache never populated. The measured binary must
-    # keep the production profile, so the budget absorbs the build instead.
-    timeout-minutes: 100
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build release rustfs
-        run: cargo build --release --bin rustfs
-
-      - name: Stage binary for cache
-        run: |
-          set -euo pipefail
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-
-      - name: Cache baseline binary by SHA
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ github.sha }}
-
  warp-ab:
    name: Warp A/B budget gate
-    # Always run on schedule / manual dispatch. Opt-in on PRs: only when the
-    # `perf-ab` label is present, and for `labeled` events only when the label
-    # being added is `perf-ab` itself (adding an unrelated label to an opted-in
-    # PR must not re-run the gate). Never on push — that event only feeds
-    # build-baseline-cache above.
+    # Opt-in on PRs: only run when the `perf-ab` label is present, and for
+    # `labeled` events only when the label being added is `perf-ab` itself —
+    # adding an unrelated label to an opted-in PR must not re-run the gate.
+    # Always run on schedule / manual dispatch.
    if: >-
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch' ||
-      (github.event_name == 'pull_request' &&
-       contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
+      github.event_name != 'pull_request' ||
+      (contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
       (github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
    runs-on: sm-standard-2
-    # With perf-3's cached baseline binary the common (cache-hit) nightly is
-    # measurement-only and finishes well under 50min. This ceiling stays
-    # generous only to absorb the same-commit cache-miss self-heal (~65min
-    # single build with the post-#4806 LTO profile + measurement). A timeout
-    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
-    # not just failure). perf-6 recalibrates the budget once the noise study
-    # lands.
+    # Phase-0 stopgap: the baseline+candidate release double-build alone is
+    # ~65min on this runner, so 90min left no room for a real full-matrix
+    # measurement (the earlier nightly runs only ever failed *before*
+    # measuring). 120min gives the 24-cell short matrix headroom until perf-3
+    # caches the baseline binary and restores a tighter budget.
    timeout-minutes: 120
    steps:
      - name: Checkout repository
@@ -173,106 +110,21 @@ jobs:
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

-      # perf-3: resolve the commits so the cache can be keyed by SHA. The
-      # baseline is origin/main; the candidate is the checked-out ref. On the
-      # nightly (checkout == main) they are the same commit, so one cached binary
-      # serves both phases and the run does zero source builds.
-      - name: Resolve baseline / candidate commits
-        id: commits
-        run: |
-          set -euo pipefail
-          baseline_sha="$(git rev-parse origin/main)"
-          candidate_sha="$(git rev-parse HEAD)"
-          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
-          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
-          echo "baseline commit: $baseline_sha"
-          echo "candidate commit: $candidate_sha"
-
-      # Exact-key restore of the baseline binary built by build-baseline-cache
-      # when origin/main last landed. A miss (binary evicted or not built yet)
-      # leaves cache-hit unset and the rig falls back to a source build.
-      - name: Restore cached baseline binary
-        id: baseline_cache
-        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
-
-      # Self-heal: on a nightly/dispatch run where the candidate commit IS the
-      # baseline commit, a cache miss would make the rig build the same commit
-      # twice (~65min per side with the post-#4806 LTO profile — no job budget
-      # fits that). Build it once here, reuse it for both phases, and save it
-      # back to the cache so the next run hits.
-      - name: Build baseline on cache miss (same-commit self-heal)
-        id: selfheal
-        if: >-
-          steps.baseline_cache.outputs.cache-hit != 'true' &&
-          steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
-        run: |
-          set -euo pipefail
-          cargo build --release --bin rustfs
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-          echo "built=true" >> "$GITHUB_OUTPUT"
-
-      - name: Save self-healed baseline to cache
-        if: steps.selfheal.outputs.built == 'true'
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
-
      - name: Run warp A/B and gate
        id: ab
        run: |
          set -euo pipefail
-          # Budget note: with perf-3's cached baseline the nightly does no source
-          # build on a cache hit, so the wall-clock is dominated by the short warp
-          # matrix — duration/rounds/cooldown are kept small to fit all 24 cells
-          # (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
-          # --health-timeout 180 outlasts the server's own 120s startup-readiness
-          # budget, which the rig's previous 60s health poll undershot (the first
-          # two nightly failures). perf-6 recalibrates these once the noise study
-          # lands.
+          # Budget note: the baseline+candidate release double-build (~65 min,
+          # cached away later by perf-3) dominates the 90-min job, so the
+          # measurement runs a short warp matrix — duration/rounds/cooldown are
+          # kept small to fit all 24 cells (6 workloads x 2 phases x 2 drive-sync)
+          # under budget rather than dropping cells. --health-timeout 180 outlasts
+          # the server's own 120s startup-readiness budget, which is what the
+          # rig's previous 60s health poll undershot (the first two nightly
+          # failures). perf-6 will recalibrate these once the pipeline is green.
          duration="${{ github.event.inputs.duration || '12s' }}"
-          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
-          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
-          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
-          selfheal_built="${{ steps.selfheal.outputs.built }}"
-
-          args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
-
-          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" ]]; then
-            chmod +x baseline-bin/rustfs
-            base_bin="$PWD/baseline-bin/rustfs"
-            args+=(--baseline-bin "$base_bin")
-            if [[ "$baseline_hit" == "true" ]]; then
-              base_src="actions-cache (rustfs-baseline-$baseline_sha)"
-            else
-              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
-            fi
-            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
-              # Nightly on main: the candidate is the same commit as the baseline,
-              # so reuse the one binary for both phases and skip all builds.
-              args+=(--candidate-bin "$base_bin" --skip-build)
-              cand_src="same binary as baseline (same commit)"
-            else
-              cand_src="source build of the checked-out ref"
-            fi
-          else
-            # Cache miss with candidate != baseline (opt-in PR gate only): fall
-            # back to the source double-build. With the post-#4806 LTO profile
-            # this will overrun the job budget and alert; rerun once the push
-            # cache build for origin/main has completed, or wait for perf-7's
-            # merge-base caching.
-            args+=(--baseline-ref origin/main)
-            base_src="source build of origin/main (cache miss)"
-            cand_src="source build of the checked-out ref"
-          fi
-
-          args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
-          args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")
-
+          args=(--baseline-ref origin/main
+            --duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
          if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
            args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
          fi
@@ -349,8 +201,12 @@ jobs:
        run: |
          gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"

-      # Scheduled failure alerting is handled by the alert-on-failure job below
-      # (perf-2 consuming ci-8's schedule-failure-issue composite action).
+      # TODO(ci-8/perf-2): scheduled/dispatch failures are currently silent. Once
+      # ci-8 lands the .github/actions/schedule-failure-issue composite action,
+      # perf-2 adds a step here guarded by
+      #   if: failure() && github.event_name != 'pull_request'
+      # that calls it (label perf-nightly-failure, append to an existing open
+      # issue) instead of hand-rolling gh CLI dedup. Do not implement it here.

      - name: Enforce gate
        if: always()
@@ -368,16 +224,7 @@ jobs:
    # `always()` is required: without it this job is skipped when a needed
    # job fails. Alerts only for scheduled (nightly) runs (backlog#1149
    # ci-8); PR and manual dispatch failures are already watched by a human.
-    # `cancelled` is included alongside `failure` on purpose: a job that hits
-    # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
-    # timeouts went silent precisely because the guard was failure-only. The
-    # composite action already reports cancelled/timed-out jobs in the issue
-    # body. (Scheduled runs get a unique concurrency group with
-    # cancel-in-progress off, so a cancellation here means a timeout/manual
-    # abort, never a superseding run.)
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
@@ -1,7 +1,45 @@
 {
    "version": "0.2.0",
    "configurations": [
-          {
+        {
+            "name": "Debug RustFS observability (OTLP)",
+            "type": "lldb",
+            "request": "launch",
+            "cargo": {
+                "args": [
+                    "build",
+                    "--bin=rustfs",
+                    "--package=rustfs"
+                ],
+                "filter": {
+                    "name": "rustfs",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=info,iam=info",
+                "RUST_BACKTRACE": "full",
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/observability/data{1...4}",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
+                "RUSTFS_OBS_ENDPOINT": "http://127.0.0.1:4318",
+                "RUSTFS_OBS_TRACES_EXPORT_ENABLED": "true",
+                "RUSTFS_OBS_METRICS_EXPORT_ENABLED": "true",
+                "RUSTFS_OBS_LOGS_EXPORT_ENABLED": "true",
+                "RUSTFS_OBS_USE_STDOUT": "true",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/observability/logs",
+                "RUSTFS_OBS_METER_INTERVAL": "5",
+                "RUSTFS_OBS_SERVICE_NAME": "rustfs-observability-local",
+                "RUSTFS_OBS_ENVIRONMENT": "development"
+            }
+        },
+        {
            "type": "lldb",
            "request": "launch",
            "name": "Debug(only) executable 'rustfs'",
@@ -11,11 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Helm Ingress**: `customAnnotations` are now merged with class-specific annotations (nginx/traefik) instead of being ignored when `ingress.className` is set.

 ### Added
- **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
-  - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
-  - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window
-  - Pre-flight stream validation, and a bounded failed-events store (count and TTL). Only a non-retryable rejection is recorded in the failed-events store. A retryable condition keeps the entry on the live queue until it is delivered
-  - Operator guide at `docs/operations/nats-jetstream.md`
 - **OpenStack Keystone Authentication Integration**: Full support for OpenStack Keystone authentication via X-Auth-Token headers
  - Tower-based middleware (`KeystoneAuthLayer`) self-contained within `rustfs-keystone` crate
  - Task-local storage for async-safe credential passing between middleware and auth handlers
@@ -31,8 +31,6 @@ make build-docker BUILD_OS=ubuntu22.04
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
 - CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
-  policy: [docs/testing/README.md](docs/testing/README.md)
 - Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
  [docs/operations/tier-ilm-debugging.md](docs/operations/tier-ilm-debugging.md)

@@ -68,8 +68,6 @@ make pre-pr

 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.

-> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
-
 ### 🔒 Automated Pre-commit Hooks
 #### What `make pre-commit` and `make pre-pr` actually run

@@ -52,7 +52,6 @@ members = [
    "crates/extension-schema", # Extension schema contracts
    "crates/signer", # client signer
    "crates/storage-api", # Storage API contracts
-    "crates/test-utils", # Shared test bootstrap helpers (dev-dependency only)
    "crates/targets", # Target-specific configurations and utilities
    "crates/trusted-proxies", # Trusted proxies management
    "crates/tls-runtime", # Project-wide TLS runtime foundation
@@ -68,7 +67,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.96.0"
-version = "1.0.0-beta.9"
+version = "1.0.0-beta.8"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -85,51 +84,50 @@ redundant_clone = "warn"

 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-beta.9" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.9" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.9" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.9" }
-rustfs-common = { path = "crates/common", version = "1.0.0-beta.9" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.9" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-beta.9" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.9" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.9" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.9" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.9" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.9" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.9" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.9" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.9" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.9" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.9" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.9" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.9" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.9" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.9" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.9" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.9" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.9" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.9" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.9" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.9" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.9" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.9" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.9" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.9" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.9" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.9" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.9" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.9" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.9" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.9" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.9" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.9" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.9" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.9" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.9" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.9" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.9" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.9" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.8" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.8" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.8" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.8" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.8" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.8" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.8" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.8" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.8" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.8" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.8" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.8" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.8" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.8" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.8" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.8" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.8" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.8" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.8" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.8" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.8" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.8" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.8" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.8" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.8" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.8" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.8" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.8" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.8" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.8" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.8" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.8" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.8" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.8" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.8" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.8" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.8" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.8" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.8" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.8" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.8" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.8" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.8" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.8" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -151,12 +149,12 @@ hyper = { version = "1.10.1", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.2"
-http-body = "1.1.0"
-http-body-util = "0.1.4"
+http-body = "1.0.1"
+http-body-util = "0.1.3"
 minlz = "1.2.3"
 reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 rustfs-kafka-async = { version = "1.2.0" }
-socket2 = { version = "0.6.5", features = ["all"] }
+socket2 = { version = "0.6.4", features = ["all"] }
 tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
@@ -198,7 +196,7 @@ jsonwebtoken = { version = "10.4.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false, features = ["accept-rfc3339-timestamps"] }
 pbkdf2 = "0.13.0"
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { version = "0.23.42", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls = { version = "0.23.41", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.0"
 sha1 = "0.11.0"
@@ -246,8 +244,8 @@ enumset = "1.1.13"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.16.0"
-google-cloud-auth = "1.14.0"
+google-cloud-storage = "1.15.0"
+google-cloud-auth = "1.13.0"
 hashbrown = { version = "0.17.1", features = ["serde", "rayon"] }
 hex = "0.4.3"
 hex-simd = "0.8.0"
@@ -276,16 +274,16 @@ pretty_assertions = "1.4.1"
 rand = { version = "0.10.2", features = ["serde"] }
 ratelimit = "0.10.1"
 rayon = "1.12.0"
-reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.0", features = ["simd-accel"] }
+reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "7.0.1", features = ["simd-accel"] }
 #reed-solomon-erasure = { version = "6.0", features = ["simd-accel"], git = "https://github.com/houseme/reed-solomon-erasure",rev = "main" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.0" }
 rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
-redis = { version = "1.4.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
+redis = { version = "1.3.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "ce69c3f10824535c7c24b2f71cdb2aaa4dffb5e0", features = ["minio"] }
+s3s = { version = "0.14.1", features = ["minio"] }
 serial_test = "3.5.0"
 shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.3"
@@ -311,7 +309,7 @@ uuid = { version = "1.23.5", features = ["v4", "fast-rng", "macro-diagnostics"]
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 windows = { version = "0.62.2" }
-xxhash-rust = { version = "0.8.17", features = ["xxh64", "xxh3"] }
+xxhash-rust = { version = "0.8.16", features = ["xxh64", "xxh3"] }
 zip = "8.6.0"
 zstd = "0.13.3"

@@ -329,7 +327,7 @@ pyroscope = { version = "2.1.0", features = ["backend-pprof-rs"] }
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "10.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.8"
 russh = { version = "0.62.2", features = ["serde"] }
 russh-sftp = "2.3.0"
@@ -355,34 +353,12 @@ debug = "line-tables-only"

 [profile.release]
 opt-level = 3
-lto = "thin"
-codegen-units = 1
-debug = 0
-split-debuginfo = "off"
-strip = "symbols"

 [profile.production]
 inherits = "release"
+lto = "fat"
+codegen-units = 1

 [profile.profiling]
 inherits = "release"
 debug = true
-strip = "none"
-
-# Pin hyper to a revision that carries the HTTP/1 "flush buffered data before
-# shutdown" fix (hyperium/hyper#4018, commit 72046cc7). This lands as a
-# `[patch.crates-io]` entry — not on the `hyper` workspace dependency — so that
-# every consumer in the tree, including the transitive `hyper-util` server path
-# (`conn::auto` / `GracefulShutdown`) that actually drives our connections,
-# resolves to the fixed hyper rather than the buggy crates.io copy.
-#
-# hyper <= 1.10.1 can call `poll_shutdown()` on the socket while response bytes
-# are still buffered (a prior `poll_flush()` returned `Poll::Pending` and the
-# result was discarded). A backpressured / slow-reading peer then receives a
-# graceful FIN before the full Content-Length body is flushed, which standard S3
-# clients (minio-go / warp) report as `unexpected EOF` on large-object GET under
-# load. The fix is not in any crates.io release yet as of hyper 1.10.1; drop
-# this patch once a released version (> 1.10.1) contains commit 72046cc7.
-# See rustfs/backlog#1232.
-[patch.crates-io]
-hyper = { git = "https://github.com/hyperium/hyper.git", rev = "ccc1e850dc0cda3e71b0acd11f60ca3d48d09034" }
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-FROM rust:1.97-trixie
+FROM rust:1.95-trixie

 RUN set -eux; \
    export DEBIAN_FRONTEND=noninteractive; \
@@ -32,7 +32,7 @@ ARG RUSTFS_BUILD_FEATURES=""
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97-trixie AS builder
+FROM rust:1.95-trixie AS builder

 # Re-declare args after FROM
 ARG TARGETPLATFORM
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.9
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
 ```

 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest

 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.9
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
 ```

 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
@@ -217,7 +217,7 @@ setup_rust_environment() {
    # Set up environment variables for musl targets
    if [[ "$PLATFORM" == *"musl"* ]]; then
        print_message $YELLOW "Setting up environment for musl target..."
-        export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-feature=-crt-static"
+        export RUSTFLAGS="--cfg tokio_unstable -C target-feature=-crt-static"

        # For cargo-zigbuild, set up additional environment variables
        if command -v cargo-zigbuild &> /dev/null; then
@@ -434,7 +434,7 @@ build_binary() {
        fi
    else
        # Native compilation
-        build_cmd="RUSTFLAGS='${RUSTFLAGS:+$RUSTFLAGS }-Clink-arg=-lm' cargo build"
+        build_cmd="RUSTFLAGS='--cfg tokio_unstable -Clink-arg=-lm' cargo build"
    fi

    if [ "$BUILD_TYPE" = "release" ]; then
@@ -32,7 +32,6 @@ const EVENT_AUDIT_BATCH_DISPATCH_COMPLETED: &str = "audit_batch_dispatch_complet
 const EVENT_AUDIT_TARGET_STATE_CHANGED: &str = "audit_target_state_changed";
 const EVENT_AUDIT_REPLAY_DELIVERED: &str = "audit_replay_delivered";
 const EVENT_AUDIT_REPLAY_RETRY_SCHEDULED: &str = "audit_replay_retry_scheduled";
-const EVENT_AUDIT_REPLAY_RETRY_EXHAUSTED: &str = "audit_replay_retry_exhausted";
 const EVENT_AUDIT_REPLAY_DROPPED: &str = "audit_replay_dropped";
 const EVENT_AUDIT_REPLAY_STREAM_STATUS: &str = "audit_replay_stream_status";

@@ -282,7 +281,6 @@ impl AuditPipeline {
                let delivery = target.delivery_snapshot();
                AuditTargetMetricSnapshot {
                    failed_messages: delivery.failed_messages,
-                    failed_store_length: delivery.failed_store_length,
                    queue_length: delivery.queue_length,
                    target_id: target.id().to_string(),
                    total_messages: delivery.total_messages,
@@ -465,16 +463,18 @@ impl AuditRuntimeFacade {
                            target.record_final_failure();
                            observability::record_target_failure();
                        }
-                        ReplayEvent::RetryExhausted { detail, key, target } => {
+                        ReplayEvent::RetryExhausted { key, target } => {
                            warn!(
-                                event = EVENT_AUDIT_REPLAY_RETRY_EXHAUSTED,
+                                event = EVENT_AUDIT_REPLAY_DROPPED,
                                component = LOG_COMPONENT_AUDIT,
                                subsystem = LOG_SUBSYSTEM_PIPELINE,
                                target_id = %target.id(),
                                replay_key = %key,
-                                error = %detail,
-                                "audit replay retry budget exhausted, entry stays queued and retries"
+                                reason = "retry_exhausted",
+                                "audit replay delivery"
                            );
+                            target.record_final_failure();
+                            observability::record_target_failure();
                        }
                        ReplayEvent::UnreadableEntry { key, error, target } => {
                            warn!(
@@ -30,7 +30,6 @@ const EVENT_AUDIT_CONFIG_RELOADED: &str = "audit_config_reloaded";
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct AuditTargetMetricSnapshot {
    pub failed_messages: u64,
-    pub failed_store_length: u64,
    pub queue_length: u64,
    pub target_id: String,
    pub total_messages: u64,
@@ -33,7 +33,6 @@ base64-simd = { workspace = true }
 md-5 = { workspace = true }
 sha1 = { workspace = true }
 sha2 = { workspace = true }
-xxhash-rust = { workspace = true }

 [dev-dependencies]
 pretty_assertions = { workspace = true }
@@ -36,7 +36,7 @@ impl fmt::Display for UnknownChecksumAlgorithmError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
-            r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256", "sha512", "xxhash3", "xxhash64", "xxhash128")"#,
+            r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256")"#,
            self.checksum_algorithm
        )
    }
@@ -16,20 +16,13 @@ use crate::base64;
 use http::header::{HeaderMap, HeaderValue};

 use crate::Crc64Nvme;
-use crate::{
-    CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256, Sha512,
-    Xxhash3, Xxhash64, Xxhash128,
-};
+use crate::{CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256};

 pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
 pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
 pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
 pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256";
 pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme";
-pub const SHA_512_HEADER_NAME: &str = "x-amz-checksum-sha512";
-pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
-pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
-pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";

 #[allow(dead_code)]
 pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
@@ -92,30 +85,6 @@ impl HttpChecksum for Sha256 {
    }
 }

-impl HttpChecksum for Sha512 {
-    fn header_name(&self) -> &'static str {
-        SHA_512_HEADER_NAME
-    }
-}
-
-impl HttpChecksum for Xxhash3 {
-    fn header_name(&self) -> &'static str {
-        XXHASH_3_HEADER_NAME
-    }
-}
-
-impl HttpChecksum for Xxhash64 {
-    fn header_name(&self) -> &'static str {
-        XXHASH_64_HEADER_NAME
-    }
-}
-
-impl HttpChecksum for Xxhash128 {
-    fn header_name(&self) -> &'static str {
-        XXHASH_128_HEADER_NAME
-    }
-}
-
 impl HttpChecksum for Md5 {
    fn header_name(&self) -> &'static str {
        MD5_HEADER_NAME
@@ -35,10 +35,6 @@ pub const CRC_32_C_NAME: &str = "crc32c";
 pub const CRC_64_NVME_NAME: &str = "crc64nvme";
 pub const SHA_1_NAME: &str = "sha1";
 pub const SHA_256_NAME: &str = "sha256";
-pub const SHA_512_NAME: &str = "sha512";
-pub const XXHASH_3_NAME: &str = "xxhash3";
-pub const XXHASH_64_NAME: &str = "xxhash64";
-pub const XXHASH_128_NAME: &str = "xxhash128";
 pub const MD5_NAME: &str = "md5";

 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -50,10 +46,6 @@ pub enum ChecksumAlgorithm {
    Sha1,
    Sha256,
    Crc64Nvme,
-    Sha512,
-    Xxhash3,
-    Xxhash64,
-    Xxhash128,
 }

 impl FromStr for ChecksumAlgorithm {
@@ -70,14 +62,6 @@ impl FromStr for ChecksumAlgorithm {
            Ok(Self::Sha256)
        } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) {
            Ok(Self::Crc64Nvme)
-        } else if checksum_algorithm.eq_ignore_ascii_case(SHA_512_NAME) {
-            Ok(Self::Sha512)
-        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_3_NAME) {
-            Ok(Self::Xxhash3)
-        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_64_NAME) {
-            Ok(Self::Xxhash64)
-        } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_128_NAME) {
-            Ok(Self::Xxhash128)
        } else {
            Err(UnknownChecksumAlgorithmError::new(checksum_algorithm))
        }
@@ -92,10 +76,6 @@ impl ChecksumAlgorithm {
            Self::Crc64Nvme => Box::<Crc64Nvme>::default(),
            Self::Sha1 => Box::<Sha1>::default(),
            Self::Sha256 => Box::<Sha256>::default(),
-            Self::Sha512 => Box::<Sha512>::default(),
-            Self::Xxhash3 => Box::<Xxhash3>::default(),
-            Self::Xxhash64 => Box::<Xxhash64>::default(),
-            Self::Xxhash128 => Box::<Xxhash128>::default(),
        }
    }

@@ -106,10 +86,6 @@ impl ChecksumAlgorithm {
            Self::Crc64Nvme => CRC_64_NVME_NAME,
            Self::Sha1 => SHA_1_NAME,
            Self::Sha256 => SHA_256_NAME,
-            Self::Sha512 => SHA_512_NAME,
-            Self::Xxhash3 => XXHASH_3_NAME,
-            Self::Xxhash64 => XXHASH_64_NAME,
-            Self::Xxhash128 => XXHASH_128_NAME,
        }
    }
 }
@@ -309,165 +285,6 @@ impl Checksum for Sha256 {
        Self::size()
    }
 }
-
-#[derive(Debug, Default)]
-struct Sha512 {
-    hasher: sha2::Sha512,
-}
-
-impl Sha512 {
-    fn update(&mut self, bytes: &[u8]) {
-        use sha2::Digest;
-        self.hasher.update(bytes);
-    }
-
-    fn finalize(self) -> Bytes {
-        use sha2::Digest;
-        Bytes::copy_from_slice(self.hasher.finalize().as_slice())
-    }
-
-    fn size() -> u64 {
-        use sha2::Digest;
-        sha2::Sha512::output_size() as u64
-    }
-}
-
-impl Checksum for Sha512 {
-    fn update(&mut self, bytes: &[u8]) {
-        Self::update(self, bytes);
-    }
-    fn finalize(self: Box<Self>) -> Bytes {
-        Self::finalize(*self)
-    }
-    fn size(&self) -> u64 {
-        Self::size()
-    }
-}
-
-/// XXH3 (64-bit) hasher with the canonical seed of 0.
-///
-/// The raw digest is a `u64` serialized as 8 big-endian bytes so that the value
-/// matches the server-side (`rustfs-rio`) computation for the same algorithm.
-struct Xxhash3 {
-    hasher: xxhash_rust::xxh3::Xxh3,
-}
-
-impl Default for Xxhash3 {
-    fn default() -> Self {
-        Self {
-            hasher: xxhash_rust::xxh3::Xxh3::new(),
-        }
-    }
-}
-
-impl Xxhash3 {
-    fn update(&mut self, bytes: &[u8]) {
-        self.hasher.update(bytes);
-    }
-
-    fn finalize(self) -> Bytes {
-        Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice())
-    }
-
-    fn size() -> u64 {
-        8
-    }
-}
-
-impl Checksum for Xxhash3 {
-    fn update(&mut self, bytes: &[u8]) {
-        Self::update(self, bytes)
-    }
-    fn finalize(self: Box<Self>) -> Bytes {
-        Self::finalize(*self)
-    }
-    fn size(&self) -> u64 {
-        Self::size()
-    }
-}
-
-/// XXH3 (128-bit) hasher with the canonical seed of 0.
-///
-/// The raw digest is a `u128` serialized as 16 big-endian bytes.
-struct Xxhash128 {
-    hasher: xxhash_rust::xxh3::Xxh3,
-}
-
-impl Default for Xxhash128 {
-    fn default() -> Self {
-        Self {
-            hasher: xxhash_rust::xxh3::Xxh3::new(),
-        }
-    }
-}
-
-impl Xxhash128 {
-    fn update(&mut self, bytes: &[u8]) {
-        self.hasher.update(bytes);
-    }
-
-    fn finalize(self) -> Bytes {
-        Bytes::copy_from_slice(self.hasher.digest128().to_be_bytes().as_slice())
-    }
-
-    fn size() -> u64 {
-        16
-    }
-}
-
-impl Checksum for Xxhash128 {
-    fn update(&mut self, bytes: &[u8]) {
-        Self::update(self, bytes)
-    }
-    fn finalize(self: Box<Self>) -> Bytes {
-        Self::finalize(*self)
-    }
-    fn size(&self) -> u64 {
-        Self::size()
-    }
-}
-
-/// XXH64 hasher with the canonical seed of 0.
-///
-/// The raw digest is a `u64` serialized as 8 big-endian bytes.
-struct Xxhash64 {
-    hasher: xxhash_rust::xxh64::Xxh64,
-}
-
-impl Default for Xxhash64 {
-    fn default() -> Self {
-        Self {
-            hasher: xxhash_rust::xxh64::Xxh64::new(0),
-        }
-    }
-}
-
-impl Xxhash64 {
-    fn update(&mut self, bytes: &[u8]) {
-        self.hasher.update(bytes);
-    }
-
-    fn finalize(self) -> Bytes {
-        Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice())
-    }
-
-    fn size() -> u64 {
-        8
-    }
-}
-
-impl Checksum for Xxhash64 {
-    fn update(&mut self, bytes: &[u8]) {
-        Self::update(self, bytes)
-    }
-    fn finalize(self: Box<Self>) -> Bytes {
-        Self::finalize(*self)
-    }
-    fn size(&self) -> u64 {
-        Self::size()
-    }
-}
-
 #[allow(dead_code)]
 #[derive(Debug, Default)]
 struct Md5 {
@@ -638,97 +455,4 @@ mod tests {
        let error = "MD5".parse::<ChecksumAlgorithm>().expect_err("md5 should not parse");
        assert_eq!("MD5", error.checksum_algorithm());
    }
-
-    #[test]
-    fn test_additional_algorithms_parse_and_round_trip() {
-        // The AWS 2026-04 additional checksum algorithms must be recognised
-        // (case-insensitively) and round-trip through as_str().
-        for (name, expected) in [
-            ("sha512", ChecksumAlgorithm::Sha512),
-            ("SHA512", ChecksumAlgorithm::Sha512),
-            ("xxhash3", ChecksumAlgorithm::Xxhash3),
-            ("XXHASH3", ChecksumAlgorithm::Xxhash3),
-            ("xxhash64", ChecksumAlgorithm::Xxhash64),
-            ("xxhash128", ChecksumAlgorithm::Xxhash128),
-        ] {
-            let parsed = name.parse::<ChecksumAlgorithm>().expect("algorithm should parse");
-            assert_eq!(parsed, expected);
-            assert_eq!(expected.as_str().parse::<ChecksumAlgorithm>().unwrap(), expected);
-        }
-    }
-
-    #[test]
-    fn test_unknown_algorithm_never_panics_and_fails_closed() {
-        // Fail-closed contract: an unknown or garbage algorithm name must return
-        // an error instead of panicking or silently substituting another hasher.
-        for name in ["", "xxhash", "sha3", "crc16", "not-a-real-algo", "🦀"] {
-            assert!(name.parse::<ChecksumAlgorithm>().is_err(), "unknown algorithm {name:?} must fail closed");
-        }
-    }
-
-    #[test]
-    fn test_sha512_matches_direct_computation() {
-        use crate::Sha512;
-        use crate::http::SHA_512_HEADER_NAME;
-        use sha2::{Digest, Sha512 as Sha512Ref};
-
-        let mut checksum = Sha512::default();
-        checksum.update(TEST_DATA.as_bytes());
-        let header = Box::new(checksum).headers();
-        let encoded = header.get(SHA_512_HEADER_NAME).expect("sha512 header present");
-        let got = base64_encoded_checksum_to_hex_string(encoded);
-
-        let mut reference = Sha512Ref::new();
-        reference.update(TEST_DATA.as_bytes());
-        let expected = reference.finalize().iter().fold(String::from("0x"), |mut acc, b| {
-            write!(acc, "{b:02X?}").unwrap();
-            acc
-        });
-        assert_eq!(got, expected);
-    }
-
-    #[test]
-    fn test_xxhash3_matches_direct_computation_big_endian_seed0() {
-        use crate::Xxhash3;
-        use xxhash_rust::xxh3::Xxh3;
-
-        let mut checksum = Xxhash3::default();
-        checksum.update(TEST_DATA.as_bytes());
-        let raw = Box::new(checksum).finalize();
-
-        let mut reference = Xxh3::new();
-        reference.update(TEST_DATA.as_bytes());
-        assert_eq!(raw.len(), 8);
-        assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice());
-    }
-
-    #[test]
-    fn test_xxhash128_matches_direct_computation_big_endian_seed0() {
-        use crate::Xxhash128;
-        use xxhash_rust::xxh3::Xxh3;
-
-        let mut checksum = Xxhash128::default();
-        checksum.update(TEST_DATA.as_bytes());
-        let raw = Box::new(checksum).finalize();
-
-        let mut reference = Xxh3::new();
-        reference.update(TEST_DATA.as_bytes());
-        assert_eq!(raw.len(), 16);
-        assert_eq!(&raw[..], reference.digest128().to_be_bytes().as_slice());
-    }
-
-    #[test]
-    fn test_xxhash64_matches_direct_computation_big_endian_seed0() {
-        use crate::Xxhash64;
-        use xxhash_rust::xxh64::Xxh64;
-
-        let mut checksum = Xxhash64::default();
-        checksum.update(TEST_DATA.as_bytes());
-        let raw = Box::new(checksum).finalize();
-
-        let mut reference = Xxh64::new(0);
-        reference.update(TEST_DATA.as_bytes());
-        assert_eq!(raw.len(), 8);
-        assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice());
-    }
 }
@@ -25,11 +25,8 @@ pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KE
 pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
 pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
 pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
-pub const ENV_AUDIT_NATS_JETSTREAM_ENABLE: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_ENABLE";
-pub const ENV_AUDIT_NATS_JETSTREAM_STREAM_NAME: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_STREAM_NAME";
-pub const ENV_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "RUSTFS_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS";

-pub const ENV_AUDIT_NATS_KEYS: &[&str; 16] = &[
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
    ENV_AUDIT_NATS_ENABLE,
    ENV_AUDIT_NATS_ADDRESS,
    ENV_AUDIT_NATS_SUBJECT,
@@ -43,9 +40,6 @@ pub const ENV_AUDIT_NATS_KEYS: &[&str; 16] = &[
    ENV_AUDIT_NATS_TLS_REQUIRED,
    ENV_AUDIT_NATS_QUEUE_DIR,
    ENV_AUDIT_NATS_QUEUE_LIMIT,
-    ENV_AUDIT_NATS_JETSTREAM_ENABLE,
-    ENV_AUDIT_NATS_JETSTREAM_STREAM_NAME,
-    ENV_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS,
 ];

 pub const AUDIT_NATS_KEYS: &[&str] = &[
@@ -62,8 +56,5 @@ pub const AUDIT_NATS_KEYS: &[&str] = &[
    crate::NATS_TLS_REQUIRED,
    crate::NATS_QUEUE_DIR,
    crate::NATS_QUEUE_LIMIT,
-    crate::NATS_JETSTREAM_ENABLE,
-    crate::NATS_JETSTREAM_STREAM_NAME,
-    crate::NATS_JETSTREAM_ACK_TIMEOUT_SECS,
    crate::COMMENT_KEY,
 ];
@@ -79,13 +79,6 @@ pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const NATS_TLS_REQUIRED: &str = "tls_required";
 pub const NATS_QUEUE_DIR: &str = "queue_dir";
 pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
-pub const NATS_JETSTREAM_ENABLE: &str = "jetstream_enable";
-pub const NATS_JETSTREAM_STREAM_NAME: &str = "jetstream_stream_name";
-pub const NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "jetstream_ack_timeout_secs";
-
-pub const NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS: u64 = 30;
-pub const NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS: u64 = 10;
-pub const NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS: u64 = 120;

 pub const PULSAR_BROKER: &str = "broker";
 pub const PULSAR_TOPIC: &str = "topic";
@@ -26,9 +26,6 @@ pub const NOTIFY_NATS_KEYS: &[&str] = &[
    crate::NATS_TLS_REQUIRED,
    crate::NATS_QUEUE_DIR,
    crate::NATS_QUEUE_LIMIT,
-    crate::NATS_JETSTREAM_ENABLE,
-    crate::NATS_JETSTREAM_STREAM_NAME,
-    crate::NATS_JETSTREAM_ACK_TIMEOUT_SECS,
    crate::COMMENT_KEY,
 ];

@@ -45,11 +42,8 @@ pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_
 pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
 pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
 pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
-pub const ENV_NOTIFY_NATS_JETSTREAM_ENABLE: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_ENABLE";
-pub const ENV_NOTIFY_NATS_JETSTREAM_STREAM_NAME: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_STREAM_NAME";
-pub const ENV_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS: &str = "RUSTFS_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS";

-pub const ENV_NOTIFY_NATS_KEYS: &[&str; 16] = &[
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
    ENV_NOTIFY_NATS_ENABLE,
    ENV_NOTIFY_NATS_ADDRESS,
    ENV_NOTIFY_NATS_SUBJECT,
@@ -63,7 +57,4 @@ pub const ENV_NOTIFY_NATS_KEYS: &[&str; 16] = &[
    ENV_NOTIFY_NATS_TLS_REQUIRED,
    ENV_NOTIFY_NATS_QUEUE_DIR,
    ENV_NOTIFY_NATS_QUEUE_LIMIT,
-    ENV_NOTIFY_NATS_JETSTREAM_ENABLE,
-    ENV_NOTIFY_NATS_JETSTREAM_STREAM_NAME,
-    ENV_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS,
 ];
@@ -37,6 +37,10 @@ pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
 pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
 pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
 pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+/// Stable OpenTelemetry service instance identity for this RustFS process.
+pub const ENV_OBS_INSTANCE_ID: &str = "RUSTFS_OBS_INSTANCE_ID";
+/// Optional stable RustFS deployment identity used to correlate cluster telemetry.
+pub const ENV_OBS_CLUSTER_ID: &str = "RUSTFS_OBS_CLUSTER_ID";

 /// Per-signal enable/disable flags (default: true)
 pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
@@ -131,6 +135,8 @@ mod tests {
        assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
        assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
        assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
+        assert_eq!(ENV_OBS_INSTANCE_ID, "RUSTFS_OBS_INSTANCE_ID");
+        assert_eq!(ENV_OBS_CLUSTER_ID, "RUSTFS_OBS_CLUSTER_ID");
        assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
        assert_eq!(ENV_OBS_LOG_STDOUT_ENABLED, "RUSTFS_OBS_LOG_STDOUT_ENABLED");
        assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
@@ -1,448 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Systematic HTTP e2e for the admin IAM management surface (backlog#1154
-//! peri-2, first batch): the full user / canned-policy / service-account CRUD
-//! lifecycle over real signed HTTP against a real binary, plus a non-admin
-//! denial probe for every management endpoint (reusing the sec-4 assertion
-//! pattern — the authorization gate itself is pinned by `admin_auth_test`).
-//!
-//! Before this suite the ~40 admin handler modules only had helper-level unit
-//! tests; no test proved that `mc admin user add` style flows work end to end
-//! (create -> attach policy -> the credential actually gains S3 access ->
-//! service account inherits it -> deletion revokes it).
-//!
-//! Later batches tracked on backlog#1154: config get/set, info, pools status,
-//! group lifecycle, import/export IAM.
-
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::{Client, Config};
-use http::header::{CONTENT_TYPE, HOST};
-use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
-use serial_test::serial;
-use std::error::Error;
-use tokio::time::{Duration, sleep};
-
-type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BoxError = Box<dyn Error + Send + Sync>;
-
-/// Signs and sends an admin HTTP request with the given credential, returning
-/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
-/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
-async fn admin_request(
-    base_url: &str,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(StatusCode, String), BoxError> {
-    let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut builder = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        builder = builder.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
-    let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request = local_http_client().request(reqwest_method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
-    let status = response.status();
-    let text = response.text().await.unwrap_or_default();
-    Ok((status, text))
-}
-
-/// Root-credential admin request that must succeed; returns the response body.
-async fn admin_ok(
-    env: &RustFSTestEnvironment,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-) -> Result<String, BoxError> {
-    let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
-    if !status.is_success() {
-        return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
-    }
-    Ok(text)
-}
-
-fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
-    let config = Config::builder()
-        .credentials_provider(Credentials::new(access_key, secret_key, None, None, "e2e-admin-crud"))
-        .region(Region::new("us-east-1"))
-        .endpoint_url(url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-    Client::from_conf(config)
-}
-
-/// Retries an S3 PUT until IAM propagation makes it succeed (bounded).
-async fn wait_for_s3_put(client: &Client, bucket: &str, key: &str, within: Duration) -> TestResult {
-    let deadline = tokio::time::Instant::now() + within;
-    loop {
-        match client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(ByteStream::from_static(b"peri-2 probe"))
-            .send()
-            .await
-        {
-            Ok(_) => return Ok(()),
-            Err(err) => {
-                if tokio::time::Instant::now() >= deadline {
-                    return Err(format!("PUT {bucket}/{key} never succeeded: {err:?}").into());
-                }
-            }
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-}
-
-/// A canned policy granting full S3 access to one bucket.
-fn bucket_rw_policy(bucket: &str) -> String {
-    serde_json::json!({
-        "Version": "2012-10-17",
-        "Statement": [{
-            "Effect": "Allow",
-            "Action": ["s3:*"],
-            "Resource": [
-                format!("arn:aws:s3:::{bucket}"),
-                format!("arn:aws:s3:::{bucket}/*")
-            ]
-        }]
-    })
-    .to_string()
-}
-
-/// Full user -> policy -> service-account lifecycle, proving each management
-/// call takes effect on the data plane, not just that the endpoint answers 200.
-#[tokio::test]
-#[serial]
-async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-
-    let bucket = "peri2-crud";
-    let user = "peri2user";
-    let user_secret = "peri2usersecret";
-    let policy = "peri2-rw";
-
-    let root_client = env.create_s3_client();
-    root_client.create_bucket().bucket(bucket).send().await?;
-
-    // --- canned policy CRUD ---------------------------------------------------
-    admin_ok(
-        &env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-canned-policy?name={policy}"),
-        Some(bucket_rw_policy(bucket)),
-    )
-    .await?;
-
-    let info = admin_ok(
-        &env,
-        http::Method::GET,
-        &format!("/rustfs/admin/v3/info-canned-policy?name={policy}"),
-        None,
-    )
-    .await?;
-    let info_json: serde_json::Value = serde_json::from_str(&info)?;
-    let statement_json = info_json.get("Policy").cloned().unwrap_or(info_json.clone());
-    assert!(
-        statement_json.to_string().contains("s3:*"),
-        "info-canned-policy must round-trip the policy document: {info}"
-    );
-
-    let listed = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-canned-policies", None).await?;
-    assert!(listed.contains(policy), "list-canned-policies must contain {policy}: {listed}");
-
-    // --- user CRUD --------------------------------------------------------------
-    admin_ok(
-        &env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
-        Some(serde_json::json!({ "secretKey": user_secret, "status": "enabled" }).to_string()),
-    )
-    .await?;
-
-    let users = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
-    assert!(users.contains(user), "list-users must contain {user}: {users}");
-
-    let user_info = admin_ok(&env, http::Method::GET, &format!("/rustfs/admin/v3/user-info?accessKey={user}"), None).await?;
-    let user_info: rustfs_madmin::UserInfo = serde_json::from_str(&user_info)
-        .map_err(|e| format!("user-info response must decode as rustfs_madmin::UserInfo: {e}"))?;
-    assert_eq!(user_info.status, rustfs_madmin::AccountStatus::Enabled);
-
-    // The fresh user is authenticatable but unauthorized for the bucket.
-    let user_client = build_s3_client(&env.url, user, user_secret);
-    let denied = user_client
-        .put_object()
-        .bucket(bucket)
-        .key("before-attach")
-        .body(ByteStream::from_static(b"x"))
-        .send()
-        .await;
-    assert!(denied.is_err(), "user without a policy must not be able to write to {bucket}");
-
-    // --- attach policy: the credential actually gains S3 access -----------------
-    admin_ok(
-        &env,
-        http::Method::POST,
-        "/rustfs/admin/v3/idp/builtin/policy/attach",
-        Some(serde_json::json!({ "policies": [policy], "user": user }).to_string()),
-    )
-    .await?;
-
-    wait_for_s3_put(&user_client, bucket, "after-attach.txt", Duration::from_secs(10)).await?;
-
-    let user_info = admin_ok(&env, http::Method::GET, &format!("/rustfs/admin/v3/user-info?accessKey={user}"), None).await?;
-    let user_info: rustfs_madmin::UserInfo = serde_json::from_str(&user_info)?;
-    assert!(
-        user_info.policy_name.as_deref().is_some_and(|p| p.contains(policy)),
-        "user-info must reflect the attached policy, got {:?}",
-        user_info.policy_name
-    );
-
-    // --- service account: create for the user, credential works, info/list pin it
-    let sa_resp = admin_ok(
-        &env,
-        http::Method::PUT,
-        "/rustfs/admin/v3/add-service-accounts",
-        Some(serde_json::json!({ "targetUser": user }).to_string()),
-    )
-    .await?;
-    let sa_json: serde_json::Value = serde_json::from_str(&sa_resp)?;
-    let sa_ak = sa_json["credentials"]["accessKey"]
-        .as_str()
-        .ok_or(format!("add-service-accounts response missing credentials.accessKey: {sa_resp}"))?
-        .to_string();
-    let sa_sk = sa_json["credentials"]["secretKey"]
-        .as_str()
-        .ok_or(format!("add-service-accounts response missing credentials.secretKey: {sa_resp}"))?
-        .to_string();
-
-    let sa_client = build_s3_client(&env.url, &sa_ak, &sa_sk);
-    wait_for_s3_put(&sa_client, bucket, "via-service-account.txt", Duration::from_secs(10)).await?;
-
-    let sa_info = admin_ok(
-        &env,
-        http::Method::GET,
-        &format!("/rustfs/admin/v3/info-service-account?accessKey={sa_ak}"),
-        None,
-    )
-    .await?;
-    let sa_info: serde_json::Value = serde_json::from_str(&sa_info)?;
-    assert_eq!(
-        sa_info["parentUser"].as_str(),
-        Some(user),
-        "info-service-account must name the parent user: {sa_info}"
-    );
-
-    let sa_list = admin_ok(
-        &env,
-        http::Method::GET,
-        &format!("/rustfs/admin/v3/list-service-accounts?user={user}"),
-        None,
-    )
-    .await?;
-    let sa_list: rustfs_madmin::ListServiceAccountsResp = serde_json::from_str(&sa_list)
-        .map_err(|e| format!("list-service-accounts must decode as rustfs_madmin::ListServiceAccountsResp: {e}"))?;
-    assert!(
-        sa_list.accounts.iter().any(|a| a.access_key == sa_ak),
-        "list-service-accounts must contain {sa_ak}"
-    );
-
-    // --- deletion revokes access -------------------------------------------------
-    admin_ok(
-        &env,
-        http::Method::DELETE,
-        &format!("/rustfs/admin/v3/delete-service-account?accessKey={sa_ak}"),
-        None,
-    )
-    .await?;
-    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
-    loop {
-        let revoked = sa_client
-            .put_object()
-            .bucket(bucket)
-            .key("after-sa-delete")
-            .body(ByteStream::from_static(b"x"))
-            .send()
-            .await;
-        if revoked.is_err() {
-            break;
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err("deleted service account credential still works".into());
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-
-    // Disable then remove the user; the credential must stop working.
-    admin_ok(
-        &env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/set-user-status?accessKey={user}&status=disabled"),
-        None,
-    )
-    .await?;
-    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
-    loop {
-        let disabled = user_client
-            .put_object()
-            .bucket(bucket)
-            .key("after-disable")
-            .body(ByteStream::from_static(b"x"))
-            .send()
-            .await;
-        if disabled.is_err() {
-            break;
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err("disabled user credential still works".into());
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-
-    admin_ok(
-        &env,
-        http::Method::DELETE,
-        &format!("/rustfs/admin/v3/remove-user?accessKey={user}"),
-        None,
-    )
-    .await?;
-    let users = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
-    assert!(!users.contains(user), "removed user must disappear from list-users: {users}");
-
-    admin_ok(
-        &env,
-        http::Method::DELETE,
-        &format!("/rustfs/admin/v3/remove-canned-policy?name={policy}"),
-        None,
-    )
-    .await?;
-    let (status, _) = admin_request(
-        &env.url,
-        http::Method::GET,
-        &format!("/rustfs/admin/v3/info-canned-policy?name={policy}"),
-        None,
-        &env.access_key,
-        &env.secret_key,
-    )
-    .await?;
-    assert!(!status.is_success(), "info-canned-policy must fail after remove-canned-policy");
-
-    env.stop_server();
-    Ok(())
-}
-
-/// Every management endpoint in the first batch must deny an authenticated but
-/// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the
-/// gate implementation itself is owned by sec-4 / admin_auth_test).
-#[tokio::test]
-#[serial]
-async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-
-    let limited = "peri2limited";
-    let limited_secret = "peri2limitedsecret";
-    let other = "peri2other";
-    for (user, secret) in [(limited, limited_secret), (other, "peri2othersecret")] {
-        admin_ok(
-            &env,
-            http::Method::PUT,
-            &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
-            Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
-        )
-        .await?;
-    }
-
-    // Admin-only management endpoints. Targeted probes aim at a DIFFERENT user
-    // (`other`): operations a principal performs on itself run in `deny_only`
-    // mode (see should_check_deny_only in rustfs/src/admin/handlers/user.rs)
-    // and would not be denied. Also deliberately excludes the self-service
-    // account endpoints (add/list service accounts), which any authenticated
-    // user may call for themselves.
-    let probes: Vec<(http::Method, String, Option<String>)> = vec![
-        (
-            http::Method::PUT,
-            "/rustfs/admin/v3/add-user?accessKey=peri2evil".to_string(),
-            Some(serde_json::json!({ "secretKey": "evilsecret123", "status": "enabled" }).to_string()),
-        ),
-        (http::Method::GET, "/rustfs/admin/v3/list-users".to_string(), None),
-        (http::Method::GET, format!("/rustfs/admin/v3/user-info?accessKey={other}"), None),
-        (
-            http::Method::PUT,
-            format!("/rustfs/admin/v3/set-user-status?accessKey={other}&status=disabled"),
-            None,
-        ),
-        (http::Method::DELETE, format!("/rustfs/admin/v3/remove-user?accessKey={other}"), None),
-        (
-            http::Method::PUT,
-            "/rustfs/admin/v3/add-canned-policy?name=peri2evilpolicy".to_string(),
-            Some(bucket_rw_policy("peri2-any")),
-        ),
-        (http::Method::GET, "/rustfs/admin/v3/list-canned-policies".to_string(), None),
-        (http::Method::GET, "/rustfs/admin/v3/info-canned-policy?name=readwrite".to_string(), None),
-        (
-            http::Method::DELETE,
-            "/rustfs/admin/v3/remove-canned-policy?name=readwrite".to_string(),
-            None,
-        ),
-        (
-            http::Method::POST,
-            "/rustfs/admin/v3/idp/builtin/policy/attach".to_string(),
-            Some(serde_json::json!({ "policies": ["readwrite"], "user": other }).to_string()),
-        ),
-    ];
-
-    for (method, path, body) in probes {
-        let (status, text) = admin_request(&env.url, method.clone(), &path, body, limited, limited_secret).await?;
-        assert_eq!(
-            status,
-            StatusCode::FORBIDDEN,
-            "non-admin credential must get 403 on {method} {path}, got {status}: {text}"
-        );
-        assert!(text.contains("AccessDenied"), "denial on {method} {path} must carry AccessDenied: {text}");
-    }
-
-    env.stop_server();
-    Ok(())
-}
@@ -19,10 +19,8 @@
 mod tests {
    use crate::common::{RustFSTestEnvironment, init_logging};
    use aws_sdk_s3::Client;
-    use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
    use aws_sdk_s3::primitives::ByteStream;
    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
-    use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
    use base64::Engine;
    use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
    use serial_test::serial;
@@ -33,25 +31,6 @@ mod tests {
        env.create_s3_client()
    }

-    /// Client with boto/SDK automatic checksum calculation disabled, so the ONLY
-    /// checksum on the wire is the one the test injects. Needed because the SDK's
-    /// default (crc32) would otherwise collide with the additional-algorithm header
-    /// we inject via `mutate_request` for XXHash/SHA-512/MD5 (which have no typed
-    /// SDK builder). Mirrors the boto3 `request_checksum_calculation=when_required`.
-    fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> Client {
-        let creds = Credentials::new(&env.access_key, &env.secret_key, None, None, "e2e-additional-checksum");
-        let config = aws_sdk_s3::Config::builder()
-            .credentials_provider(creds)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(format!("http://{}", env.address))
-            .force_path_style(true)
-            .behavior_version_latest()
-            .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
-            .http_client(SmithyHttpClientBuilder::new().build_http())
-            .build();
-        Client::from_conf(config)
-    }
-
    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        match client.create_bucket().bucket(bucket).send().await {
            Ok(_) => {
@@ -480,87 +459,4 @@ mod tests {
            "Multipart object should report the same full-object CRC64NVME as direct upload"
        );
    }
-
-    /// Integration test for the AWS 2026-04 additional checksum algorithms
-    /// (XXHash3/64/128, SHA-512, MD5). aws_sdk_s3 has no typed builder for these, so the
-    /// `x-amz-checksum-<algo>` header is injected via `mutate_request` (value computed by
-    /// rustfs-rio, which is byte-for-byte identical to awscrt). Verifies the server
-    /// verifies-on-write: a correct value is accepted and the object stored; a wrong
-    /// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header
-    /// echo round-trip is additionally exercised by the boto3+awscrt e2e.
-    #[tokio::test]
-    #[serial]
-    async fn test_additional_checksums_verify_on_write() {
-        init_logging();
-        info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client_no_auto_checksum(&env);
-        let bucket = "test-additional-checksums";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let content: &[u8] = b"additional-checksum verify-on-write payload for xxhash/sha512/md5";
-
-        for (ty, header) in [
-            (RioChecksumType::XXHASH3, "x-amz-checksum-xxhash3"),
-            (RioChecksumType::XXHASH64, "x-amz-checksum-xxhash64"),
-            (RioChecksumType::XXHASH128, "x-amz-checksum-xxhash128"),
-            (RioChecksumType::SHA512, "x-amz-checksum-sha512"),
-            (RioChecksumType::MD5, "x-amz-checksum-md5"),
-        ] {
-            // Correct checksum -> accepted, and the object is stored intact.
-            let good = Checksum::new_from_data(ty, content).expect("compute checksum").encoded;
-            let ok_key = format!("ok-{header}");
-            let put = client
-                .put_object()
-                .bucket(bucket)
-                .key(&ok_key)
-                .body(ByteStream::from_static(content))
-                .customize()
-                .mutate_request(move |req| {
-                    req.headers_mut().insert(header, good.clone());
-                })
-                .send()
-                .await;
-            assert!(put.is_ok(), "{header}: correct checksum must be accepted: {:?}", put.err());
-            let got = client
-                .get_object()
-                .bucket(bucket)
-                .key(&ok_key)
-                .send()
-                .await
-                .expect("GetObject");
-            let body = got.body.collect().await.expect("collect body").into_bytes();
-            assert_eq!(body.as_ref(), content, "{header}: stored body must match uploaded content");
-
-            // Wrong checksum -> rejected with BadDigest, and nothing is stored.
-            let bad = Checksum::new_from_data(ty, b"a totally different payload")
-                .expect("compute checksum")
-                .encoded;
-            let bad_key = format!("bad-{header}");
-            let put_bad = client
-                .put_object()
-                .bucket(bucket)
-                .key(&bad_key)
-                .body(ByteStream::from_static(content))
-                .customize()
-                .mutate_request(move |req| {
-                    req.headers_mut().insert(header, bad.clone());
-                })
-                .send()
-                .await;
-            assert!(put_bad.is_err(), "{header}: a mismatched checksum must be rejected");
-            let msg = format!("{:?}", put_bad.err().unwrap());
-            assert!(
-                msg.contains("BadDigest") || msg.to_lowercase().contains("digest") || msg.to_lowercase().contains("checksum"),
-                "{header}: expected a BadDigest/checksum error, got: {msg}"
-            );
-            let head = client.head_object().bucket(bucket).key(&bad_key).send().await;
-            assert!(head.is_err(), "{header}: nothing must be stored after a rejected PutObject");
-
-            info!("PASSED additional-checksum verify-on-write: {header}");
-        }
-    }
 }
@@ -514,30 +514,6 @@ impl RustFSTestEnvironment {
            }
        }
    }
-
-    /// Restart this server in place, preserving its on-disk data directory,
-    /// listen address, and credentials.
-    ///
-    /// Failure-recovery tests (backlog#1147 repl-5) need the *same* instance to
-    /// come back up after a crash/stop with its persisted state intact:
-    /// per-object replication status in `xl.meta`, the MRF recovery file, and
-    /// resync metadata all live under `temp_dir`, which this method keeps. The
-    /// address is reused too (the server sets `SO_REUSEADDR`/`SO_REUSEPORT`), so
-    /// existing S3 clients keep working without reconfiguration.
-    ///
-    /// The previous process is stopped first. `cleanup_existing` is false so a
-    /// sibling server sharing the harness (e.g. a replication target on another
-    /// port) is left untouched — only this instance is recycled. Pass the same
-    /// `extra_args` / `extra_env` the instance was originally started with;
-    /// background tasks read their env once at startup.
-    pub async fn restart_server_preserving_data(
-        &mut self,
-        extra_args: Vec<&str>,
-        extra_env: &[(&str, &str)],
-    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        self.stop_server();
-        self.start_rustfs_server_inner(extra_args, extra_env, false).await
-    }
 }

 impl Drop for RustFSTestEnvironment {
@@ -1,191 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Over-the-wire smoke net for the embedded console listener (backlog#1154
-//! peri-4). The console is a second HTTP listener that serves the full S3/admin
-//! surface plus the unauthenticated console routes under `/rustfs/console`;
-//! before this test its behaviour was only covered by in-process router tests
-//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
-//! credentials through the public console endpoints or fail-opened the
-//! protected surface on the console port had no regression net.
-//!
-//! Pinned wire contract (real binary, real TCP):
-//!   * `/rustfs/console/version` and `/rustfs/console/license` answer 200 JSON
-//!     without authentication, with complete fields and no credential material.
-//!   * `/rustfs/console/license` exposes only the coarse `licensed` flag.
-//!   * the console SPA prefix dispatches to the static handler, never to the
-//!     S3 API (no S3 error XML), whether or not console assets are embedded.
-//!   * unauthenticated requests to the admin API and the S3 root on the console
-//!     listener are denied (403 AccessDenied) — the extra listener does not
-//!     fail-open the protected surface.
-//!   * a listener with the console disabled (the main S3 port here) does not
-//!     serve the unauthenticated console endpoints at all.
-
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use serial_test::serial;
-use std::error::Error;
-use tokio::time::{Duration, sleep};
-
-type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-
-/// Polls the console version endpoint until the console listener accepts
-/// requests. The harness readiness check only covers the S3 listener; the
-/// console listener of the same process may come up moments later.
-async fn wait_for_console_ready(console_base: &str) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
-    let client = local_http_client();
-    let url = format!("{console_base}/rustfs/console/version");
-    let mut last_err = String::new();
-    for _ in 0..40 {
-        match client.get(&url).send().await {
-            Ok(response) if response.status() == reqwest::StatusCode::OK => return Ok(response),
-            Ok(response) => last_err = format!("status {}", response.status()),
-            Err(err) => last_err = err.to_string(),
-        }
-        sleep(Duration::from_millis(250)).await;
-    }
-    Err(format!("console listener at {console_base} never became ready: {last_err}").into())
-}
-
-#[tokio::test]
-#[serial]
-async fn test_console_over_the_wire_smoke() -> TestResult {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    // A unique secret makes the credential-leak assertions below sharp: any
-    // occurrence of it in a console response body is a genuine leak, not a
-    // collision with a common default string.
-    env.access_key = format!("peri4ak{}", uuid::Uuid::new_v4().simple());
-    env.secret_key = format!("peri4sk{}", uuid::Uuid::new_v4().simple());
-
-    let console_port = RustFSTestEnvironment::find_available_port().await?;
-    let console_address = format!("127.0.0.1:{console_port}");
-    let console_base = format!("http://{console_address}");
-    env.start_rustfs_server_with_env(
-        vec![],
-        &[
-            ("RUSTFS_CONSOLE_ENABLE", "true"),
-            ("RUSTFS_CONSOLE_ADDRESS", console_address.as_str()),
-        ],
-    )
-    .await?;
-
-    let client = local_http_client();
-
-    // --- /rustfs/console/version: 200 JSON, complete fields, no secrets ------
-    let version_response = wait_for_console_ready(&console_base).await?;
-    let content_type = version_response
-        .headers()
-        .get(reqwest::header::CONTENT_TYPE)
-        .and_then(|v| v.to_str().ok())
-        .unwrap_or_default()
-        .to_string();
-    assert!(content_type.starts_with("application/json"), "version content-type: {content_type}");
-    let version_body = version_response.text().await?;
-    let version_json: serde_json::Value = serde_json::from_str(&version_body)?;
-    for field in ["version", "version_info", "date"] {
-        assert!(
-            version_json[field].as_str().is_some_and(|v| !v.is_empty()),
-            "console version field {field} missing or empty: {version_body}"
-        );
-    }
-    assert!(
-        !version_body.contains(&env.secret_key) && !version_body.contains(&env.access_key),
-        "console version response leaks credentials: {version_body}"
-    );
-
-    // --- /rustfs/console/license: coarse licensed flag only ------------------
-    let license_response = client.get(format!("{console_base}/rustfs/console/license")).send().await?;
-    assert_eq!(license_response.status(), reqwest::StatusCode::OK);
-    let license_body = license_response.text().await?;
-    let license_json: serde_json::Value = serde_json::from_str(&license_body)?;
-    assert!(
-        license_json["licensed"].is_boolean(),
-        "license payload must expose a licensed bool: {license_body}"
-    );
-    let license_keys: Vec<&String> = license_json.as_object().map(|o| o.keys().collect()).unwrap_or_default();
-    assert_eq!(
-        license_keys,
-        vec!["licensed"],
-        "public license endpoint must not expose license metadata: {license_body}"
-    );
-    assert!(
-        !license_body.contains(&env.secret_key) && !license_body.contains(&env.access_key),
-        "console license response leaks credentials: {license_body}"
-    );
-
-    // --- console SPA prefix dispatches to the static handler, not the S3 API -
-    // With release console assets embedded this is 200 text/html; a from-source
-    // binary embeds an empty static dir and serves the handler's 404 fallback.
-    // Either way it must never fall through to an S3 handler (S3 error XML).
-    let spa_response = client.get(format!("{console_base}/rustfs/console/")).send().await?;
-    let spa_status = spa_response.status();
-    let spa_content_type = spa_response
-        .headers()
-        .get(reqwest::header::CONTENT_TYPE)
-        .and_then(|v| v.to_str().ok())
-        .unwrap_or_default()
-        .to_string();
-    let spa_body = spa_response.text().await?;
-    assert!(
-        !spa_body.contains("<Error>"),
-        "console SPA route fell through to the S3 API: status {spa_status}, body {spa_body}"
-    );
-    match spa_status {
-        reqwest::StatusCode::OK => {
-            assert!(
-                spa_content_type.starts_with("text/html"),
-                "embedded console index must be html, got {spa_content_type}"
-            );
-        }
-        reqwest::StatusCode::NOT_FOUND => {
-            assert!(
-                spa_body.contains("RustFS"),
-                "asset-less console 404 must come from the console static handler: {spa_body}"
-            );
-        }
-        other => panic!("console SPA route returned unexpected status {other}: {spa_body}"),
-    }
-
-    // --- protected surface on the console listener stays authenticated -------
-    let admin_response = client.get(format!("{console_base}/rustfs/admin/v3/info")).send().await?;
-    assert_eq!(
-        admin_response.status(),
-        reqwest::StatusCode::FORBIDDEN,
-        "unauthenticated admin API on the console listener must be denied"
-    );
-    let admin_body = admin_response.text().await?;
-    assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
-
-    let s3_root_response = client.get(format!("{console_base}/")).send().await?;
-    assert_eq!(
-        s3_root_response.status(),
-        reqwest::StatusCode::FORBIDDEN,
-        "unauthenticated S3 root on the console listener must be denied"
-    );
-
-    // --- console disabled on a listener means no console surface at all ------
-    // The main S3 listener of this same process runs with the console disabled,
-    // so its console paths must not answer with the unauthenticated console
-    // payloads (they fall through to the authenticated S3 surface instead).
-    let s3_listener_console = client.get(format!("{}/rustfs/console/version", env.url)).send().await?;
-    assert_ne!(
-        s3_listener_console.status(),
-        reqwest::StatusCode::OK,
-        "console endpoints must not be served on a console-disabled listener"
-    );
-
-    env.stop_server();
-    Ok(())
-}
@@ -191,30 +191,6 @@ mod stale_multipart_cleanup_cluster_test;
 #[cfg(test)]
 mod object_lambda_test;

-// S3 event-notification webhook delivery end-to-end (backlog#1154 peri-1):
-// configure webhook target -> PutBucketNotificationConfiguration -> object
-// operation -> event delivered, plus filter negatives and store-queue redelivery.
-#[cfg(test)]
-mod notification_webhook_test;
-
-// TLS certificate hot-reload live-listener e2e (backlog#1154 peri-5): swap
-// certificates without a restart, existing sessions survive, bad material is
-// fail-safe (old certificate keeps serving, failure is logged).
-#[cfg(test)]
-mod tls_hot_reload_test;
-
-// Console listener over-the-wire smoke (backlog#1154 peri-4): public console
-// endpoints answer without credentials or leaks, the SPA prefix never falls
-// through to the S3 API, and the protected surface stays authenticated.
-#[cfg(test)]
-mod console_smoke_test;
-
-// Admin IAM management CRUD e2e (backlog#1154 peri-2): user / canned-policy /
-// service-account lifecycle over signed HTTP with data-plane effect assertions,
-// plus non-admin 403 probes per endpoint (sec-4 pattern).
-#[cfg(test)]
-mod admin_iam_crud_test;
-
 // Replication extension end-to-end regression tests
 #[cfg(test)]
 mod replication_extension_test;
@@ -1,594 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! End-to-end regression net for the S3 event-notification pipeline
-//! (backlog#1154 peri-1). Proves the full "configure webhook target ->
-//! PutBucketNotificationConfiguration -> object operation -> event delivered"
-//! chain against a real rustfs binary and a real HTTP receiver, which no other
-//! test covers: the target-plugin suites stop at broker integration and the
-//! object-lambda suite only exercises the webhook target as a transform
-//! function, never as an S3 event sink.
-//!
-//! Coverage:
-//!   * PUT / multipart-complete / DELETE each deliver one event with the correct
-//!     eventName, bucket, key, versionId and eTag.
-//!   * prefix/suffix filters drop non-matching keys (rule-engine gate).
-//!   * an event queued while the target endpoint is unreachable is redelivered
-//!     from the on-disk store once the endpoint recovers (store-and-forward).
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
-    NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
-};
-use http::header::{CONTENT_TYPE, HOST};
-use local_ip_address::local_ip;
-use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
-use serde_json::Value;
-use serial_test::serial;
-use std::error::Error;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::TcpListener;
-use tokio::sync::mpsc;
-use tokio::task::JoinHandle;
-use tokio::time::{Duration, Instant, timeout};
-
-type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BoxError = Box<dyn Error + Send + Sync>;
-
-/// Region embedded in the target ARN. Only the `id:name` tail of the ARN is used
-/// for target lookup (see `process_queue_configurations`), so the region is
-/// nominal, but it must be present for `ARN::parse` to succeed.
-const NOTIFY_REGION: &str = "us-east-1";
-
-/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
-/// so the ARN a notification rule references is
-/// `arn:rustfs:sqs:<region>:<name>:webhook`.
-fn target_arn(target_name: &str) -> String {
-    format!("arn:rustfs:sqs:{NOTIFY_REGION}:{target_name}:webhook")
-}
-
-// ---------------------------------------------------------------------------
-// In-test HTTP event receiver
-// ---------------------------------------------------------------------------
-
-/// Reads one HTTP request off the stream, returning its method and body. Handles
-/// the target's HEAD reachability probe (no body) and POST event deliveries.
-async fn read_http_message(stream: &mut tokio::net::TcpStream) -> Result<(String, Vec<u8>), BoxError> {
-    let mut buffer = Vec::new();
-    let mut chunk = [0_u8; 4096];
-
-    let header_end = loop {
-        let read = stream.read(&mut chunk).await?;
-        if read == 0 {
-            return Err("connection closed before request headers were complete".into());
-        }
-        buffer.extend_from_slice(&chunk[..read]);
-        if let Some(pos) = buffer.windows(4).position(|w| w == b"\r\n\r\n") {
-            break pos;
-        }
-    };
-
-    let header_text = std::str::from_utf8(&buffer[..header_end])?;
-    let mut lines = header_text.split("\r\n");
-    let request_line = lines.next().ok_or("missing request line")?;
-    let method = request_line.split_whitespace().next().ok_or("missing method")?.to_string();
-
-    let mut content_length = 0usize;
-    for line in lines {
-        if let Some((name, value)) = line.split_once(':')
-            && name.trim().eq_ignore_ascii_case("content-length")
-        {
-            content_length = value.trim().parse::<usize>().unwrap_or(0);
-        }
-    }
-
-    let body_offset = header_end + 4;
-    while buffer.len().saturating_sub(body_offset) < content_length {
-        let read = stream.read(&mut chunk).await?;
-        if read == 0 {
-            return Err("connection closed before request body was complete".into());
-        }
-        buffer.extend_from_slice(&chunk[..read]);
-    }
-
-    Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
-}
-
-/// Drives an accepted listener: answers every request 200 OK (so the target's
-/// HEAD reachability probe reports it online) and forwards each non-empty POST
-/// body, parsed as the S3 event envelope JSON, to `tx`.
-fn serve_event_collector(listener: TcpListener, tx: mpsc::UnboundedSender<Value>) -> JoinHandle<()> {
-    tokio::spawn(async move {
-        loop {
-            let Ok((mut stream, _)) = listener.accept().await else {
-                return;
-            };
-            let tx = tx.clone();
-            tokio::spawn(async move {
-                if let Ok(Ok((method, body))) = timeout(Duration::from_secs(5), read_http_message(&mut stream)).await {
-                    let _ = stream
-                        .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
-                        .await;
-                    let _ = stream.shutdown().await;
-                    if method == "POST"
-                        && !body.is_empty()
-                        && let Ok(value) = serde_json::from_slice::<Value>(&body)
-                    {
-                        let _ = tx.send(value);
-                    }
-                }
-            });
-        }
-    })
-}
-
-/// Binds a fresh collector on a random port. Returns its `/events`
-/// endpoint URL, a receiver of parsed event envelopes, and the serving task.
-async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>), BoxError> {
-    let listener = TcpListener::bind("0.0.0.0:0").await?;
-    let port = listener.local_addr()?.port();
-    let endpoint_ip = local_ip()?;
-    let (tx, rx) = mpsc::unbounded_channel();
-    let handle = serve_event_collector(listener, tx);
-    Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
-}
-
-/// Decoded object key of the first record in an event envelope.
-fn event_key(envelope: &Value) -> Option<String> {
-    let raw = envelope["Records"][0]["s3"]["object"]["key"].as_str()?;
-    Some(
-        urlencoding::decode(raw)
-            .map(|d| d.into_owned())
-            .unwrap_or_else(|_| raw.to_string()),
-    )
-}
-
-/// Waits until an event targeting `key` whose `EventName` starts with
-/// `event_prefix` arrives, returning the full envelope. Other envelopes are
-/// discarded, so a lingering duplicate of an earlier event (delivery is
-/// at-least-once) or the create event of a key that is later deleted never
-/// satisfies a wait for the opposite event type.
-async fn wait_for_event(
-    rx: &mut mpsc::UnboundedReceiver<Value>,
-    key: &str,
-    event_prefix: &str,
-    within: Duration,
-) -> Result<Value, BoxError> {
-    let deadline = Instant::now() + within;
-    loop {
-        let remaining = deadline.saturating_duration_since(Instant::now());
-        if remaining.is_zero() {
-            return Err(format!("no {event_prefix}* event for key {key} within {within:?}").into());
-        }
-        match timeout(remaining, rx.recv()).await {
-            Ok(Some(envelope)) => {
-                let key_matches = event_key(&envelope).as_deref() == Some(key);
-                let name_matches = envelope["EventName"].as_str().is_some_and(|n| n.starts_with(event_prefix));
-                if key_matches && name_matches {
-                    return Ok(envelope);
-                }
-            }
-            Ok(None) => return Err("event collector channel closed".into()),
-            Err(_) => return Err(format!("no {event_prefix}* event for key {key} within {within:?}").into()),
-        }
-    }
-}
-
-/// Collects every event until `stop_key` is seen (plus a short grace window to
-/// catch stragglers), or until `max_wait` elapses. Used to prove a negative:
-/// non-matching keys must never appear even though a matching sentinel does.
-async fn collect_until(
-    rx: &mut mpsc::UnboundedReceiver<Value>,
-    stop_key: &str,
-    max_wait: Duration,
-    grace: Duration,
-) -> Vec<Value> {
-    let hard_deadline = Instant::now() + max_wait;
-    let mut soft_deadline: Option<Instant> = None;
-    let mut collected = Vec::new();
-    loop {
-        let deadline = soft_deadline.unwrap_or(hard_deadline).min(hard_deadline);
-        let remaining = deadline.saturating_duration_since(Instant::now());
-        if remaining.is_zero() {
-            break;
-        }
-        match timeout(remaining, rx.recv()).await {
-            Ok(Some(envelope)) => {
-                let matched = event_key(&envelope).as_deref() == Some(stop_key);
-                collected.push(envelope);
-                if matched && soft_deadline.is_none() {
-                    soft_deadline = Some(Instant::now() + grace);
-                }
-            }
-            Ok(None) => break,
-            Err(_) => break,
-        }
-    }
-    collected
-}
-
-// ---------------------------------------------------------------------------
-// Admin target configuration (signed admin HTTP)
-// ---------------------------------------------------------------------------
-
-async fn signed_admin_request(
-    env: &RustFSTestEnvironment,
-    method: http::Method,
-    url: &str,
-    body: Option<Vec<u8>>,
-) -> Result<reqwest::Response, BoxError> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut builder = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        builder = builder.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        builder.body(Body::empty())?,
-        content_len,
-        &env.access_key,
-        &env.secret_key,
-        "",
-        "us-east-1",
-    );
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request = crate::common::local_http_client().request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    Ok(request.send().await?)
-}
-
-async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
-    let payload = serde_json::json!({
-        "notify_enabled": true,
-        "audit_enabled": false,
-    });
-    let url = format!("{}/rustfs/admin/v3/module-switches", env.url);
-    let response = signed_admin_request(env, http::Method::PUT, &url, Some(payload.to_string().into_bytes())).await?;
-    if response.status() != StatusCode::OK {
-        let status = response.status();
-        let text = response.text().await.unwrap_or_default();
-        return Err(format!("failed to enable notify module: {status} {text}").into());
-    }
-    Ok(())
-}
-
-/// Registers a webhook notification target with a persistent queue directory, so
-/// delivery goes through the durable store-and-forward path.
-async fn configure_webhook_target(env: &RustFSTestEnvironment, target_name: &str, endpoint: &str) -> TestResult {
-    let queue_dir = format!("{}/notify-queue-{target_name}", env.temp_dir);
-    tokio::fs::create_dir_all(&queue_dir).await?;
-    let payload = serde_json::json!({
-        "key_values": [
-            { "key": "endpoint", "value": endpoint },
-            { "key": "queue_dir", "value": queue_dir },
-        ]
-    });
-    let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}", env.url);
-    let response = signed_admin_request(env, http::Method::PUT, &url, Some(payload.to_string().into_bytes())).await?;
-    let status = response.status();
-    if status != StatusCode::OK {
-        let text = response.text().await.unwrap_or_default();
-        return Err(format!("failed to configure webhook target {target_name}: {status} {text}").into());
-    }
-    Ok(())
-}
-
-/// Polls the admin target ARN list until the freshly configured target is
-/// registered in the runtime, so notification rules and object events do not
-/// race target activation.
-async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
-    let suffix = format!(":{target_name}:webhook");
-    let url = format!("{}/rustfs/admin/v3/target/arns", env.url);
-    for _ in 0..80 {
-        let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
-        if response.status() == StatusCode::OK {
-            let arns: Vec<String> = serde_json::from_slice(&response.bytes().await?)?;
-            if arns.iter().any(|arn| arn.ends_with(&suffix)) {
-                return Ok(());
-            }
-        }
-        tokio::time::sleep(Duration::from_millis(250)).await;
-    }
-    Err(format!("target {target_name} was not registered in admin ARNs").into())
-}
-
-/// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
-/// filtered to `prefix` + `suffix`.
-async fn put_notification_config(client: &Client, bucket: &str, target_name: &str, prefix: &str, suffix: &str) -> TestResult {
-    let key_filter = S3KeyFilter::builder()
-        .filter_rules(FilterRule::builder().name(FilterRuleName::Prefix).value(prefix).build())
-        .filter_rules(FilterRule::builder().name(FilterRuleName::Suffix).value(suffix).build())
-        .build();
-    let queue = QueueConfiguration::builder()
-        .id(format!("{target_name}-rule"))
-        .queue_arn(target_arn(target_name))
-        .events(Event::from("s3:ObjectCreated:*"))
-        .events(Event::from("s3:ObjectRemoved:*"))
-        .filter(NotificationConfigurationFilter::builder().key(key_filter).build())
-        .build()?;
-    let config = NotificationConfiguration::builder().queue_configurations(queue).build();
-    client
-        .put_bucket_notification_configuration()
-        .bucket(bucket)
-        .notification_configuration(config)
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn enable_versioning(client: &Client, bucket: &str) -> TestResult {
-    client
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Enabled)
-                .build(),
-        )
-        .send()
-        .await?;
-    Ok(())
-}
-
-fn trimmed_etag(value: Option<&str>) -> Option<String> {
-    value.map(|e| e.trim_matches('"').to_string())
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-/// PUT / multipart-complete / DELETE each deliver one event with correct fields,
-/// and the prefix/suffix filter drops non-matching keys.
-#[tokio::test]
-#[serial]
-async fn test_webhook_event_delivery_and_filtering() -> TestResult {
-    init_logging();
-
-    let (endpoint, mut rx, handle) = spawn_event_collector().await?;
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    enable_notify_module(&env).await?;
-
-    let bucket = "peri1-events";
-    let target = "peri1events";
-    let client = env.create_s3_client();
-    client.create_bucket().bucket(bucket).send().await?;
-    enable_versioning(&client, bucket).await?;
-
-    configure_webhook_target(&env, target, &endpoint).await?;
-    wait_for_target_registered(&env, target).await?;
-    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
-
-    // --- PUT: ObjectCreated:Put with exact eTag + versionId ------------------
-    let put_key = "uploads/report.dat";
-    let put = client
-        .put_object()
-        .bucket(bucket)
-        .key(put_key)
-        .body(ByteStream::from_static(b"peri-1 put body"))
-        .send()
-        .await?;
-    let put_version = put
-        .version_id()
-        .ok_or("PUT response missing versionId (versioning not enabled?)")?;
-
-    let created = wait_for_event(&mut rx, put_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
-    let record = &created["Records"][0];
-    let object = &record["s3"]["object"];
-    assert_eq!(
-        created["EventName"].as_str(),
-        Some("s3:ObjectCreated:Put"),
-        "envelope EventName: {created}"
-    );
-    assert!(
-        record["eventName"]
-            .as_str()
-            .is_some_and(|n| n.starts_with("s3:ObjectCreated:")),
-        "record eventName: {record}"
-    );
-    assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
-    assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
-    assert_eq!(
-        trimmed_etag(object["eTag"].as_str()),
-        trimmed_etag(put.e_tag()),
-        "eTag in event: {object}"
-    );
-
-    // --- Multipart complete: ObjectCreated:CompleteMultipartUpload -----------
-    let mp_key = "uploads/multi.dat";
-    let created_mp = client.create_multipart_upload().bucket(bucket).key(mp_key).send().await?;
-    let upload_id = created_mp.upload_id().ok_or("missing upload id")?;
-    let part = client
-        .upload_part()
-        .bucket(bucket)
-        .key(mp_key)
-        .upload_id(upload_id)
-        .part_number(1)
-        .body(ByteStream::from_static(b"peri-1 multipart part body"))
-        .send()
-        .await?;
-    let complete = client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(mp_key)
-        .upload_id(upload_id)
-        .multipart_upload(
-            CompletedMultipartUpload::builder()
-                .parts(
-                    CompletedPart::builder()
-                        .part_number(1)
-                        .e_tag(part.e_tag().unwrap_or_default())
-                        .build(),
-                )
-                .build(),
-        )
-        .send()
-        .await?;
-
-    let mp_event = wait_for_event(&mut rx, mp_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
-    let mp_record = &mp_event["Records"][0];
-    assert_eq!(
-        mp_event["EventName"].as_str(),
-        Some("s3:ObjectCreated:CompleteMultipartUpload"),
-        "multipart EventName: {mp_event}"
-    );
-    assert_eq!(mp_record["s3"]["bucket"]["name"].as_str(), Some(bucket));
-    assert_eq!(
-        trimmed_etag(mp_record["s3"]["object"]["eTag"].as_str()),
-        trimmed_etag(complete.e_tag()),
-        "multipart eTag in event: {mp_record}"
-    );
-
-    // --- Filter: non-matching prefix and suffix must never be delivered ------
-    let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
-    let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
-    let sentinel = "uploads/keep.dat";
-    for key in [wrong_prefix, wrong_suffix, sentinel] {
-        client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(ByteStream::from_static(b"filter probe"))
-            .send()
-            .await?;
-    }
-    let collected = collect_until(&mut rx, sentinel, Duration::from_secs(20), Duration::from_secs(2)).await;
-    let seen: Vec<String> = collected.iter().filter_map(event_key).collect();
-    assert!(
-        seen.iter().any(|k| k == sentinel),
-        "matching sentinel {sentinel} was not delivered; saw {seen:?}"
-    );
-    assert!(
-        !seen.iter().any(|k| k == wrong_prefix),
-        "wrong-prefix key {wrong_prefix} bypassed the filter; saw {seen:?}"
-    );
-    assert!(
-        !seen.iter().any(|k| k == wrong_suffix),
-        "wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
-    );
-
-    // --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
-    let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
-    let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
-    let removed_record = &removed["Records"][0];
-    assert!(
-        removed["EventName"]
-            .as_str()
-            .is_some_and(|n| n.starts_with("s3:ObjectRemoved:")),
-        "delete EventName: {removed}"
-    );
-    if let Some(marker_version) = delete.version_id() {
-        assert_eq!(
-            removed_record["s3"]["object"]["versionId"].as_str(),
-            Some(marker_version),
-            "delete-marker versionId in event: {removed_record}"
-        );
-    }
-
-    env.stop_server();
-    handle.abort();
-    Ok(())
-}
-
-/// An event queued while the target endpoint is unreachable survives on the
-/// durable store and is redelivered once the endpoint comes back.
-#[tokio::test]
-#[serial]
-#[ignore = "FAILING deterministically on main since it landed (#4821): the target is created but never appears in /rustfs/admin/v3/target/arns, so wait_for_target_registered times out. Quarantined per the flake policy; remove with the fix for rustfs#4852"]
-async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    enable_notify_module(&env).await?;
-
-    let bucket = "peri1-redeliver";
-    let target = "peri1redeliver";
-    let client = env.create_s3_client();
-    client.create_bucket().bucket(bucket).send().await?;
-
-    // Configure the target while its endpoint is reachable so activation and
-    // ARN registration complete deterministically. Registering against a dead
-    // endpoint stalls behind the reachability probe's timeout and flakes the
-    // registration wait on loaded runners (observed on the merge run of
-    // rustfs#4821).
-    let listener = TcpListener::bind("0.0.0.0:0").await?;
-    let port = listener.local_addr()?.port();
-    let endpoint_ip = local_ip()?;
-    let endpoint = format!("http://{endpoint_ip}.nip.io:{port}/events");
-    let (setup_tx, _setup_rx) = mpsc::unbounded_channel();
-    let setup_handle = serve_event_collector(listener, setup_tx);
-
-    configure_webhook_target(&env, target, &endpoint).await?;
-    wait_for_target_registered(&env, target).await?;
-    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
-
-    // Take the endpoint down (drops the listener, so connections are refused —
-    // a retryable NotConnected), then PUT: the event cannot be delivered and
-    // must survive on the durable queue store.
-    setup_handle.abort();
-    let _ = setup_handle.await;
-
-    let key = "uploads/redeliver.dat";
-    client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from_static(b"queued while target down"))
-        .send()
-        .await?;
-
-    // Hold the endpoint down long enough for at least one replay attempt to
-    // fail (the replay worker scans the store every 500ms), so recovery below
-    // exercises real redelivery rather than a first-attempt success.
-    tokio::time::sleep(Duration::from_secs(2)).await;
-
-    // Bring the endpoint back on the same port; the replay worker retries with
-    // exponential backoff and delivers the queued event.
-    let listener = TcpListener::bind(("0.0.0.0", port)).await?;
-    let (tx, mut rx) = mpsc::unbounded_channel();
-    let handle = serve_event_collector(listener, tx);
-
-    let redelivered = wait_for_event(&mut rx, key, "s3:ObjectCreated:", Duration::from_secs(45)).await?;
-    assert_eq!(
-        redelivered["EventName"].as_str(),
-        Some("s3:ObjectCreated:Put"),
-        "redelivered EventName: {redelivered}"
-    );
-    assert_eq!(redelivered["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
-
-    env.stop_server();
-    handle.abort();
-    Ok(())
-}
@@ -22,4 +22,3 @@ mod lifecycle;
 mod lock;
 mod node_interact_test;
 mod sql;
-mod tiering;
@@ -1,380 +0,0 @@
-#![cfg(test)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Hermetic ILM transition main-path end-to-end test (backlog#1148 ilm-7).
-//!
-//! Two embedded `rustfs` servers, each with independent credentials, port and
-//! data directory:
-//!
-//! * `cold` — a second RustFS server wired as a [`TierType::RustFS`] remote tier.
-//! * `hot` — the source server that transitions objects to `cold`.
-//!
-//! There are no containers, no external S3 backend and no `awscurl`: the
-//! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like
-//! the other admin-API e2e suites in this crate. The RustFS warm backend has no
-//! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier
-//! to `cold` over `http://127.0.0.1:<port>`.
-//!
-//! A single test drives the full transition main path and pins the chain
-//! required by ilm-7:
-//!   1. `AddTier(RustFS)` on `hot` targeting `cold` — the real connectivity /
-//!      in-use probe runs (no `force`), so this also proves the tier is reachable.
-//!   2. A `Transition Days=0` rule installed before a multipart PUT transitions
-//!      the object immediately (the completion path enqueues it; the 1s scanner
-//!      cycle is only a backstop).
-//!   3. `HEAD` reports `x-amz-storage-class == <tier name>` and no `x-amz-restore`.
-//!   4. `GET` streams byte-identical data back through the warm backend, and the
-//!      content-type and user metadata survive the round trip (rustfs#2246).
-//!   5. Range `GET` within a part and across the part boundary read the correct
-//!      bytes from the tier (backlog#807).
-//!   6. The remote object is present in the cold-tier bucket after transition.
-//!   7. `DeleteObject` on `hot` drives free-version cleanup: the cold-tier copy
-//!      eventually disappears and the hot object is gone (no local residue).
-
-use crate::common::{RustFSTestEnvironment, local_http_client};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{
-    BucketLifecycleConfiguration, CompletedMultipartUpload, CompletedPart, ExpirationStatus, LifecycleRule, LifecycleRuleFilter,
-    Transition, TransitionStorageClass,
-};
-use http::Method;
-use http::header::HOST;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
-use std::time::{Duration as StdDuration, Instant};
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const TIER_NAME: &str = "COLDTIER";
-const TIER_BUCKET: &str = "ilm7-cold-tier";
-const TIER_PREFIX: &str = "tiered";
-const SOURCE_BUCKET: &str = "ilm7-hot";
-const OBJECT_KEY: &str = "tier/report.bin";
-const CONTENT_TYPE: &str = "application/x-ilm7";
-const USER_META_KEY: &str = "ilm7-origin";
-const USER_META_VAL: &str = "hermetic-transition";
-
-/// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
-/// internal part boundary sits at this offset.
-const PART0_SIZE: usize = 5 * 1024 * 1024;
-/// 1 MiB tail so the completed object is genuinely multipart (two parts).
-const PART1_SIZE: usize = 1024 * 1024;
-const OBJECT_SIZE: usize = PART0_SIZE + PART1_SIZE;
-
-/// Deterministic, position-dependent payload: adjacent offsets differ, so a
-/// misaligned range read is caught.
-fn payload() -> Vec<u8> {
-    (0..OBJECT_SIZE).map(|i| (i % 251) as u8).collect()
-}
-
-/// Sign and send an admin request in-process (no `awscurl`).
-///
-/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed
-/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being
-/// pre-hashed. Returns the response status and body text.
-async fn signed_admin_request(
-    base_url: &str,
-    method: Method,
-    path: &str,
-    body: Option<&str>,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{base_url}{path}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
-
-    let request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
-
-    let client = local_http_client();
-    let mut request_builder = client.request(method, url.as_str());
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if !body_bytes.is_empty() {
-        request_builder = request_builder.body(body_bytes);
-    }
-    let response = request_builder.send().await?;
-    let status = response.status();
-    let text = response.text().await?;
-    Ok((status, text))
-}
-
-/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
-///
-/// No `force`, so the server runs the real in-use / connectivity probe against
-/// `cold` (which requires the tier bucket to already exist there).
-async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
-    let body = serde_json::json!({
-        "type": "rustfs",
-        "rustfs": {
-            "name": TIER_NAME,
-            "endpoint": cold.url.as_str(),
-            "accessKey": cold.access_key.as_str(),
-            "secretKey": cold.secret_key.as_str(),
-            "bucket": TIER_BUCKET,
-            "prefix": TIER_PREFIX,
-            "region": "us-east-1",
-            "storageClass": ""
-        }
-    })
-    .to_string();
-
-    let (status, resp) = signed_admin_request(
-        &hot.url,
-        Method::PUT,
-        "/rustfs/admin/v3/tier",
-        Some(&body),
-        &hot.access_key,
-        &hot.secret_key,
-    )
-    .await?;
-    if !status.is_success() {
-        return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
-    }
-    Ok(())
-}
-
-/// A current-version `Transition Days=0` rule scoped to the object's prefix.
-fn transition_rule() -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
-    Ok(LifecycleRule::builder()
-        .id("ilm7-transition")
-        .filter(LifecycleRuleFilter::builder().prefix("tier/").build())
-        .transitions(
-            Transition::builder()
-                .days(0)
-                .storage_class(TransitionStorageClass::from(TIER_NAME))
-                .build(),
-        )
-        .status(ExpirationStatus::Enabled)
-        .build()?)
-}
-
-/// Upload `data` as a two-part multipart object with a content-type and one
-/// user-metadata entry.
-async fn put_multipart_object(client: &Client, bucket: &str, key: &str, data: &[u8]) -> TestResult {
-    let create = client
-        .create_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .content_type(CONTENT_TYPE)
-        .metadata(USER_META_KEY, USER_META_VAL)
-        .send()
-        .await?;
-    let upload_id = create
-        .upload_id()
-        .ok_or("CreateMultipartUpload returned no upload id")?
-        .to_string();
-
-    let mut completed = Vec::new();
-    for (idx, chunk) in [&data[..PART0_SIZE], &data[PART0_SIZE..]].into_iter().enumerate() {
-        let part_number = (idx + 1) as i32;
-        let uploaded = client
-            .upload_part()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(&upload_id)
-            .part_number(part_number)
-            .body(ByteStream::from(chunk.to_vec()))
-            .send()
-            .await?;
-        completed.push(
-            CompletedPart::builder()
-                .part_number(part_number)
-                .e_tag(uploaded.e_tag().unwrap_or_default())
-                .build(),
-        );
-    }
-
-    client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(&upload_id)
-        .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
-        .send()
-        .await?;
-    Ok(())
-}
-
-/// Number of objects currently stored in the cold-tier bucket.
-async fn cold_tier_object_count(cold_client: &Client) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
-    let resp = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
-    Ok(resp.contents().len())
-}
-
-/// Poll `HEAD` until the object's storage class is the tier name (transition
-/// complete), or fail after `deadline`.
-async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
-    let start = Instant::now();
-    loop {
-        let head = client.head_object().bucket(bucket).key(key).send().await?;
-        if head.storage_class().map(|sc| sc.as_str()) == Some(TIER_NAME) {
-            return Ok(());
-        }
-        if start.elapsed() >= deadline {
-            return Err(format!(
-                "object {bucket}/{key} was not transitioned to {TIER_NAME} within {}s (storage_class={:?})",
-                deadline.as_secs(),
-                head.storage_class()
-            )
-            .into());
-        }
-        tokio::time::sleep(StdDuration::from_millis(500)).await;
-    }
-}
-
-/// Poll until the cold-tier bucket is empty (remote free-version cleanup done),
-/// or fail after `deadline`.
-async fn wait_for_cold_tier_empty(cold_client: &Client, deadline: StdDuration) -> TestResult {
-    let start = Instant::now();
-    loop {
-        let count = cold_tier_object_count(cold_client).await?;
-        if count == 0 {
-            return Ok(());
-        }
-        if start.elapsed() >= deadline {
-            return Err(format!(
-                "cold-tier bucket still holds {count} object(s) {}s after DeleteObject; \
-                 free-version remote cleanup did not converge",
-                deadline.as_secs()
-            )
-            .into());
-        }
-        tokio::time::sleep(StdDuration::from_millis(500)).await;
-    }
-}
-
-/// GET `bytes=start-end` (inclusive) and assert it equals `data[start..=end]`.
-async fn assert_range(client: &Client, start: usize, end: usize, data: &[u8]) -> TestResult {
-    let range = format!("bytes={start}-{end}");
-    let resp = client
-        .get_object()
-        .bucket(SOURCE_BUCKET)
-        .key(OBJECT_KEY)
-        .range(&range)
-        .send()
-        .await?;
-    let got = resp.body.collect().await?.into_bytes();
-    let expected = &data[start..=end];
-    assert_eq!(got.len(), expected.len(), "range {range}: length mismatch");
-    assert_eq!(got.as_ref(), expected, "range {range}: bytes mismatch reading from the tier");
-    Ok(())
-}
-
-/// Full ilm-7 hermetic transition main path across two embedded RustFS servers.
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_hermetic_transition_main_path() -> TestResult {
-    // Cold-tier server (independent credentials). It is a passive tier target,
-    // so it needs no lifecycle/scanner configuration.
-    let mut cold = RustFSTestEnvironment::new().await?;
-    cold.access_key = "coldtieradmin".to_string();
-    cold.secret_key = "coldtiersecret".to_string();
-    cold.start_rustfs_server_without_cleanup(vec![]).await?;
-    let cold_client = cold.create_s3_client();
-    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
-
-    // Hot/source server. A 1s scanner cycle is a backstop; transition is
-    // primarily driven immediately by the multipart completion path.
-    let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1")])
-        .await?;
-    let hot_client = hot.create_s3_client();
-
-    // Wire the RustFS remote tier (real connectivity probe, no force).
-    add_rustfs_tier(&hot, &cold).await?;
-
-    // Source bucket + Days=0 transition rule installed BEFORE the write so the
-    // completion path enqueues the transition immediately.
-    hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?;
-    let lifecycle = BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?;
-    hot_client
-        .put_bucket_lifecycle_configuration()
-        .bucket(SOURCE_BUCKET)
-        .lifecycle_configuration(lifecycle)
-        .send()
-        .await?;
-
-    let data = payload();
-    put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?;
-
-    // 1) Transition completes: HEAD reports the tier name and no restore state.
-    wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?;
-    let head = hot_client.head_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
-    assert_eq!(
-        head.storage_class().map(|sc| sc.as_str()),
-        Some(TIER_NAME),
-        "transitioned object must report the tier name as its storage class"
-    );
-    assert!(
-        head.restore().is_none(),
-        "a freshly transitioned object must not advertise x-amz-restore, got {:?}",
-        head.restore()
-    );
-
-    // 2) The remote object now lives in the cold-tier bucket.
-    assert!(
-        cold_tier_object_count(&cold_client).await? >= 1,
-        "cold-tier bucket must hold the transitioned object"
-    );
-
-    // 3) GET streams identical bytes back through the warm backend; content-type
-    //    and user metadata survive the transition round trip (rustfs#2246).
-    let get = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
-    assert_eq!(get.content_type(), Some(CONTENT_TYPE), "content-type must survive transition");
-    assert_eq!(
-        get.metadata().and_then(|m| m.get(USER_META_KEY)).map(String::as_str),
-        Some(USER_META_VAL),
-        "user metadata must survive transition"
-    );
-    let body = get.body.collect().await?.into_bytes();
-    assert_eq!(body.len(), data.len(), "full transitioned GET length mismatch");
-    assert_eq!(body.as_ref(), data.as_slice(), "full transitioned GET must be byte-identical");
-
-    // 4) Range GET within a single part and across the part boundary
-    //    (backlog#807): both must read the correct bytes from the tier.
-    assert_range(&hot_client, 1000, 1099, &data).await?;
-    assert_range(&hot_client, PART0_SIZE - 5, PART0_SIZE + 4, &data).await?;
-
-    // 5) DeleteObject drives free-version cleanup. The local object is gone
-    //    immediately; the remote copy is removed asynchronously.
-    hot_client
-        .delete_object()
-        .bucket(SOURCE_BUCKET)
-        .key(OBJECT_KEY)
-        .send()
-        .await?;
-
-    let get_after = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await;
-    let err = get_after.expect_err("hot object must be gone immediately after delete");
-    assert_eq!(
-        err.as_service_error().and_then(|e| e.code()),
-        Some("NoSuchKey"),
-        "hot GET after delete must be NoSuchKey, got {err:?}"
-    );
-
-    wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?;
-
-    Ok(())
-}
@@ -56,32 +56,6 @@ type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
 /// fail-closed and every other e2e scenario keeps exercising the production SSRF policy.
 const LOOPBACK_REPLICATION_TARGET_ENV: &[(&str, &str)] = &[("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true")];

-/// Short data-scanner cycle for the failure-recovery tests (backlog#1147 repl-5).
-///
-/// When a replication target is unreachable, `replicate_object` marks the source
-/// object's status FAILED in `xl.meta` (it is NOT queued to the on-disk MRF
-/// overflow file, which only backstops worker-queue saturation). The mechanism
-/// that re-drives those persisted PENDING/FAILED objects — including after a
-/// source restart — is the data scanner's replication heal pass
-/// (`heal_replication` -> `queue_replication_heal`). The scanner cycle floors at
-/// 1s, so this pins it to the floor to keep convergence within seconds. Combine
-/// with [`replication_fast_env`] (health-check / MRF-flush / resync polling) so
-/// every recovery loop runs at its minimum interval and each scenario finishes
-/// well under the two-minute budget.
-///
-/// `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1` matters for changes to
-/// already-scanned objects (e.g. a delete marker stacked on a replicated key):
-/// existing compacted folders are hash-sharded across that many cycles before
-/// being rescanned (default 16, `scanner_folder.rs`), so on a long-lived source
-/// a failed delete-marker replication may otherwise wait 16 scan cycles before
-/// the heal pass revisits it. New keys are unaffected (new folders are always
-/// scanned), which is why only restart-free recovery of EXISTING keys needs it.
-const FAST_SCANNER_ENV: &[(&str, &str)] = &[
-    ("RUSTFS_SCANNER_CYCLE", "1"),
-    ("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
-    ("RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES", "1"),
-];
-
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
    #[serde(rename = "Targets", default)]
@@ -562,7 +536,6 @@ async fn start_https_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Pa
        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
        .env("RUSTFS_TLS_PATH", tls_dir)
        .env("RUSTFS_CONSOLE_ENABLE", "false")
-        .env("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true")
        .args([
            "--address",
            &env.address,
@@ -676,7 +649,7 @@ async fn wait_for_replicated_object_over_https(
            }
            status if tokio::time::Instant::now() < deadline => {
                let body = response.text().await.unwrap_or_default();
-                if body.contains("NoSuchKey") || body.contains("NoSuchBucket") || body.contains("NotFound") {
+                if body.contains("NoSuchKey") || body.contains("NotFound") {
                    sleep(Duration::from_secs(1)).await;
                    continue;
                }
@@ -971,65 +944,6 @@ async fn assert_replication_converged(
    }
 }

-/// Poll the source object until it reports a not-yet-replicated status.
-///
-/// A source object with a reachable replication config carries an
-/// `x-amz-replication-status` header (surfaced by the SDK as
-/// `replication_status()`). While the target is down it must read `PENDING` or
-/// `FAILED`; it can never be `COMPLETED`. Used by the failure-recovery tests to
-/// prove the outage was actually observed before recovery is driven.
-async fn wait_for_source_replication_pending_or_failed(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-) -> Result<(), Box<dyn Error + Send + Sync>> {
-    let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
-    loop {
-        let head = client.head_object().bucket(bucket).key(key).send().await?;
-        match head.replication_status().map(|status| status.as_str()) {
-            Some("PENDING") | Some("FAILED") => return Ok(()),
-            other => {
-                if tokio::time::Instant::now() >= deadline {
-                    return Err(format!("source object {key} never reported PENDING/FAILED; last status={other:?}").into());
-                }
-                sleep(Duration::from_millis(200)).await;
-            }
-        }
-    }
-}
-
-/// Return the `LastModified` of the (single) delete marker for `key`, if present.
-async fn delete_marker_last_modified(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-) -> Result<Option<aws_sdk_s3::primitives::DateTime>, Box<dyn Error + Send + Sync>> {
-    let output = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
-    Ok(output
-        .delete_markers()
-        .iter()
-        .filter(|marker| marker.key() == Some(key))
-        .find_map(|marker| marker.last_modified().cloned()))
-}
-
-/// Poll the target until a delete marker for `key` appears, returning its mtime.
-async fn wait_for_target_delete_marker(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-) -> Result<aws_sdk_s3::primitives::DateTime, Box<dyn Error + Send + Sync>> {
-    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
-    loop {
-        if let Some(mtime) = delete_marker_last_modified(client, bucket, key).await? {
-            return Ok(mtime);
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err(format!("target never received a delete marker for {key}").into());
-        }
-        sleep(Duration::from_millis(250)).await;
-    }
-}
-
 async fn run_replication_check(
    env: &RustFSTestEnvironment,
    bucket: &str,
@@ -2930,272 +2844,6 @@ async fn test_single_bucket_multipart_replication_fans_out_to_multiple_targets()
    Ok(())
 }

-/// backlog#1147 repl-5, scenario (a) — target outage + recovery (rustfs#3421 / #2071).
-///
-/// Kills the replication target mid-workload, asserts the source records the
-/// undelivered objects as PENDING/FAILED, then recovers the target with its data
-/// directory intact and asserts both sides converge with no data loss. The
-/// still-running source's data scanner (short cycle via [`FAST_SCANNER_ENV`])
-/// re-drives the failed objects once the target is reachable again.
-#[tokio::test]
-#[serial]
-async fn test_bucket_replication_recovers_after_target_outage() -> TestResult {
-    init_logging();
-
-    let mut source_env = RustFSTestEnvironment::new().await?;
-    let mut source_env_vars = replication_fast_env();
-    source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    source_env_vars.extend_from_slice(FAST_SCANNER_ENV);
-    source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
-
-    let mut target_env = RustFSTestEnvironment::new().await?;
-    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
-
-    let source_bucket = "repl-outage-recovery-src";
-    let target_bucket = "repl-outage-recovery-dst";
-    let source_client = source_env.create_s3_client();
-    let target_client = target_env.create_s3_client();
-
-    source_client.create_bucket().bucket(source_bucket).send().await?;
-    target_client.create_bucket().bucket(target_bucket).send().await?;
-    enable_bucket_versioning(&source_env, source_bucket).await?;
-    enable_bucket_versioning(&target_env, target_bucket).await?;
-
-    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
-    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
-
-    // Baseline object replicated while the target is healthy; it must remain on
-    // both sides through the outage (no loss of already-replicated data).
-    source_client
-        .put_object()
-        .bucket(source_bucket)
-        .key("before-outage.txt")
-        .body(ByteStream::from_static(b"baseline written before the target outage"))
-        .send()
-        .await?;
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    // Target outage: everything written now cannot replicate.
-    target_env.stop_server();
-
-    let outage_keys = ["during-outage-1.txt", "during-outage-2.txt"];
-    for key in outage_keys {
-        source_client
-            .put_object()
-            .bucket(source_bucket)
-            .key(key)
-            .body(ByteStream::from(format!("written while the target was down: {key}").into_bytes()))
-            .send()
-            .await?;
-    }
-
-    // The outage is observable on the source: the objects are not yet replicated.
-    wait_for_source_replication_pending_or_failed(&source_client, source_bucket, outage_keys[0]).await?;
-
-    // Recover the target in place; the source scanner re-drives the failures.
-    target_env.restart_server_preserving_data(vec![], &[]).await?;
-
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    // Explicit no-loss check: baseline + every outage write reached the target.
-    let target_state = list_replication_state(&target_client, target_bucket).await?;
-    for key in ["before-outage.txt"].into_iter().chain(outage_keys) {
-        assert!(
-            target_state.iter().any(|entry| entry.key == key && !entry.delete_marker),
-            "target missing object {key} after outage recovery; state={target_state:?}"
-        );
-    }
-
-    Ok(())
-}
-
-/// backlog#1147 repl-5, scenario (b) — failure state survives a source restart
-/// (mirrors backlog#858 delete-decision re-derivation and #859 no-drop).
-///
-/// Several object writes plus a delete of an already-replicated object all fail
-/// while the target is down. The SOURCE is then restarted with the target still
-/// unreachable, so replication can only resume from the per-object status
-/// persisted in `xl.meta` — nothing in memory survives. Bringing the target back
-/// must converge every persisted failure, including the replayed delete marker
-/// (whose replication decision is re-derived from the live config).
-#[tokio::test]
-#[serial]
-async fn test_bucket_replication_replays_failed_entries_after_source_restart() -> TestResult {
-    init_logging();
-
-    let mut source_env = RustFSTestEnvironment::new().await?;
-    let mut source_env_vars = replication_fast_env();
-    source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    source_env_vars.extend_from_slice(FAST_SCANNER_ENV);
-    source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
-
-    let mut target_env = RustFSTestEnvironment::new().await?;
-    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
-
-    let source_bucket = "repl-source-restart-src";
-    let target_bucket = "repl-source-restart-dst";
-    let source_client = source_env.create_s3_client();
-    let target_client = target_env.create_s3_client();
-
-    source_client.create_bucket().bucket(source_bucket).send().await?;
-    target_client.create_bucket().bucket(target_bucket).send().await?;
-    enable_bucket_versioning(&source_env, source_bucket).await?;
-    enable_bucket_versioning(&target_env, target_bucket).await?;
-
-    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
-    put_bucket_replication_with_delete_statuses(&source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
-
-    // Replicate an object before the outage so its later deletion produces a
-    // delete marker whose replay decision must be re-derived (backlog#858).
-    let deleted_key = "deleted-during-outage.txt";
-    source_client
-        .put_object()
-        .bucket(source_bucket)
-        .key(deleted_key)
-        .body(ByteStream::from_static(b"exists on both sides before deletion"))
-        .send()
-        .await?;
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    // Outage: two fresh objects plus a delete marker all fail to replicate.
-    target_env.stop_server();
-
-    let failed_keys = ["failed-1.txt", "failed-2.txt"];
-    for key in failed_keys {
-        source_client
-            .put_object()
-            .bucket(source_bucket)
-            .key(key)
-            .body(ByteStream::from(
-                format!("queued for replay while the target was down: {key}").into_bytes(),
-            ))
-            .send()
-            .await?;
-    }
-    source_client
-        .delete_object()
-        .bucket(source_bucket)
-        .key(deleted_key)
-        .send()
-        .await?;
-
-    wait_for_source_replication_pending_or_failed(&source_client, source_bucket, failed_keys[0]).await?;
-
-    // Restart the SOURCE while the target is still down: recovery must reload the
-    // failure state from disk, since no in-memory queue survives the restart.
-    source_env.restart_server_preserving_data(vec![], &source_env_vars).await?;
-
-    // Bring the target back; the restarted source re-drives every persisted
-    // failure to convergence.
-    target_env.restart_server_preserving_data(vec![], &[]).await?;
-
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    // No entry dropped across the restart (backlog#859) and the delete marker
-    // replayed to the target (backlog#858).
-    let target_state = list_replication_state(&target_client, target_bucket).await?;
-    for key in failed_keys {
-        assert!(
-            target_state.iter().any(|entry| entry.key == key && !entry.delete_marker),
-            "source-restart replay dropped object {key}; state={target_state:?}"
-        );
-    }
-    assert!(
-        target_state
-            .iter()
-            .any(|entry| entry.key == deleted_key && entry.delete_marker),
-        "replayed delete marker for {deleted_key} did not reach the target (backlog#858); state={target_state:?}"
-    );
-
-    Ok(())
-}
-
-/// backlog#1147 repl-5, scenario (c) — replayed delete marker keeps the source
-/// mtime (mirrors backlog#867).
-///
-/// A delete marker is created while the target is down; the source is then
-/// restarted (data preserved) and the target brought back, so the marker
-/// replicates through the failure-replay path. The replayed marker must carry
-/// the SOURCE's `LastModified`, not the replay time. A deliberate gap before
-/// recovery makes any regression (replay-time stamping) obvious.
-///
-/// The source restart is load-bearing, not just paranoia: on a live
-/// (never-restarted) source, the failed delete-marker replication wedges the
-/// per-object `/[replicate]/<key>` namespace lock and the marker never
-/// replicates even after the target recovers — tracked as backlog#1278. Once
-/// that is fixed, a restart-free variant of this scenario should be added.
-#[tokio::test]
-#[serial]
-async fn test_bucket_replication_replayed_delete_marker_preserves_source_mtime() -> TestResult {
-    init_logging();
-
-    let mut source_env = RustFSTestEnvironment::new().await?;
-    let mut source_env_vars = replication_fast_env();
-    source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    source_env_vars.extend_from_slice(FAST_SCANNER_ENV);
-    source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
-
-    let mut target_env = RustFSTestEnvironment::new().await?;
-    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
-
-    let source_bucket = "repl-dm-mtime-src";
-    let target_bucket = "repl-dm-mtime-dst";
-    let object_key = "delete-marker-mtime.txt";
-    let source_client = source_env.create_s3_client();
-    let target_client = target_env.create_s3_client();
-
-    source_client.create_bucket().bucket(source_bucket).send().await?;
-    target_client.create_bucket().bucket(target_bucket).send().await?;
-    enable_bucket_versioning(&source_env, source_bucket).await?;
-    enable_bucket_versioning(&target_env, target_bucket).await?;
-
-    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
-    put_bucket_replication_with_delete_statuses(&source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
-
-    source_client
-        .put_object()
-        .bucket(source_bucket)
-        .key(object_key)
-        .body(ByteStream::from_static(b"object to be delete-marked during the outage"))
-        .send()
-        .await?;
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    // Create the delete marker while the target is down so it replicates through
-    // the failure-replay path rather than the immediate path. (HEAD on the key
-    // now returns the delete marker, so the source status is read from the
-    // version listing instead of head_object.)
-    target_env.stop_server();
-    let delete = source_client
-        .delete_object()
-        .bucket(source_bucket)
-        .key(object_key)
-        .send()
-        .await?;
-    assert_eq!(delete.delete_marker(), Some(true));
-
-    let source_mtime = delete_marker_last_modified(&source_client, source_bucket, object_key)
-        .await?
-        .ok_or("source has no delete marker after DELETE")?;
-
-    // Widen the gap so a replay-time-stamping regression is unmistakable.
-    sleep(Duration::from_secs(3)).await;
-
-    // Restart the source (see the doc comment: live-source replay is wedged by
-    // backlog#1278), then bring the target back; the restarted source's scanner
-    // heal pass replays the failed delete marker.
-    source_env.restart_server_preserving_data(vec![], &source_env_vars).await?;
-    target_env.restart_server_preserving_data(vec![], &[]).await?;
-
-    let target_mtime = wait_for_target_delete_marker(&target_client, target_bucket, object_key).await?;
-    assert_eq!(
-        target_mtime, source_mtime,
-        "replayed delete marker did not preserve the source mtime (backlog#867): source={source_mtime:?}, target={target_mtime:?}"
-    );
-
-    Ok(())
-}
-
 #[tokio::test]
 #[serial]
 async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -3284,178 +2932,6 @@ async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Re
    Ok(())
 }

-#[tokio::test]
-#[serial]
-async fn test_site_replication_allows_self_signed_https_with_skip_tls_verify_real_dual_node() -> TestResult {
-    init_logging();
-
-    let mut source_env = new_replication_source_env().await?;
-    source_env
-        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
-        .await?;
-
-    let mut target_env = new_replication_https_target_env().await?;
-    let tls_dir = std::path::PathBuf::from(&target_env.temp_dir).join("tls");
-    let target_host = target_env
-        .url
-        .trim_start_matches("https://")
-        .split(':')
-        .next()
-        .ok_or_else(|| std::io::Error::other("target HTTPS URL missing host"))?
-        .to_string();
-    generate_self_signed_tls_material(&tls_dir, &target_host).await?;
-    start_https_rustfs_server(&mut target_env, &tls_dir).await?;
-    let https_client = insecure_https_client()?;
-    wait_for_https_server_ready(&https_client, &target_env).await?;
-
-    let source_site = PeerSite {
-        name: "source-site".to_string(),
-        endpoint: source_env.url.clone(),
-        access_key: source_env.access_key.clone(),
-        secret_key: source_env.secret_key.clone(),
-        ..Default::default()
-    };
-    let target_site = PeerSite {
-        name: "target-site".to_string(),
-        endpoint: target_env.url.clone(),
-        access_key: target_env.access_key.clone(),
-        secret_key: target_env.secret_key.clone(),
-        ..Default::default()
-    };
-
-    let add_error = site_replication_add(&source_env, &[source_site.clone(), target_site.clone()])
-        .await
-        .expect_err("site replication add must reject an untrusted self-signed HTTPS peer");
-    let add_error = add_error.to_string();
-    assert!(
-        add_error.contains("400 Bad Request"),
-        "unexpected untrusted self-signed peer error: {add_error}"
-    );
-    assert!(
-        add_error.to_ascii_lowercase().contains("tls") || add_error.to_ascii_lowercase().contains("certificate"),
-        "self-signed peer rejection did not report a TLS certificate error: {add_error}"
-    );
-    let disabled = wait_for_site_replication_disabled(&source_env).await?;
-    assert!(!disabled.enabled && disabled.sites.is_empty());
-
-    let add_status = site_replication_add(
-        &source_env,
-        &[
-            source_site,
-            PeerSite {
-                skip_tls_verify: true,
-                ..target_site
-            },
-        ],
-    )
-    .await?;
-    assert!(add_status.success, "unexpected site add result: {add_status:?}");
-    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
-
-    let source_client = source_env.create_s3_client();
-    let bucket = "site-repl-self-signed-tls";
-    let key = "self-signed.txt";
-    let body = "site replication over self-signed https";
-    source_client.create_bucket().bucket(bucket).send().await?;
-    source_client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from(body.as_bytes().to_vec()))
-        .send()
-        .await?;
-
-    wait_for_replicated_object_over_https(&https_client, &target_env, bucket, key, body).await?;
-
-    Ok(())
-}
-
-#[tokio::test]
-#[serial]
-async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dual_node() -> TestResult {
-    init_logging();
-
-    let mut source_env = new_replication_source_env().await?;
-    source_env
-        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
-        .await?;
-
-    let mut target_env = new_replication_https_target_env().await?;
-    let tls_dir = std::path::PathBuf::from(&target_env.temp_dir).join("tls");
-    let target_host = target_env
-        .url
-        .trim_start_matches("https://")
-        .split(':')
-        .next()
-        .ok_or_else(|| std::io::Error::other("target HTTPS URL missing host"))?
-        .to_string();
-    let ca_cert_pem = generate_private_ca_tls_material(&tls_dir, &target_host).await?;
-    start_https_rustfs_server(&mut target_env, &tls_dir).await?;
-    let https_client = trusted_https_client(&ca_cert_pem)?;
-    wait_for_https_server_ready(&https_client, &target_env).await?;
-
-    let source_site = PeerSite {
-        name: "source-site".to_string(),
-        endpoint: source_env.url.clone(),
-        access_key: source_env.access_key.clone(),
-        secret_key: source_env.secret_key.clone(),
-        ..Default::default()
-    };
-    let target_site = PeerSite {
-        name: "target-site".to_string(),
-        endpoint: target_env.url.clone(),
-        access_key: target_env.access_key.clone(),
-        secret_key: target_env.secret_key.clone(),
-        ..Default::default()
-    };
-
-    let add_error = site_replication_add(&source_env, &[source_site.clone(), target_site.clone()])
-        .await
-        .expect_err("site replication add must reject a private CA HTTPS peer without caCertPem");
-    let add_error = add_error.to_string();
-    assert!(
-        add_error.contains("400 Bad Request"),
-        "unexpected untrusted private CA peer error: {add_error}"
-    );
-    assert!(
-        add_error.to_ascii_lowercase().contains("tls") || add_error.to_ascii_lowercase().contains("certificate"),
-        "private CA peer rejection did not report a TLS certificate error: {add_error}"
-    );
-    let disabled = wait_for_site_replication_disabled(&source_env).await?;
-    assert!(!disabled.enabled && disabled.sites.is_empty());
-
-    let add_status = site_replication_add(
-        &source_env,
-        &[
-            source_site,
-            PeerSite {
-                ca_cert_pem,
-                ..target_site
-            },
-        ],
-    )
-    .await?;
-    assert!(add_status.success, "unexpected site add result: {add_status:?}");
-    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
-
-    let source_client = source_env.create_s3_client();
-    let bucket = "site-repl-private-ca-tls";
-    let key = "private-ca.txt";
-    let body = "site replication over private ca https";
-    source_client.create_bucket().bucket(bucket).send().await?;
-    source_client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from(body.as_bytes().to_vec()))
-        .send()
-        .await?;
-
-    wait_for_replicated_object_over_https(&https_client, &target_env, bucket, key, body).await?;
-
-    Ok(())
-}
-
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -3487,14 +2963,12 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -3618,21 +3092,18 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "relay-site".to_string(),
                endpoint: relay_env.url.clone(),
                access_key: relay_env.access_key.clone(),
                secret_key: relay_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -3855,14 +3326,12 @@ async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -3965,14 +3434,12 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -4078,14 +3545,12 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -4162,14 +3627,12 @@ async fn test_site_replication_replicates_policy_backed_user_access_real_dual_no
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -4250,14 +3713,12 @@ async fn test_site_replication_replicates_group_policy_backed_access_real_dual_n
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -4380,14 +3841,12 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -4484,14 +3943,12 @@ async fn test_site_replication_replicates_service_accounts_created_from_sts_sess
                endpoint: source_env.url.clone(),
                access_key: source_env.access_key.clone(),
                secret_key: source_env.secret_key.clone(),
-                ..Default::default()
            },
            PeerSite {
                name: "target-site".to_string(),
                endpoint: target_env.url.clone(),
                access_key: target_env.access_key.clone(),
                secret_key: target_env.secret_key.clone(),
-                ..Default::default()
            },
        ],
    )
@@ -1,306 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Live-listener TLS certificate hot-reload e2e (backlog#1154 peri-5).
-//!
-//! `crates/tls-runtime` had ~20 unit tests but no test ever rotated a
-//! certificate under a listening HTTPS server, so the operational promise
-//! "swap certificates without a restart" (RUSTFS_TLS_RELOAD_ENABLE) had no
-//! automated proof. This suite pins, against a real binary over real TLS
-//! handshakes:
-//!
-//!   * the server starts with certificate A and serves its fingerprint;
-//!   * after the on-disk material is replaced with certificate B, new
-//!     connections receive B within the reload interval — no restart;
-//!   * a connection established under A keeps working across the swap
-//!     (existing sessions are not torn down);
-//!   * replacing the material with garbage does not take down the listener or
-//!     the previously loaded certificate (fail-safe, no fail-open) and leaves
-//!     an assertable `tls_reload_failed` log event.
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use rcgen::generate_simple_self_signed;
-use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
-use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
-use rustls::{ClientConfig, ClientConnection, DigitallySignedStruct, Error as RustlsError, SignatureScheme, StreamOwned};
-use serial_test::serial;
-use sha2::{Digest, Sha256};
-use std::error::Error;
-use std::io::{Read, Write};
-use std::net::TcpStream;
-use std::sync::Arc;
-use tokio::time::{Duration, sleep};
-
-type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BoxError = Box<dyn Error + Send + Sync>;
-
-const CERT_FILE: &str = "rustfs_cert.pem";
-const KEY_FILE: &str = "rustfs_key.pem";
-/// Minimum value the server accepts for RUSTFS_TLS_RELOAD_INTERVAL (seconds).
-const RELOAD_INTERVAL_SECS: u64 = 5;
-
-#[derive(Debug)]
-struct AcceptAnyServerCertVerifier;
-
-impl ServerCertVerifier for AcceptAnyServerCertVerifier {
-    fn verify_server_cert(
-        &self,
-        _end_entity: &CertificateDer<'_>,
-        _intermediates: &[CertificateDer<'_>],
-        _server_name: &ServerName<'_>,
-        _ocsp_response: &[u8],
-        _now: UnixTime,
-    ) -> Result<ServerCertVerified, RustlsError> {
-        Ok(ServerCertVerified::assertion())
-    }
-
-    fn verify_tls12_signature(
-        &self,
-        _message: &[u8],
-        _cert: &CertificateDer<'_>,
-        _dss: &DigitallySignedStruct,
-    ) -> Result<HandshakeSignatureValid, RustlsError> {
-        Ok(HandshakeSignatureValid::assertion())
-    }
-
-    fn verify_tls13_signature(
-        &self,
-        _message: &[u8],
-        _cert: &CertificateDer<'_>,
-        _dss: &DigitallySignedStruct,
-    ) -> Result<HandshakeSignatureValid, RustlsError> {
-        Ok(HandshakeSignatureValid::assertion())
-    }
-
-    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
-        rustls::crypto::aws_lc_rs::default_provider()
-            .signature_verification_algorithms
-            .supported_schemes()
-    }
-}
-
-fn tls_client_config() -> Arc<ClientConfig> {
-    Arc::new(
-        ClientConfig::builder()
-            .dangerous()
-            .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
-            .with_no_client_auth(),
-    )
-}
-
-/// A live TLS session over a blocking TCP stream, kept open across reloads.
-struct TlsSession {
-    stream: StreamOwned<ClientConnection, TcpStream>,
-    fingerprint: String,
-    host: String,
-}
-
-/// Completes a TLS handshake with `addr` and returns the session together with
-/// the SHA-256 fingerprint of the served leaf certificate.
-fn tls_connect(addr: &str) -> Result<TlsSession, BoxError> {
-    let host = addr.split(':').next().unwrap_or("127.0.0.1").to_string();
-    let server_name = ServerName::try_from(host.clone())?;
-    let mut conn = ClientConnection::new(tls_client_config(), server_name)?;
-    let mut tcp = TcpStream::connect(addr)?;
-    tcp.set_read_timeout(Some(std::time::Duration::from_secs(10)))?;
-    tcp.set_write_timeout(Some(std::time::Duration::from_secs(10)))?;
-
-    // Drive the handshake to completion so peer_certificates() is populated.
-    while conn.is_handshaking() {
-        conn.complete_io(&mut tcp)?;
-    }
-
-    let leaf = conn
-        .peer_certificates()
-        .and_then(|certs| certs.first())
-        .ok_or("server presented no certificate")?;
-    let fingerprint = Sha256::digest(leaf.as_ref())
-        .iter()
-        .map(|b| format!("{b:02x}"))
-        .collect::<String>();
-
-    Ok(TlsSession {
-        stream: StreamOwned::new(conn, tcp),
-        fingerprint,
-        host,
-    })
-}
-
-/// Sends a keep-alive HTTP request on the session and reads the response head,
-/// proving the TLS session is still fully functional.
-fn http_roundtrip(session: &mut TlsSession) -> Result<String, BoxError> {
-    let request = format!("GET / HTTP/1.1\r\nHost: {}\r\nConnection: keep-alive\r\n\r\n", session.host);
-    session.stream.write_all(request.as_bytes())?;
-    session.stream.flush()?;
-
-    // Read up to the end of the headers plus a body chunk; anonymous ListBuckets
-    // answers a small error payload, which is fine — we only need a valid
-    // HTTP status line over the existing TLS session.
-    let mut buf = vec![0_u8; 8192];
-    let read = session.stream.read(&mut buf)?;
-    if read == 0 {
-        return Err("connection closed by server".into());
-    }
-    let head = String::from_utf8_lossy(&buf[..read]).to_string();
-    if !head.starts_with("HTTP/1.1") {
-        return Err(format!("unexpected response head: {head}").into());
-    }
-    Ok(head)
-}
-
-async fn write_cert_pair(tls_dir: &std::path::Path, sans: Vec<String>) -> Result<(), BoxError> {
-    let cert = generate_simple_self_signed(sans)?;
-    tokio::fs::write(tls_dir.join(CERT_FILE), cert.cert.pem()).await?;
-    tokio::fs::write(tls_dir.join(KEY_FILE), cert.signing_key.serialize_pem()).await?;
-    Ok(())
-}
-
-/// Polls new TLS connections until the served fingerprint changes away from
-/// `old_fingerprint`, returning the new one.
-async fn wait_for_new_fingerprint(addr: &str, old_fingerprint: &str, within: Duration) -> Result<String, BoxError> {
-    let deadline = tokio::time::Instant::now() + within;
-    loop {
-        let addr_owned = addr.to_string();
-        let observed = tokio::task::spawn_blocking(move || tls_connect(&addr_owned).map(|s| s.fingerprint)).await?;
-        if let Ok(fp) = observed
-            && fp != old_fingerprint
-        {
-            return Ok(fp);
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err(format!("served certificate never rotated away from {old_fingerprint} within {within:?}").into());
-        }
-        sleep(Duration::from_secs(1)).await;
-    }
-}
-
-/// Starts the rustfs binary with HTTPS + hot reload enabled, capturing its
-/// stdout/stderr to `log_path`. The harness's own start path is unusable here:
-/// its readiness probe drives the AWS SDK over the environment URL, and the SDK
-/// rejects the self-signed test certificate.
-async fn start_https_server_with_reload(
-    env: &mut RustFSTestEnvironment,
-    tls_dir: &std::path::Path,
-    log_path: &str,
-) -> TestResult {
-    let log_file = std::fs::OpenOptions::new().create(true).append(true).open(log_path)?;
-    let log_file_err = log_file.try_clone()?;
-    let process = std::process::Command::new(crate::common::rustfs_binary_path())
-        .env("RUST_LOG", "rustfs=info")
-        .env("RUSTFS_CONSOLE_ENABLE", "false")
-        .env("RUSTFS_TLS_PATH", tls_dir)
-        .env("RUSTFS_TLS_RELOAD_ENABLE", "true")
-        .env("RUSTFS_TLS_RELOAD_INTERVAL", RELOAD_INTERVAL_SECS.to_string())
-        .stdout(std::process::Stdio::from(log_file))
-        .stderr(std::process::Stdio::from(log_file_err))
-        .args([
-            "--address",
-            &env.address,
-            "--access-key",
-            &env.access_key,
-            "--secret-key",
-            &env.secret_key,
-            &env.temp_dir,
-        ])
-        .spawn()?;
-    env.process = Some(process);
-
-    // Readiness = a TLS handshake completes on the listener.
-    let addr = env.address.clone();
-    for attempt in 0..60 {
-        let addr_clone = addr.clone();
-        if tokio::task::spawn_blocking(move || tls_connect(&addr_clone)).await?.is_ok() {
-            return Ok(());
-        }
-        if attempt == 59 {
-            return Err("HTTPS server never completed a TLS handshake within 60s".into());
-        }
-        sleep(Duration::from_secs(1)).await;
-    }
-    Ok(())
-}
-
-/// Runs `http_roundtrip` on a session inside the blocking pool, handing the
-/// session back for later reuse (the point: the same TLS session survives).
-async fn roundtrip_and_return(mut session: TlsSession) -> Result<TlsSession, BoxError> {
-    tokio::task::spawn_blocking(move || {
-        http_roundtrip(&mut session)?;
-        Ok::<_, BoxError>(session)
-    })
-    .await?
-}
-
-#[tokio::test]
-#[serial]
-async fn test_tls_certificate_hot_reload_live_listener() -> TestResult {
-    init_logging();
-    // Install the process-wide rustls crypto provider (idempotent).
-    let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    let tls_dir = std::path::PathBuf::from(format!("{}/tls", env.temp_dir));
-    tokio::fs::create_dir_all(&tls_dir).await?;
-    write_cert_pair(&tls_dir, vec!["localhost".into(), "127.0.0.1".into()]).await?;
-
-    let log_path = format!("{}/server.log", env.temp_dir);
-    start_https_server_with_reload(&mut env, &tls_dir, &log_path).await?;
-
-    let addr = env.address.clone();
-
-    // --- certificate A is served, and the session stays usable ---------------
-    let session_a = {
-        let addr = addr.clone();
-        tokio::task::spawn_blocking(move || tls_connect(&addr)).await??
-    };
-    let fingerprint_a = session_a.fingerprint.clone();
-    let session_a = roundtrip_and_return(session_a).await?;
-
-    // --- swap to certificate B: new connections pick it up without restart ---
-    write_cert_pair(&tls_dir, vec!["localhost".into(), "127.0.0.1".into()]).await?;
-    let reload_budget = Duration::from_secs(RELOAD_INTERVAL_SECS * 4 + 10);
-    let fingerprint_b = wait_for_new_fingerprint(&addr, &fingerprint_a, reload_budget).await?;
-    assert_ne!(fingerprint_a, fingerprint_b);
-
-    // The connection opened under certificate A must survive the rotation.
-    let _session_a = roundtrip_and_return(session_a).await?;
-
-    // --- garbage material: fail-safe, keep serving B, log the failure --------
-    tokio::fs::write(tls_dir.join(CERT_FILE), b"not a certificate").await?;
-    tokio::fs::write(tls_dir.join(KEY_FILE), b"not a key").await?;
-
-    // Give the reload loop at least two ticks to observe the bad material.
-    sleep(Duration::from_secs(RELOAD_INTERVAL_SECS * 2 + 2)).await;
-
-    let after_bad = {
-        let addr = addr.clone();
-        tokio::task::spawn_blocking(move || tls_connect(&addr)).await??
-    };
-    assert_eq!(
-        after_bad.fingerprint, fingerprint_b,
-        "bad on-disk material must not change or drop the served certificate"
-    );
-
-    let log = tokio::fs::read_to_string(&log_path).await.unwrap_or_default();
-    assert!(
-        log.contains("tls_reload_failed") || log.contains("TLS reload failed"),
-        "a failed reload must leave an assertable log event; captured log did not contain one"
-    );
-
-    // The process must still be alive and serving.
-    let final_session = tokio::task::spawn_blocking(move || tls_connect(&addr)).await??;
-    let _ = roundtrip_and_return(final_session).await?;
-
-    env.stop_server();
-    Ok(())
-}
@@ -1023,49 +1023,20 @@ fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
    http_client_fn(move |_settings, _components| connector.clone())
 }

-fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
-    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
+fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
+    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem.as_bytes())
        .collect::<Result<Vec<_>, _>>()
-        .map_err(|err| format!("invalid PEM encoding: {err}"))?;
+        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))?;

    if certs.is_empty() {
-        return Err("no certificates found".to_string());
+        return Err(BucketTargetError::Io(std::io::Error::other(
+            "invalid target CA PEM: no certificates found",
+        )));
    }

-    // Smithy's rustls adapter defers parsing custom certificates and assumes
-    // they are valid when the HTTPS connector is built. Validate every DER
-    // certificate first so malformed configuration is reported rather than
-    // reaching an `expect` in the dependency.
-    let mut validation_store = rustls::RootCertStore::empty();
-    for cert in certs {
-        validation_store
-            .add(cert)
-            .map_err(|err| format!("invalid X.509 certificate: {err}"))?;
-    }
+    let mut trust_store = smithy_tls::TrustStore::empty();
+    trust_store.add_pem_certificate(ca_cert_pem.as_bytes());

-    Ok(())
-}
-
-fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), BucketTargetError> {
-    validate_ca_pem_bundle(ca_cert_pem.as_bytes())
-        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))
-}
-
-fn compose_replication_trust_store(certificate_bundles: impl IntoIterator<Item = Vec<u8>>) -> (smithy_tls::TrustStore, usize) {
-    // `TrustStore::default()` keeps the platform-native roots enabled. Target
-    // and RUSTFS_TLS_PATH certificates extend that baseline instead of
-    // replacing it with a target-specific trust island.
-    let mut trust_store = smithy_tls::TrustStore::default();
-    let mut custom_bundle_count = 0;
-    for pem in certificate_bundles {
-        trust_store.add_pem_certificate(pem);
-        custom_bundle_count += 1;
-    }
-
-    (trust_store, custom_bundle_count)
-}
-
-fn build_aws_s3_http_client_with_trust_store(trust_store: smithy_tls::TrustStore) -> Result<SharedHttpClient, BucketTargetError> {
    let tls_context = smithy_tls::TlsContext::builder()
        .with_trust_store(trust_store)
        .build()
@@ -1077,60 +1048,6 @@ fn build_aws_s3_http_client_with_trust_store(trust_store: smithy_tls::TrustStore
        .build_https())
 }

-async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
-    let mut certificate_bundles = Vec::new();
-
-    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
-    match tokio::fs::read(&ca_path).await {
-        Ok(pem) => match validate_ca_pem_bundle(&pem) {
-            Ok(()) => certificate_bundles.push(pem),
-            Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
-        },
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
-    }
-
-    if trust_leaf_cert_as_ca {
-        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
-        match tokio::fs::read(&leaf_cert_path).await {
-            Ok(pem) => match validate_ca_pem_bundle(&pem) {
-                Ok(()) => certificate_bundles.push(pem),
-                Err(err) => warn!(
-                    "ignoring invalid leaf certificate {:?} for replication client trust store: {}",
-                    leaf_cert_path, err
-                ),
-            },
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
-        }
-    }
-
-    certificate_bundles
-}
-
-async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
-    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
-    if tls_path.is_empty() {
-        return Vec::new();
-    }
-
-    load_tls_path_ca_bundles(
-        Path::new(&tls_path),
-        rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
-    )
-    .await
-}
-
-async fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
-    validate_target_ca_pem(ca_cert_pem)?;
-
-    let mut certificate_bundles = load_configured_tls_ca_bundles().await;
-    certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-
-    build_aws_s3_http_client_with_trust_store(trust_store)
-}
-
 async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Option<SharedHttpClient>, BucketTargetError> {
    if !target.secure {
        return Ok(None);
@@ -1141,28 +1058,62 @@ async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Op
    }

    if has_custom_ca_pem(target) {
-        return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem)
-            .await
-            .map(Some);
+        return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem).map(Some);
    }

    Ok(build_aws_s3_http_client_from_tls_path().await)
 }

 async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
-    let certificate_bundles = load_configured_tls_ca_bundles().await;
-    if certificate_bundles.is_empty() {
+    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
+    if tls_path.is_empty() {
        return None;
    }

-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-    match build_aws_s3_http_client_with_trust_store(trust_store) {
-        Ok(client) => Some(client),
-        Err(e) => {
-            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
-            None
+    let tls_dir = Path::new(&tls_path);
+    let mut trust_store = smithy_tls::TrustStore::empty();
+    let mut has_custom_certs = false;
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => {
+            trust_store.add_pem_certificate(pem);
+            has_custom_certs = true;
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => {
+                trust_store.add_pem_certificate(pem);
+                has_custom_certs = true;
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
        }
    }
+
+    if !has_custom_certs {
+        return None;
+    }
+
+    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
+        Ok(ctx) => ctx,
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            return None;
+        }
+    };
+
+    Some(
+        SmithyHttpClientBuilder::new()
+            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+            .tls_context(tls_context)
+            .build_https(),
+    )
 }

 fn should_force_path_style(target: &BucketTarget) -> bool {
@@ -1969,63 +1920,6 @@ mod tests {
    use super::*;
    use rcgen::generate_simple_self_signed;

-    fn spawn_single_request_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>) -> (u16, std::thread::JoinHandle<()>) {
-        use std::io::{Read, Write};
-
-        ensure_rustls_crypto_provider();
-        let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test TLS listener should bind");
-        let port = listener
-            .local_addr()
-            .expect("test TLS listener should have an address")
-            .port();
-        let server_config = rustls::ServerConfig::builder()
-            .with_no_client_auth()
-            .with_single_cert(
-                vec![cert.cert.der().clone()],
-                rustls_pki_types::PrivateKeyDer::try_from(cert.signing_key.serialize_der())
-                    .expect("test TLS private key should convert"),
-            )
-            .expect("test TLS server config should build");
-
-        let handle = std::thread::spawn(move || {
-            let (stream, _) = listener.accept().expect("test TLS client should connect");
-            stream
-                .set_read_timeout(Some(Duration::from_secs(10)))
-                .expect("test TLS read timeout should configure");
-            stream
-                .set_write_timeout(Some(Duration::from_secs(10)))
-                .expect("test TLS write timeout should configure");
-            let connection = rustls::ServerConnection::new(Arc::new(server_config)).expect("test TLS connection should build");
-            let mut stream = rustls::StreamOwned::new(connection, stream);
-            let mut request = [0_u8; 8192];
-            let _ = stream.read(&mut request).expect("test TLS request should be readable");
-            stream
-                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
-                .expect("test TLS response should be written");
-            stream.flush().expect("test TLS response should flush");
-        });
-
-        (port, handle)
-    }
-
-    fn s3_client_with_http_client(port: u16, http_client: SharedHttpClient) -> S3Client {
-        let credentials = SdkCredentials::builder()
-            .access_key_id("test-access")
-            .secret_access_key("test-secret")
-            .provider_name("bucket_target_tls_test")
-            .build();
-        let config = S3Config::builder()
-            .endpoint_url(format!("https://localhost:{port}"))
-            .credentials_provider(SharedCredentialsProvider::new(credentials))
-            .region(SdkRegion::new("us-east-1"))
-            .force_path_style(true)
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-            .http_client(http_client)
-            .build();
-
-        S3Client::from_conf(config)
-    }
-
    #[test]
    fn replication_target_versioning_enabled_requires_enabled_status() {
        let enabled = BucketVersioningStatus::Enabled;
@@ -2349,78 +2243,6 @@ mod tests {
        assert_eq!(client.endpoint, "https://192.168.1.10:9000");
    }

-    #[tokio::test]
-    async fn replication_trust_store_composes_system_global_and_target_roots_for_real_tls() {
-        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
-        let global_ca =
-            generate_simple_self_signed(vec!["localhost".to_string()]).expect("global CA certificate should generate");
-        let target_ca =
-            generate_simple_self_signed(vec!["localhost".to_string()]).expect("target CA certificate should generate");
-
-        tokio::fs::write(tls_dir.path().join(RUSTFS_CA_CERT), global_ca.cert.pem())
-            .await
-            .expect("global CA bundle should be written");
-
-        let mut certificate_bundles = load_tls_path_ca_bundles(tls_dir.path(), false).await;
-        certificate_bundles.push(target_ca.cert.pem().into_bytes());
-        let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-        assert!(
-            format!("{trust_store:?}").contains("enable_native_roots: true"),
-            "per-target trust must retain the SDK's platform-native roots"
-        );
-        let http_client = build_aws_s3_http_client_with_trust_store(trust_store).expect("composed TLS client should build");
-
-        let (global_port, global_server) = spawn_single_request_https_server(&global_ca);
-        s3_client_with_http_client(global_port, http_client.clone())
-            .head_bucket()
-            .bucket("test-bucket")
-            .send()
-            .await
-            .expect("global RUSTFS_TLS_PATH CA should authenticate its TLS server");
-        global_server.join().expect("global CA TLS server should finish");
-
-        let (target_port, target_server) = spawn_single_request_https_server(&target_ca);
-        s3_client_with_http_client(target_port, http_client)
-            .head_bucket()
-            .bucket("test-bucket")
-            .send()
-            .await
-            .expect("per-target CA should authenticate its TLS server alongside global roots");
-        target_server.join().expect("target CA TLS server should finish");
-    }
-
-    #[tokio::test]
-    async fn tls_path_leaf_trust_remains_opt_in() {
-        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
-        let global_ca =
-            generate_simple_self_signed(vec!["global-ca.example".to_string()]).expect("global CA certificate should generate");
-        let trusted_leaf =
-            generate_simple_self_signed(vec!["leaf.example".to_string()]).expect("trusted leaf certificate should generate");
-        tokio::fs::write(tls_dir.path().join(RUSTFS_CA_CERT), global_ca.cert.pem())
-            .await
-            .expect("global CA bundle should be written");
-        tokio::fs::write(tls_dir.path().join(RUSTFS_TLS_CERT), trusted_leaf.cert.pem())
-            .await
-            .expect("trusted leaf certificate should be written");
-
-        assert_eq!(load_tls_path_ca_bundles(tls_dir.path(), true).await.len(), 2);
-        assert_eq!(load_tls_path_ca_bundles(tls_dir.path(), false).await.len(), 1);
-    }
-
-    #[tokio::test]
-    async fn skip_tls_verify_takes_priority_over_invalid_custom_ca_pem() {
-        let client = build_aws_s3_http_client_for_target(&BucketTarget {
-            secure: true,
-            skip_tls_verify: true,
-            ca_cert_pem: "not a pem".to_string(),
-            ..Default::default()
-        })
-        .await
-        .expect("skip verification should bypass custom CA parsing");
-
-        assert!(client.is_some(), "secure targets with skip verification need a custom HTTP client");
-    }
-
    #[tokio::test]
    async fn get_remote_target_client_internal_rejects_invalid_custom_ca_pem() {
        let sys = BucketTargetSys::default();
@@ -2445,31 +2267,6 @@ mod tests {
        assert!(err.to_string().contains("invalid target CA PEM"));
    }

-    #[test]
-    fn target_ca_rejects_pem_wrapped_invalid_der_before_smithy_builds() {
-        let err = validate_target_ca_pem("-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n")
-            .expect_err("PEM-wrapped invalid DER must be rejected");
-
-        assert!(err.to_string().contains("invalid target CA PEM"));
-        assert!(err.to_string().contains("invalid X.509 certificate"));
-    }
-
-    #[tokio::test]
-    async fn invalid_global_ca_is_ignored_without_reaching_smithy() {
-        let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
-        tokio::fs::write(
-            tls_dir.path().join(RUSTFS_CA_CERT),
-            b"-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n",
-        )
-        .await
-        .expect("invalid global CA fixture should be written");
-
-        assert!(
-            load_tls_path_ca_bundles(tls_dir.path(), false).await.is_empty(),
-            "invalid global CA must fall back to default roots instead of reaching Smithy's panic path"
-        );
-    }
-
    // backlog#806-16 regression tests for the rolling one-minute latency window.

    #[test]
@@ -836,12 +836,6 @@ pub struct TransitionState {
    compensation_scheduled_tasks: AtomicI64,
    compensation_running_tasks: AtomicI64,
    compensation_buckets: Arc<Mutex<HashSet<String>>>,
-    // (bucket, object, version) currently queued or being transitioned. A single
-    // PUT can enqueue the same object twice (immediate transition + startup
-    // compensation backfill); without this guard the duplicates run concurrently,
-    // and the loser races the winner's source cleanup — reading data the winner
-    // already removed and logging a spurious NotFound failure (rustfs/backlog#1268).
-    in_flight_transitions: Arc<Mutex<HashSet<(String, String, String)>>>,
    last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
 }

@@ -875,39 +869,10 @@ impl TransitionState {
            compensation_scheduled_tasks: AtomicI64::new(0),
            compensation_running_tasks: AtomicI64::new(0),
            compensation_buckets: Arc::new(Mutex::new(HashSet::new())),
-            in_flight_transitions: Arc::new(Mutex::new(HashSet::new())),
            last_day_stats: Arc::new(Mutex::new(HashMap::new())),
        })
    }

-    fn transition_key(oi: &ObjectInfo) -> (String, String, String) {
-        (
-            oi.bucket.clone(),
-            oi.name.clone(),
-            oi.version_id.map(|v| v.to_string()).unwrap_or_default(),
-        )
-    }
-
-    /// Try to claim a transition for this exact (bucket, object, version). Returns
-    /// `false` when one is already queued or running, so the caller can skip the
-    /// duplicate enqueue (rustfs/backlog#1268). The claim is released in the
-    /// worker once the transition finishes, or here if the enqueue fails.
-    fn reserve_transition(&self, oi: &ObjectInfo) -> bool {
-        let key = Self::transition_key(oi);
-        match self.in_flight_transitions.lock() {
-            Ok(mut set) => set.insert(key),
-            Err(poisoned) => poisoned.into_inner().insert(key),
-        }
-    }
-
-    fn release_transition(&self, oi: &ObjectInfo) {
-        let key = Self::transition_key(oi);
-        match self.in_flight_transitions.lock() {
-            Ok(mut set) => set.remove(&key),
-            Err(poisoned) => poisoned.into_inner().remove(&key),
-        };
-    }
-
    fn reserve_bucket_compensation(&self, bucket: &str) -> bool {
        let inserted = match self.compensation_buckets.lock() {
            Ok(mut scheduled) => scheduled.insert(bucket.to_string()),
@@ -1081,15 +1046,6 @@ impl TransitionState {
            return false;
        }

-        // Deduplicate concurrent enqueues of the same object version. The claim is
-        // released by the worker after the transition finishes (rustfs/backlog#1268).
-        // This is a no-op, not a new enqueue, so it must not touch the enqueue
-        // counters (the first enqueue already counted this object).
-        if !self.reserve_transition(oi) {
-            self.record_scanner_transition_state();
-            return true;
-        }
-
        let task = TransitionTask {
            obj_info: oi.clone(),
            src: src.clone(),
@@ -1128,9 +1084,6 @@ impl TransitionState {
                    self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::QueueClosed { timeout_ms: None });
                }
            }
-            if !queued {
-                self.release_transition(oi);
-            }
            record_scanner_transition_enqueue_result(src, 1, queued);
            self.record_scanner_transition_state();
            return queued;
@@ -1160,9 +1113,6 @@ impl TransitionState {
                );
            }
        }
-        if !queued {
-            self.release_transition(oi);
-        }
        record_scanner_transition_enqueue_result(src, 1, queued);
        self.record_scanner_transition_state();
        queued
@@ -1289,9 +1239,6 @@ impl TransitionState {

                            emit_transition_complete_event(obj_info_for_event);
                        }
-                        // Release the dedup claim so a later lifecycle pass can
-                        // re-transition this object if needed (rustfs/backlog#1268).
-                        transition_state.release_transition(&task.obj_info);
                        TransitionState::add_counter(&transition_state.active_tasks, -1);
                        transition_state.record_scanner_transition_state();
                    }
@@ -3297,16 +3244,8 @@ mod tests {
            ..Default::default()
        };

-        // A distinct object fills past the capacity-1 queue and is reported as a
-        // missed enqueue. (Re-enqueuing the same object would instead be deduped;
-        // see transition_reserve_dedupes_same_object_version.)
-        let other = ObjectInfo {
-            bucket: "bucket".to_string(),
-            name: "object-2".to_string(),
-            ..Default::default()
-        };
        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-        let second = state.queue_transition_task(&other, &event, &LcEventSrc::Scanner).await;
+        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;

        assert!(first);
        assert!(!second);
@@ -3328,13 +3267,8 @@ mod tests {
            ..Default::default()
        };

-        let other = ObjectInfo {
-            bucket: "bucket".to_string(),
-            name: "object-2".to_string(),
-            ..Default::default()
-        };
        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-        let second = state.queue_transition_task(&other, &event, &LcEventSrc::Scanner).await;
+        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;

        assert!(first);
        assert!(!second);
@@ -3835,62 +3769,6 @@ mod tests {
        assert_eq!(state.scanner_transition_state_update().compensation_pending, 1);
    }

-    #[test]
-    fn transition_reserve_dedupes_same_object_version() {
-        // Regression for rustfs/backlog#1268: a single object version can be
-        // enqueued twice (immediate transition + compensation backfill). The
-        // second claim must be rejected until the first is released, so the two
-        // do not run concurrently and race the source cleanup.
-        let state = TransitionState::new_with_capacity(4);
-
-        let oi = ObjectInfo {
-            bucket: "foo".to_string(),
-            name: "payload.bin".to_string(),
-            version_id: None,
-            ..Default::default()
-        };
-
-        assert!(state.reserve_transition(&oi), "first claim must succeed");
-        assert!(!state.reserve_transition(&oi), "duplicate claim must be rejected");
-
-        // A different object is independent.
-        let other = ObjectInfo {
-            bucket: "foo".to_string(),
-            name: "other.bin".to_string(),
-            version_id: None,
-            ..Default::default()
-        };
-        assert!(state.reserve_transition(&other), "distinct object must claim independently");
-
-        // After release, the same object can be re-claimed (later lifecycle pass).
-        state.release_transition(&oi);
-        assert!(state.reserve_transition(&oi), "re-claim after release must succeed");
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn queue_transition_task_dedupes_same_object_without_second_enqueue() {
-        // Capacity 4 leaves room, so a rejected second enqueue can only be the
-        // dedup guard, not queue pressure (rustfs/backlog#1268).
-        let state = TransitionState::new_with_capacity(4);
-        let object = ObjectInfo {
-            bucket: "bucket".to_string(),
-            name: "object".to_string(),
-            ..Default::default()
-        };
-        let event = crate::bucket::lifecycle::lifecycle::Event {
-            action: IlmAction::TransitionAction,
-            ..Default::default()
-        };
-
-        let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-        let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
-
-        assert!(first, "first enqueue must succeed");
-        assert!(second, "duplicate enqueue is reported handled (already queued)");
-        assert_eq!(state.transition_rx.len(), 1, "only one task must actually be queued");
-    }
-
    #[tokio::test]
    #[serial]
    async fn transition_state_init_honors_runtime_configured_worker_count() {
@@ -585,43 +585,4 @@ mod tests {

        assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
    }
-
-    // T3 (#1264): the outbound replication path forwards a stored object checksum into
-    // user_metadata via decrypt_checksums, which is algorithm-agnostic. This locks that
-    // the AWS 2026-04 additional algorithms (XXHash3/64/128, SHA-512, MD5) are forwarded
-    // identically to the classic five — i.e. replication treats the new algorithms
-    // consistently, with no new-algorithm-specific gap on the outbound side.
-    #[test]
-    fn replication_put_object_options_forwards_new_algorithm_checksums_like_classic() {
-        use rustfs_rio::{Checksum, ChecksumType};
-
-        let payload = b"replication checksum consistency payload";
-        let cases = [
-            // classic five (baseline)
-            ("CRC32", ChecksumType::CRC32),
-            ("SHA256", ChecksumType::SHA256),
-            // AWS 2026-04 additional algorithms
-            ("XXHASH3", ChecksumType::XXHASH3),
-            ("XXHASH64", ChecksumType::XXHASH64),
-            ("XXHASH128", ChecksumType::XXHASH128),
-            ("SHA512", ChecksumType::SHA512),
-            ("MD5", ChecksumType::MD5),
-        ];
-
-        for (name, ty) in cases {
-            let checksum = Checksum::new_from_data(ty, payload).expect("compute checksum");
-            let object_info = ObjectInfo {
-                checksum: Some(checksum.to_bytes(&[])),
-                ..Default::default()
-            };
-
-            let (opts, _is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
-
-            assert_eq!(
-                opts.user_metadata.get(name),
-                Some(&checksum.encoded),
-                "replication must forward the {name} checksum into user_metadata identically to the classic algorithms"
-            );
-        }
-    }
 }
@@ -120,11 +120,7 @@ impl Default for PutObjectOptions {
            storage_class: "".to_string(),
            website_redirect_location: "".to_string(),
            part_size: 0,
-            // Empty, not OFF: `header()` emits x-amz-object-lock-legal-hold for
-            // any non-empty status, and CompleteMultipartUpload rejects requests
-            // that carry object-lock headers, breaking multipart transitions
-            // (rustfs/rustfs#4811). Only send the header when a status is set.
-            legalhold: ObjectLockLegalHoldStatus::from_static(""),
+            legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
            send_content_md5: false,
            disable_content_sha256: false,
            disable_multipart: false,
@@ -18,7 +18,6 @@
 #![allow(clippy::all)]

 use http::{HeaderMap, HeaderName, StatusCode};
-use http_body_util::BodyExt;
 use hyper::body::Bytes;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
@@ -245,23 +244,7 @@ impl TransitionClient {
            )));
        }
        //}
-        // Parse the CreateMultipartUpload response for the UploadId. Returning a
-        // default (empty) result here made every multipart transition fail at the
-        // first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
-        let mut body_vec = Vec::new();
-        let mut body = resp.into_body();
-        while let Some(frame) = body.frame().await {
-            let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
-            if let Some(data) = frame.data_ref() {
-                body_vec.extend_from_slice(data);
-            }
-        }
-        let initiate_multipart_upload_result =
-            quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
-                .map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
-        if initiate_multipart_upload_result.upload_id.is_empty() {
-            return Err(std::io::Error::other("CreateMultipartUpload response missing UploadId"));
-        }
+        let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
        Ok(initiate_multipart_upload_result)
    }

@@ -449,56 +432,3 @@ pub struct UploadPartParams {
    pub custom_header: HeaderMap,
    pub trailer: HeaderMap,
 }
-
-#[cfg(test)]
-mod tests {
-    use crate::client::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
-
-    #[test]
-    fn complete_multipart_upload_serializes_s3_part_elements() {
-        // Regression for rustfs/rustfs#4811: without serde renames quick-xml emits
-        // <parts>/<part_num>/<etag>, so the remote parses zero <Part> elements and
-        // completes a 0-byte object (while still returning 200). The body must use
-        // S3 element names, and MD5-only transitions must not emit empty checksum
-        // elements.
-        let complete = CompleteMultipartUpload {
-            parts: vec![
-                CompletePart {
-                    part_num: 1,
-                    etag: "etag-one".to_string(),
-                    ..Default::default()
-                },
-                CompletePart {
-                    part_num: 2,
-                    etag: "etag-two".to_string(),
-                    ..Default::default()
-                },
-            ],
-        };
-        let xml = complete.marshal_msg().expect("marshal");
-        assert!(xml.contains("<Part>"), "missing <Part>: {xml}");
-        assert!(xml.contains("<PartNumber>1</PartNumber>"), "missing PartNumber: {xml}");
-        assert!(xml.contains("<ETag>etag-one</ETag>"), "missing ETag: {xml}");
-        assert!(xml.contains("<PartNumber>2</PartNumber>"), "missing part 2: {xml}");
-        assert!(!xml.contains("part_num"), "leaked rust field name: {xml}");
-        assert!(!xml.contains("<ChecksumCRC32>"), "emitted empty checksum: {xml}");
-    }
-
-    #[test]
-    fn parses_create_multipart_upload_response() {
-        // Regression for rustfs/rustfs#4811: `initiate_multipart_upload` used to
-        // return a default (empty) result, so every multipart transition failed
-        // the first UploadPart with "UploadID cannot be empty". The UploadId must
-        // be parsed out of the S3 XML response.
-        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
-<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-  <Bucket>bar</Bucket>
-  <Key>foo/payload.bin</Key>
-  <UploadId>a1b2c3-d4e5-f6</UploadId>
-</InitiateMultipartUploadResult>"#;
-        let parsed: InitiateMultipartUploadResult = quick_xml::de::from_str(xml).expect("parse");
-        assert_eq!(parsed.upload_id, "a1b2c3-d4e5-f6");
-        assert_eq!(parsed.bucket, "bar");
-        assert_eq!(parsed.key, "foo/payload.bin");
-    }
-}
@@ -25,7 +25,6 @@ use std::io::Error;
 use std::sync::RwLock;
 use std::{collections::HashMap, sync::Arc};
 use time::{OffsetDateTime, format_description};
-use tokio::io::AsyncReadExt;
 use tokio::{select, sync::mpsc};
 use tokio_util::sync::CancellationToken;
 use tracing::warn;
@@ -46,33 +45,6 @@ use crate::client::utils::base64_encode;
 use rustfs_utils::path::trim_etag;
 use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};

-/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
-/// reaches EOF first. Advances the reader so the next call returns the following
-/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
-/// the entire source into the first part and left later parts empty
-/// (rustfs/rustfs#4811).
-async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec<u8>, std::io::Error> {
-    match reader {
-        ReaderImpl::Body(content_body) => {
-            let take = content_body.len().min(want);
-            Ok(content_body.split_to(take).to_vec())
-        }
-        ReaderImpl::ObjectBody(content_body) => {
-            let mut buf = vec![0u8; want];
-            let mut filled = 0;
-            while filled < want {
-                let n = content_body.read(&mut buf[filled..]).await?;
-                if n == 0 {
-                    break;
-                }
-                filled += n;
-            }
-            buf.truncate(filled);
-            Ok(buf)
-        }
-    }
-}
-
 pub struct UploadedPartRes {
    pub error: std::io::Error,
    pub part_num: i64,
@@ -161,6 +133,7 @@ impl TransitionClient {
        let mut total_uploaded_size: i64 = 0;

        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);

        let mut md5_base64: String = "".to_string();
        for part_number in 1..=total_parts_count {
@@ -168,12 +141,14 @@ impl TransitionClient {
                part_size = lastpart_size;
            }

-            // Read exactly this part's bytes. Using `read_all()`/`to_vec()` here
-            // drained the whole source into the first part and left every later
-            // part empty, silently corrupting any multipart upload of a streamed
-            // (`ObjectBody`) source — e.g. ILM transitions of >128 MiB objects,
-            // which split into 128 MiB parts (rustfs/rustfs#4811).
-            let buf = read_multipart_part(&mut reader, part_size as usize).await?;
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
            let length = buf.len();

            if opts.send_content_md5 {
@@ -184,7 +159,7 @@ impl TransitionClient {
                };
                let hash = md5_hash.hash_encode(&buf[..length]);
                md5_base64 = base64_encode(hash.as_ref());
-            } else if opts.auto_checksum.is_set() {
+            } else {
                let mut crc = opts.auto_checksum.hasher()?;
                crc.update(&buf[..length]);
                let csum = crc.finalize();
@@ -199,10 +174,6 @@ impl TransitionClient {
                    warn!("Invalid header name: {}", opts.auto_checksum.key());
                }
            }
-            // else: neither MD5 nor a concrete additional checksum was requested,
-            // so upload the part without a per-part checksum header. Guarding the
-            // branch on `is_set()` avoids calling `hasher()` on `ChecksumNone`,
-            // which errors with "unsupported checksum type" (rustfs/rustfs#4811).

            let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
            let mut p = UploadPartParams {
@@ -212,9 +183,7 @@ impl TransitionClient {
                reader: hooked,
                part_number,
                md5_base64: md5_base64.clone(),
-                // Use the bytes actually read, not the planned part_size, so the
-                // uploaded Content-Length matches the body even on a short read.
-                size: length as i64,
+                size: part_size,
                //sse: opts.server_side_encryption,
                stream_sha256: !opts.disable_content_sha256,
                custom_header: custom_header.clone(),
@@ -225,7 +194,7 @@ impl TransitionClient {

            parts_info.entry(part_number).or_insert(obj_part);

-            total_uploaded_size += length as i64;
+            total_uploaded_size += part_size as i64;
        }

        if size > 0 && total_uploaded_size != size {
@@ -624,79 +593,9 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou

 #[cfg(test)]
 mod tests {
-    use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
-    use crate::object_api::GetObjectReader;
-    use bytes::Bytes;
+    use super::{ObjectPart, collect_complete_parts};
    use std::collections::HashMap;

-    // Drive a reader through the same per-part loop the multipart stream uses and
-    // collect the size of every part. Regression for rustfs/rustfs#4811: the old
-    // `read_all()` per part drained the whole source into part 1.
-    async fn collect_part_sizes(mut reader: ReaderImpl, total: usize, part_size: usize, last_part_size: usize) -> Vec<usize> {
-        let parts = total.div_ceil(part_size);
-        let mut sizes = Vec::new();
-        for part_number in 1..=parts {
-            let want = if part_number == parts { last_part_size } else { part_size };
-            let buf = read_multipart_part(&mut reader, want).await.unwrap();
-            sizes.push(buf.len());
-        }
-        // Nothing must remain after the planned parts are consumed.
-        assert!(read_multipart_part(&mut reader, part_size).await.unwrap().is_empty());
-        sizes
-    }
-
-    #[tokio::test]
-    async fn read_multipart_part_splits_streamed_object_body_evenly() {
-        // 250 bytes at part_size 100 -> parts [100, 100, 50], mirroring the
-        // >128 MiB / 128 MiB split from the bug report on a small deterministic
-        // stream.
-        let total = 250usize;
-        let (mut w, r) = tokio::io::duplex(64);
-        tokio::spawn(async move {
-            use tokio::io::AsyncWriteExt;
-            let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
-            w.write_all(&data).await.unwrap();
-        });
-        let reader = ReaderImpl::ObjectBody(GetObjectReader {
-            stream: Box::new(r),
-            object_info: Default::default(),
-            buffered_body: None,
-            body_source: Default::default(),
-        });
-
-        let sizes = collect_part_sizes(reader, total, 100, 50).await;
-        assert_eq!(sizes, vec![100, 100, 50]);
-    }
-
-    #[tokio::test]
-    async fn read_multipart_part_splits_in_memory_body_evenly() {
-        let total = 250usize;
-        let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
-        let reader = ReaderImpl::Body(Bytes::from(data));
-
-        let sizes = collect_part_sizes(reader, total, 100, 50).await;
-        assert_eq!(sizes, vec![100, 100, 50]);
-    }
-
-    #[tokio::test]
-    async fn read_multipart_part_stops_at_eof_without_overrun() {
-        // Reader shorter than the requested part size must return only what is
-        // available, not block or pad.
-        let (mut w, r) = tokio::io::duplex(64);
-        tokio::spawn(async move {
-            use tokio::io::AsyncWriteExt;
-            w.write_all(&[1u8; 30]).await.unwrap();
-        });
-        let mut reader = ReaderImpl::ObjectBody(GetObjectReader {
-            stream: Box::new(r),
-            object_info: Default::default(),
-            buffered_body: None,
-            body_source: Default::default(),
-        });
-        let buf = read_multipart_part(&mut reader, 100).await.unwrap();
-        assert_eq!(buf.len(), 30);
-    }
-
    fn parts_map(n: i64) -> HashMap<i64, ObjectPart> {
        let mut m = HashMap::new();
        for i in 1..=n {
@@ -209,8 +209,7 @@ pub struct ListObjectPartsResult {
    pub encoding_type: String,
 }

-#[derive(Debug, Default, Serialize, Deserialize)]
-#[serde(default, rename_all = "PascalCase")]
+#[derive(Debug, Default)]
 pub struct InitiateMultipartUploadResult {
    pub bucket: String,
    pub key: String,
@@ -230,28 +229,15 @@ pub struct CompleteMultipartUploadResult {
    pub checksum_crc64nvme: String,
 }

-// Field renames drive the CompleteMultipartUpload XML body sent to the remote.
-// Without them quick-xml emits the Rust field names (<part_num>/<etag>), so the
-// remote parses zero <Part> elements and completes a 0-byte object while still
-// returning 200 — the multipart transition silently produces an empty object
-// (rustfs/rustfs#4811). Empty checksum fields are skipped so an MD5-only
-// transition does not emit blank <ChecksumCRC32/> elements.
 #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
 pub struct CompletePart {
    //api has
-    #[serde(rename = "PartNumber")]
-    pub part_num: i64,
-    #[serde(rename = "ETag")]
    pub etag: String,
-    #[serde(rename = "ChecksumCRC32", skip_serializing_if = "String::is_empty")]
+    pub part_num: i64,
    pub checksum_crc32: String,
-    #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "String::is_empty")]
    pub checksum_crc32c: String,
-    #[serde(rename = "ChecksumSHA1", skip_serializing_if = "String::is_empty")]
    pub checksum_sha1: String,
-    #[serde(rename = "ChecksumSHA256", skip_serializing_if = "String::is_empty")]
    pub checksum_sha256: String,
-    #[serde(rename = "ChecksumCRC64NVME", skip_serializing_if = "String::is_empty")]
    pub checksum_crc64nvme: String,
 }

@@ -286,9 +272,7 @@ pub struct CopyObjectPartResult {
 }

 #[derive(Debug, Default, serde::Serialize)]
-#[serde(rename = "CompleteMultipartUpload")]
 pub struct CompleteMultipartUpload {
-    #[serde(rename = "Part")]
    pub parts: Vec<CompletePart>,
 }

@@ -71,11 +71,7 @@ impl ChecksumMode {
            8_u8 => ChecksumMode::ChecksumCRC32,
            16_u8 => ChecksumMode::ChecksumCRC32C,
            32_u8 => ChecksumMode::ChecksumCRC64NVME,
-            // Fail closed: any mode without a concrete base algorithm (e.g. a
-            // bare ChecksumFullObject flag) is treated as "no checksum" rather
-            // than panicking. Callers already gate real work behind
-            // is_set()/can_composite()/hasher(), so this only removes a crash.
-            _ => ChecksumMode::ChecksumNone,
+            _ => panic!("enum err."),
        }
    }

@@ -181,17 +177,6 @@ impl ChecksumMode {
    }

    pub fn is_set(&self) -> bool {
-        // `ChecksumNone` is the zeroth enum variant, so it occupies bit 0 of the
-        // `EnumSet` repr and a naive `len() == 1` check reports "no checksum" as a
-        // configured checksum. A checksum is only "set" when a concrete algorithm
-        // (one with a real hasher) is selected; the bare `ChecksumFullObject` flag
-        // has no base algorithm and is likewise not set. Treating `ChecksumNone`
-        // as set made ILM transitions of >128 MiB objects fail with
-        // "unsupported checksum type" (rustfs/rustfs#4811): the multipart put path
-        // took the checksum branch and called `ChecksumNone.hasher()`.
-        if matches!(self, ChecksumMode::ChecksumNone) {
-            return false;
-        }
        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
        s.len() == 1
    }
@@ -287,70 +272,6 @@ impl ChecksumMode {
    }
 }

-#[cfg(test)]
-mod tests {
-    use super::ChecksumMode;
-
-    #[test]
-    fn test_base_is_fail_closed_and_never_panics() {
-        // Every mode must resolve to a concrete base without panicking. The bare
-        // ChecksumFullObject flag has no base algorithm and must fall back to
-        // ChecksumNone instead of crashing (previously `panic!("enum err.")`).
-        assert_eq!(ChecksumMode::ChecksumFullObject.base(), ChecksumMode::ChecksumNone);
-        assert_eq!(ChecksumMode::ChecksumNone.base(), ChecksumMode::ChecksumNone);
-        assert_eq!(ChecksumMode::ChecksumCRC32.base(), ChecksumMode::ChecksumCRC32);
-        assert_eq!(ChecksumMode::ChecksumCRC32C.base(), ChecksumMode::ChecksumCRC32C);
-        assert_eq!(ChecksumMode::ChecksumSHA1.base(), ChecksumMode::ChecksumSHA1);
-        assert_eq!(ChecksumMode::ChecksumSHA256.base(), ChecksumMode::ChecksumSHA256);
-        assert_eq!(ChecksumMode::ChecksumCRC64NVME.base(), ChecksumMode::ChecksumCRC64NVME);
-    }
-
-    #[test]
-    fn test_hasher_fails_closed_for_unsupported_mode() {
-        // Modes without a real hasher must return an error, not panic.
-        assert!(ChecksumMode::ChecksumNone.hasher().is_err());
-        assert!(ChecksumMode::ChecksumFullObject.hasher().is_err());
-        assert!(ChecksumMode::ChecksumCRC32.hasher().is_ok());
-    }
-
-    #[test]
-    fn test_is_set_is_false_for_none_and_bare_full_object() {
-        // Regression for rustfs/rustfs#4811: `ChecksumNone` must NOT be reported as
-        // a configured checksum. It is the zeroth enum variant (bit 0 of the
-        // EnumSet repr), so the old `len() == 1` check treated it as set and drove
-        // the multipart put path into `ChecksumNone.hasher()` → "unsupported
-        // checksum type". Every mode reported as set must also have a real hasher.
-        assert!(!ChecksumMode::ChecksumNone.is_set());
-        assert!(!ChecksumMode::ChecksumFullObject.is_set());
-
-        for mode in [
-            ChecksumMode::ChecksumCRC32,
-            ChecksumMode::ChecksumCRC32C,
-            ChecksumMode::ChecksumSHA1,
-            ChecksumMode::ChecksumSHA256,
-            ChecksumMode::ChecksumCRC64NVME,
-        ] {
-            assert!(mode.is_set(), "{mode:?} should be set");
-            assert!(mode.hasher().is_ok(), "{mode:?} reported set but has no hasher");
-        }
-    }
-
-    #[test]
-    fn test_set_default_upgrades_none() {
-        // With `is_set()` fixed, `set_default` must upgrade an unset mode to the
-        // provided default (previously `ChecksumNone` was seen as set and never
-        // upgraded).
-        let mut mode = ChecksumMode::ChecksumNone;
-        mode.set_default(ChecksumMode::ChecksumCRC32C);
-        assert_eq!(mode, ChecksumMode::ChecksumCRC32C);
-
-        // An already-set mode is left untouched.
-        let mut existing = ChecksumMode::ChecksumSHA256;
-        existing.set_default(ChecksumMode::ChecksumCRC32C);
-        assert_eq!(existing, ChecksumMode::ChecksumSHA256);
-    }
-}
-
 #[derive(Default)]
 pub struct Checksum {
    checksum_type: ChecksumMode,
@@ -121,11 +121,8 @@ pub fn new_getobjectreader<'a>(
    let is_compressed = false; //oi.is_compressed_ok();

    let rs_;
-    if rs.is_none()
-        && let Some(part_number) = opts.part_number
-        && part_number > 0
-    {
-        rs_ = part_number_to_rangespec(oi.clone(), part_number);
+    if rs.is_none() && opts.part_number.is_some() && opts.part_number.expect("operation should succeed") > 0 {
+        rs_ = part_number_to_rangespec(oi.clone(), opts.part_number.expect("operation should succeed"));
    } else {
        rs_ = rs.clone();
    }
@@ -161,16 +158,6 @@ pub fn new_getobjectreader<'a>(

        return Ok((get_fn, off as i64, length as i64));
    }
-    if rs.is_none() && opts.part_number.is_none() && oi.size >= 0 {
-        get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| GetObjectReader {
-            object_info: oi.clone(),
-            stream: Box::new(input_reader),
-            buffered_body: None,
-            body_source: Default::default(),
-        });
-
-        return Ok((get_fn, 0, oi.size));
-    }
    Err(ErrorResponse {
        code: S3ErrorCode::InvalidRange,
        message: "Invalid range".to_string(),
@@ -211,57 +198,6 @@ pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use std::sync::Arc;
-
-    fn multipart_object_info() -> ObjectInfo {
-        ObjectInfo {
-            bucket: "bucket".to_string(),
-            name: "object".to_string(),
-            size: 6 * 1024 * 1024,
-            actual_size: 6 * 1024 * 1024,
-            parts: Arc::new(vec![
-                ObjectPartInfo {
-                    number: 1,
-                    size: 5 * 1024 * 1024,
-                    actual_size: 5 * 1024 * 1024,
-                    ..Default::default()
-                },
-                ObjectPartInfo {
-                    number: 2,
-                    size: 1024 * 1024,
-                    actual_size: 1024 * 1024,
-                    ..Default::default()
-                },
-            ]),
-            ..Default::default()
-        }
-    }
-
-    #[test]
-    fn test_new_getobjectreader_uses_full_range_for_unranged_multipart_object() {
-        let oi = multipart_object_info();
-        let opts = ObjectOptions::default();
-
-        let result = new_getobjectreader(&None, &oi, &opts, &HeaderMap::new())
-            .expect("unranged multipart object should build a full-object reader");
-
-        assert_eq!(result.1, 0);
-        assert_eq!(result.2, oi.size);
-    }
-
-    #[test]
-    fn test_new_getobjectreader_keeps_part_number_range_for_multipart_object() {
-        let oi = multipart_object_info();
-        let opts = ObjectOptions {
-            part_number: Some(2),
-            ..Default::default()
-        };
-
-        let result = new_getobjectreader(&None, &oi, &opts, &HeaderMap::new()).expect("part number should build that part range");
-
-        assert_eq!(result.1, 5 * 1024 * 1024);
-        assert_eq!(result.2, 1024 * 1024);
-    }

    #[test]
    fn test_to_s3s_etag() {
@@ -91,41 +91,10 @@ pub fn is_minio_header(header_key: &str) -> bool {
    header_key.to_lowercase().starts_with("x-minio-")
 }

-/// Standard base64 (with `+`/`/` and `=` padding). Every base64 value this
-/// transition client emits or parses — `Content-MD5`, `x-amz-checksum-*`, and
-/// checksum digests in request/response bodies — is S3 wire format, which is
-/// standard base64. The URL-safe, unpadded alphabet used previously made remotes
-/// reject `Content-MD5` with "Invalid content MD5: Base64Error" and could not
-/// even decode a padded checksum coming back from the peer (rustfs/rustfs#4811).
 pub fn base64_encode(input: &[u8]) -> String {
-    base64_simd::STANDARD.encode_to_string(input)
+    base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
 }

 pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
-    base64_simd::STANDARD.decode_to_vec(input)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{base64_decode, base64_encode};
-
-    #[test]
-    fn base64_encode_is_standard_s3_wire_format() {
-        // S3 reads Content-MD5 / checksum values with a standard base64 decoder,
-        // so the encoder must emit '+'/'/' and '=' padding and round-trip through
-        // one. Regression for rustfs/rustfs#4811 ("Invalid content MD5:
-        // Base64Error"). 16-byte MD5-length input chosen to force '=' padding.
-        let digest: [u8; 16] = [
-            0xfb, 0xff, 0xff, 0xef, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0,
-        ];
-        let encoded = base64_encode(&digest);
-        assert!(encoded.ends_with('='), "16-byte input must be padded: {encoded}");
-        assert!(!encoded.contains(['-', '_']), "must use the standard alphabet: {encoded}");
-        let via_standard = base64_simd::STANDARD
-            .decode_to_vec(encoded.as_bytes())
-            .expect("standard decode");
-        assert_eq!(via_standard, digest);
-        // Our own decoder must accept the same wire format it produces.
-        assert_eq!(base64_decode(encoded.as_bytes()).expect("round-trip"), digest);
-    }
+    base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
 }
@@ -23,19 +23,17 @@ use rustfs_config::{
    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
-    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
-    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
-    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
-    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
-    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
-    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
-    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
-    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
-    WEBHOOK_SKIP_TLS_VERIFY,
+    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
+    POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
+    POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
+    PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
+    PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
+    REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
+    REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
+    REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;

@@ -352,21 +350,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
            value: DEFAULT_LIMIT.to_string(),
            hidden_if_empty: false,
        },
-        KV {
-            key: NATS_JETSTREAM_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
-            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
-            hidden_if_empty: false,
-        },
        KV {
            key: COMMENT_KEY.to_owned(),
            value: "".to_owned(),
@@ -23,18 +23,16 @@ use rustfs_config::{
    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
-    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
-    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
-    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
-    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
-    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
-    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
-    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
-    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
-    WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
+    POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
+    POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
+    PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
+    PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
+    REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
+    REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
+    REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;

@@ -330,21 +328,6 @@ pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
            value: DEFAULT_LIMIT.to_string(),
            hidden_if_empty: false,
        },
-        KV {
-            key: NATS_JETSTREAM_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
-            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
-            hidden_if_empty: false,
-        },
        KV {
            key: COMMENT_KEY.to_owned(),
            value: "".to_owned(),
@@ -1,146 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Deterministic crash-point injection for erasure-coded commit sequences.
-//!
-//! Each [`CrashPoint`] names one instant inside a multi-step commit at which a
-//! `#[cfg(test)]` scenario can simulate a hard power loss: the commit stops dead
-//! at the armed step with **no** cleanup, leaving the on-disk state exactly as
-//! the preceding steps left it. The test then reopens the disk (or rebuilds the
-//! erasure set) and asserts the raw state is coherent — the object reads back as
-//! either the whole old version or the whole new version, never a torn mix — and
-//! that any staged leftovers are reclaimable and the operation is safely
-//! retryable. This is the crash-consistency counterpart to the graceful
-//! `should_fail_*` failpoints, which exercise in-process rollback rather than an
-//! abrupt loss.
-//!
-//! Unlike the graceful failpoints, a crash point runs no rollback: it models the
-//! process disappearing, so the assertion is purely over the bytes left on disk.
-//!
-//! Test-only: the arming registry and the real [`should_crash_at`] are
-//! `#[cfg(test)]`. In every other build `should_crash_at` is a const-`false`
-//! `#[inline(always)]` no-op, so the injection points on the real commit paths
-//! (`rename_data`, `write_all_meta`, `complete_multipart_upload`) compile away to
-//! nothing and add no branch to production writes. The [`CrashPoint`] variants
-//! are still constructed at those call sites in every build, so no variant is
-//! dead code.
-//!
-//! Coverage plan: rustfs/backlog#935 / rustfs/backlog#896 (rename_data),
-//! rustfs/backlog#864 (fault-interrupted overwrite/delete must not mutate a
-//! committed object), rustfs/backlog#878 (old-or-new-never-mixed).
-
-/// A named instant inside an erasure-coded commit sequence at which a test can
-/// simulate a hard power loss.
-///
-/// All points share one keyed registry (each arm is matched on `(point, key)`),
-/// so a scenario arms exactly the window under test and every other commit path
-/// runs untouched.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub(crate) enum CrashPoint {
-    /// `rename_data`: after the data dir is renamed into its destination but
-    /// before the old-metadata rollback backup is written. Pre-commit — xl.meta
-    /// is untouched, so a crash here must leave the object readable as the old
-    /// version (or absent when there was none).
-    RenameAfterDataRename,
-    /// `rename_data`: after the rollback backup is durable, immediately before
-    /// the xl.meta commit rename that makes the new version visible. Still
-    /// pre-commit, so a crash here must also leave the old version readable.
-    RenameAfterBackupBeforeMetaCommit,
-    /// `rename_data`: after the xl.meta commit rename has made the new version
-    /// visible, before the durability fsync and with **no** rollback. The commit
-    /// rename already landed, so a crash here must leave the object readable as
-    /// the new version (rustfs/backlog#878, new-version side).
-    RenameAfterMetaCommit,
-    /// `write_all_meta` (the atomic temp+rename behind `update_metadata` and
-    /// `write_metadata`): after the replacement xl.meta is staged in the tmp
-    /// bucket but before the rename that publishes it. Pre-commit — the
-    /// destination xl.meta is untouched, so a crash here must leave the object's
-    /// metadata byte-for-byte the old version (rustfs/backlog#864); the staged
-    /// tmp file is a harmless orphan swept by tmp-bucket GC.
-    MetaWriteAfterTmpBeforeRename,
-    /// `complete_multipart_upload`: after the upload is fully staged and locked
-    /// but before the authoritative `rename_data` commit. Pre-commit — no disk
-    /// has moved staged data, so a crash here must leave any prior committed
-    /// version byte-for-byte intact (rustfs/backlog#864) and the upload fully
-    /// retryable.
-    MultipartBeforeCommitRename,
-    /// `complete_multipart_upload`: after the `rename_data` commit succeeds but
-    /// before the stale `part.N.meta` cleanup. Post-commit — the new version is
-    /// durably committed and visible, so a crash here must leave the object
-    /// readable as the new version; the un-reclaimed staging parts are swept by
-    /// a retried completion or upload GC (rustfs/backlog#946).
-    MultipartAfterCommitBeforePartsCleanup,
-}
-
-#[cfg(test)]
-pub(crate) use armed::{arm, disarm, should_crash_at};
-
-/// Production no-op: the compiler inlines this to `false`, so armed crash points
-/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
-#[cfg(not(test))]
-#[inline(always)]
-pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
-    false
-}
-
-#[cfg(test)]
-mod armed {
-    use super::CrashPoint;
-    use std::sync::{Mutex, MutexGuard};
-
-    // A keyed multiset of pending arms, not a single slot: the rename_data /
-    // xl.meta scenarios serialize via `durability_mode_override` while the
-    // multipart scenarios use `#[serial]`, so the two groups can overlap under
-    // `cargo test`'s shared-process threads. Keying every arm on a unique object
-    // path lets concurrent scenarios coexist without clobbering each other; a
-    // match on `(point, key)` is unambiguous. `#[serial]` still gives the
-    // multipart tests their own serialization for unrelated global state.
-    static ARMED: Mutex<Vec<(CrashPoint, String)>> = Mutex::new(Vec::new());
-
-    fn guard() -> MutexGuard<'static, Vec<(CrashPoint, String)>> {
-        // A poisoned lock only means a prior test panicked mid-scenario; recover
-        // the registry rather than cascade the panic into unrelated tests.
-        ARMED.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
-    }
-
-    /// Arm a one-shot crash: the next operation that reaches `point` while
-    /// committing `key` stops there. Consumed on the first match so it never
-    /// leaks into an unrelated commit. Arming the same `(point, key)` twice is
-    /// idempotent.
-    pub(crate) fn arm(point: CrashPoint, key: &str) {
-        let mut armed = guard();
-        if !armed.iter().any(|(p, k)| *p == point && k == key) {
-            armed.push((point, key.to_string()));
-        }
-    }
-
-    /// Clear the pending arm for exactly this `(point, key)`. A scenario retries
-    /// the same key after the crash, so teardown disarms in case the injection
-    /// was somehow never reached and would otherwise fire inside the retry.
-    pub(crate) fn disarm(point: CrashPoint, key: &str) {
-        guard().retain(|(p, k)| !(*p == point && k == key));
-    }
-
-    /// Returns `true` (consuming the arm) iff a crash is armed for exactly this
-    /// `point` and `key`.
-    pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
-        let mut armed = guard();
-        if let Some(idx) = armed.iter().position(|(p, k)| *p == point && k == key) {
-            armed.swap_remove(idx);
-            true
-        } else {
-            false
-        }
-    }
-}
@@ -13,7 +13,6 @@
 // limitations under the License.

 use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
-use crate::crash_inject::{self, CrashPoint};
 use crate::data_usage::local_snapshot::ensure_data_usage_layout;
 use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout};
 use crate::disk::{
@@ -1479,6 +1478,79 @@ fn should_fail_before_old_metadata_backup(_dst_path: &str) -> bool {
    false
 }

+/// Commit-sequence points where the rename_data crash-consistency harness
+/// (rustfs/backlog#935, test plan in rustfs/backlog#896) can simulate an
+/// abrupt power loss.
+///
+/// Unlike [`should_fail_before_old_metadata_backup`], which exercises the
+/// graceful in-process rollback (delete the staged data dir, return an
+/// error), a crash point models a hard power loss: the commit sequence stops
+/// dead at the armed step with **no** cleanup, leaving the on-disk state
+/// exactly as the preceding steps left it. The harness then reopens the disk
+/// and asserts the raw state is coherent — the object reads back as either the
+/// old version or the new version, never a mixed or corrupt one — without any
+/// rollback code having run.
+///
+/// The variants are constructed at the real commit-path call sites in every
+/// build, but the arming static and [`should_crash_rename_data_at`] are
+/// `#[cfg(test)]`; in production the guard is a const-`false` no-op, so the
+/// injection points compile away to nothing.
+// The `After<step>` naming is deliberate: every crash point names the
+// commit-sequence step it fires immediately after, so the shared prefix is the
+// point, not noise.
+#[allow(clippy::enum_variant_names)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum RenameDataCrashPoint {
+    /// After the data dir has been renamed into its destination but before the
+    /// old-metadata rollback backup is written. xl.meta has not been committed
+    /// yet, so a crash here must leave the object readable as the old version.
+    AfterDataRename,
+    /// After the rollback backup is persisted, immediately before the xl.meta
+    /// commit rename that makes the new version visible. Still pre-commit, so a
+    /// crash here must also leave the object readable as the old version.
+    AfterBackupBeforeMetaCommit,
+    /// After the xl.meta commit rename has made the new version visible, in the
+    /// same window the graceful [`should_fail_after_metadata_commit`] failpoint
+    /// covers — but as a hard power loss with **no** rollback. The commit rename
+    /// already landed on disk, so a crash here must leave the object readable as
+    /// the *new* version. This is the post-commit counterpart to the two
+    /// pre-commit points above, closing the "old or new, never mixed" invariant
+    /// (rustfs/backlog#878) from the new-version side.
+    AfterMetaCommit,
+}
+
+#[cfg(test)]
+static RENAME_DATA_CRASH_POINT: std::sync::Mutex<Option<(RenameDataCrashPoint, String)>> = std::sync::Mutex::new(None);
+
+/// Arm a one-shot crash injection: the next `rename_data` committing into
+/// `dst_path` stops at `point`. Consumed on the first match so it never leaks
+/// into an unrelated commit.
+#[cfg(test)]
+fn arm_rename_data_crash(point: RenameDataCrashPoint, dst_path: &str) {
+    *RENAME_DATA_CRASH_POINT
+        .lock()
+        .expect("test crash point lock should not be poisoned") = Some((point, dst_path.to_string()));
+}
+
+#[cfg(test)]
+fn should_crash_rename_data_at(point: RenameDataCrashPoint, dst_path: &str) -> bool {
+    let mut armed = RENAME_DATA_CRASH_POINT
+        .lock()
+        .expect("test crash point lock should not be poisoned");
+    if armed.as_ref().is_some_and(|(p, path)| *p == point && path == dst_path) {
+        armed.take();
+        true
+    } else {
+        false
+    }
+}
+
+#[cfg(not(test))]
+#[inline(always)]
+fn should_crash_rename_data_at(_point: RenameDataCrashPoint, _dst_path: &str) -> bool {
+    false
+}
+
 #[cfg(not(test))]
 fn should_fail_after_metadata_commit(_dst_path: &str) -> bool {
    false
@@ -4683,7 +4755,9 @@ impl LocalDisk {
                        .await);
                    }
                    let rollback_data_path = rollback_path.join(dir.to_string());
-                    if let Err(err) = rename_all_ignore_missing_source(&dir_path, &rollback_data_path, &rollback_path).await {
+                    if let Err(err) = rename_all(&dir_path, &rollback_data_path, &rollback_path).await
+                        && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
+                    {
                        return Err(restore_delete_rollback_after_error(
                            object_dir,
                            &xlpath,
@@ -4798,16 +4872,6 @@ impl LocalDisk {
        self.write_all_internal(&tmp_file_path, InternalBuf::Ref(buf), tmp_sync, &tmp_volume_dir)
            .await?;

-        // Crash-consistency injection: hard power loss after the replacement
-        // xl.meta is staged in the tmp bucket but before the atomic rename that
-        // publishes it. The destination xl.meta is untouched, so a crash here
-        // must leave the object's metadata byte-for-byte the old version
-        // (rustfs/backlog#864); the staged tmp file is a harmless orphan swept by
-        // tmp-bucket GC. Compiles to a no-op outside `#[cfg(test)]`.
-        if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, path) {
-            return Err(DiskError::Unexpected);
-        }
-
        rename_all(tmp_file_path, &file_path, volume_dir).await?;

        if sync
@@ -6791,7 +6855,7 @@ impl DiskAPI for LocalDisk {
            // is in place but before xl.meta commits. No cleanup — the harness
            // reopens the disk and asserts the object still reads as the old
            // version (the staged data dir is a harmless orphan for GC).
-            if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) {
+            if should_crash_rename_data_at(RenameDataCrashPoint::AfterDataRename, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -6849,7 +6913,7 @@ impl DiskAPI for LocalDisk {
            // backup is durable but before the xl.meta commit rename. No
            // cleanup — the harness asserts the object still reads as the old
            // version, since the destination xl.meta is untouched here.
-            if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) {
+            if should_crash_rename_data_at(RenameDataCrashPoint::AfterBackupBeforeMetaCommit, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -6882,7 +6946,7 @@ impl DiskAPI for LocalDisk {
            // graceful failpoint above, no rollback runs — the commit rename is
            // already on disk, so the harness asserts the object reads back as
            // the new version.
-            if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) {
+            if should_crash_rename_data_at(RenameDataCrashPoint::AfterMetaCommit, dst_path) {
                return Err(DiskError::Unexpected);
            }

@@ -7617,7 +7681,10 @@ impl DiskAPI for LocalDisk {
                    .await);
                }
                let rollback_data_path = rollback_path.join(uuid.to_string());
-                if let Err(err) = rename_all_ignore_missing_source(&old_path, &rollback_data_path, &rollback_path).await {
+                if let Err(err) = rename_all(&old_path, &rollback_data_path, &rollback_path).await
+                    && err != DiskError::FileNotFound
+                    && err != DiskError::VolumeNotFound
+                {
                    return Err(restore_delete_rollback_after_error(
                        file_path.as_path(),
                        &xl_path,
@@ -7895,56 +7962,10 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
 mod test {
    use super::*;
    use rustfs_filemeta::ErasureInfo;
-    use std::io::{self, Write};
+    use std::io;
    use std::pin::Pin;
-    use std::sync::{Arc, Mutex};
    use std::task::{Context, Poll};
    use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
-    use tracing_subscriber::fmt::MakeWriter;
-
-    #[derive(Clone, Default)]
-    struct CapturedLogs {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    struct CapturedLogWriter {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl CapturedLogs {
-        fn contents(&self) -> String {
-            let buffer = self
-                .buffer
-                .lock()
-                .expect("captured logs mutex should not be poisoned")
-                .clone();
-            String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
-        }
-    }
-
-    impl Write for CapturedLogWriter {
-        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-            self.buffer
-                .lock()
-                .expect("captured logs mutex should not be poisoned")
-                .extend_from_slice(buf);
-            Ok(buf.len())
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    impl<'a> MakeWriter<'a> for CapturedLogs {
-        type Writer = CapturedLogWriter;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            CapturedLogWriter {
-                buffer: Arc::clone(&self.buffer),
-            }
-        }
-    }

    fn test_file_info(name: &str, version_id: Uuid, data_dir: Option<Uuid>, data: Option<Bytes>) -> FileInfo {
        let size = data
@@ -8122,14 +8143,13 @@ mod test {
    /// old-or-new invariant, only with a wider (documented) power-loss window.
    mod crash_consistency {
        use super::*;
-        use crate::crash_inject::{self, CrashPoint};
        use tempfile::tempdir;

        const VERSION_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
        const OLD_DATA_DIR: &str = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
        const NEW_DATA_DIR: &str = "cccccccc-cccc-cccc-cccc-cccccccccccc";

-        async fn run_scenario(mode: DurabilityMode, crash: Option<CrashPoint>, with_old_version: bool) {
+        async fn run_scenario(mode: DurabilityMode, crash: Option<RenameDataCrashPoint>, with_old_version: bool) {
            // Serializes with every other durability-sensitive test and pins the
            // resolved tier for the whole scenario (held until dropped).
            let _mode = durability_mode_override::set(mode);
@@ -8180,7 +8200,7 @@ mod test {
                .expect("new tmp data should be written");

            if let Some(point) = crash {
-                crash_inject::arm(point, object);
+                arm_rename_data_crash(point, object);
            }
            let new_fi = test_file_info(object, version_id, Some(new_data_dir), None);
            let result = disk
@@ -8195,7 +8215,7 @@ mod test {
                .await;

            match crash {
-                Some(CrashPoint::RenameAfterMetaCommit) => {
+                Some(RenameDataCrashPoint::AfterMetaCommit) => {
                    // Hard power loss right after the xl.meta commit rename: the
                    // commit landed and no rollback ran, so the object must read
                    // back as the new version — whether or not an old version
@@ -8263,9 +8283,9 @@ mod test {
            }
        }

-        const CRASH_POINTS: [CrashPoint; 2] = [
-            CrashPoint::RenameAfterDataRename,
-            CrashPoint::RenameAfterBackupBeforeMetaCommit,
+        const CRASH_POINTS: [RenameDataCrashPoint; 2] = [
+            RenameDataCrashPoint::AfterDataRename,
+            RenameDataCrashPoint::AfterBackupBeforeMetaCommit,
        ];

        #[tokio::test]
@@ -8311,138 +8331,22 @@ mod test {
        // the same invariant.
        #[tokio::test]
        async fn overwrite_post_commit_crash_keeps_new_version_strict() {
-            run_scenario(DurabilityMode::Strict, Some(CrashPoint::RenameAfterMetaCommit), true).await;
+            run_scenario(DurabilityMode::Strict, Some(RenameDataCrashPoint::AfterMetaCommit), true).await;
        }

        #[tokio::test]
        async fn overwrite_post_commit_crash_keeps_new_version_relaxed() {
-            run_scenario(DurabilityMode::Relaxed, Some(CrashPoint::RenameAfterMetaCommit), true).await;
+            run_scenario(DurabilityMode::Relaxed, Some(RenameDataCrashPoint::AfterMetaCommit), true).await;
        }

        #[tokio::test]
        async fn fresh_post_commit_crash_keeps_new_version_strict() {
-            run_scenario(DurabilityMode::Strict, Some(CrashPoint::RenameAfterMetaCommit), false).await;
+            run_scenario(DurabilityMode::Strict, Some(RenameDataCrashPoint::AfterMetaCommit), false).await;
        }

        #[tokio::test]
        async fn fresh_post_commit_crash_keeps_new_version_relaxed() {
-            run_scenario(DurabilityMode::Relaxed, Some(CrashPoint::RenameAfterMetaCommit), false).await;
-        }
-    }
-
-    /// Crash-consistency for the in-place xl.meta update path — the atomic
-    /// temp+rename inside [`LocalDisk::write_all_meta`] shared by `update_metadata`
-    /// and `write_metadata` (delete markers, tag/metadata rewrites, decommission).
-    ///
-    /// rustfs/backlog#864: a fault that interrupts an in-place metadata rewrite
-    /// must not mutate the committed object. [`CrashPoint::MetaWriteAfterTmpBeforeRename`]
-    /// models a hard power loss after the replacement xl.meta is staged in the tmp
-    /// bucket but before the publishing rename. Both durability tiers are held to
-    /// the same rule: the destination xl.meta survives byte-for-byte, the object
-    /// still reads as the old version, the staged tmp file is a reclaimable orphan
-    /// confined to the tmp bucket, and a later un-injected rewrite publishes
-    /// cleanly (retryable). This path previously had only parser-level unit tests.
-    mod meta_write_crash_consistency {
-        use super::*;
-        use crate::crash_inject::{self, CrashPoint};
-        use tempfile::tempdir;
-
-        async fn run(mode: DurabilityMode) {
-            // Serialize with every other durability-sensitive test and pin the
-            // resolved tier for the whole scenario.
-            let _mode = durability_mode_override::set(mode);
-
-            let dir = tempdir().expect("temp dir should be created");
-            let endpoint =
-                Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
-            let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
-
-            let bucket = "bucket";
-            let object = "meta-write-crash-object";
-            ensure_test_volume(&disk, bucket).await;
-            ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
-
-            // Seed a committed object (version 1) by writing its xl.meta directly:
-            // read_data=false reads never touch the data dir, so no shard staging
-            // is needed to exercise the metadata commit window.
-            let object_dir = dir.path().join(bucket).join(object);
-            fs::create_dir_all(&object_dir).await.expect("object dir should be created");
-            let meta_path = object_dir.join(STORAGE_FORMAT_FILE);
-            let v1 = Uuid::new_v4();
-            let old_meta = test_meta(test_file_info(object, v1, Some(Uuid::new_v4()), None));
-            fs::write(&meta_path, &old_meta)
-                .await
-                .expect("committed xl.meta should be written");
-
-            // Arm the crash, then attempt an in-place rewrite that adds version 2.
-            let meta_key = format!("{object}/{STORAGE_FORMAT_FILE}");
-            crash_inject::arm(CrashPoint::MetaWriteAfterTmpBeforeRename, &meta_key);
-            let v2 = Uuid::new_v4();
-            let result = disk
-                .write_metadata("", bucket, object, test_file_info(object, v2, Some(Uuid::new_v4()), None))
-                .await;
-            assert!(
-                matches!(result, Err(DiskError::Unexpected)),
-                "{mode:?}: the armed crash point must be the failure that surfaced, got {result:?}"
-            );
-
-            // Reopen the disk to model a process restart after the crash.
-            drop(disk);
-            let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should reopen");
-
-            // rustfs/backlog#864: the committed xl.meta is byte-for-byte the old
-            // version — the interrupted rewrite never published.
-            let after = fs::read(&meta_path).await.expect("xl.meta must survive the crash");
-            assert_eq!(&after, &old_meta, "{mode:?}: xl.meta must remain the old version byte-for-byte");
-
-            // The old version still reads; the un-published new version is absent.
-            disk.read_version("", bucket, object, &v1.to_string(), &ReadOptions::default())
-                .await
-                .expect("the old version must remain readable after the crash");
-            let new_read = disk
-                .read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
-                .await;
-            assert!(
-                matches!(new_read, Err(DiskError::FileVersionNotFound)),
-                "{mode:?}: the interrupted update must not publish the new version, got {new_read:?}"
-            );
-
-            // The crash leaves no staging debris in the object directory; the
-            // staged replacement is a reclaimable orphan under the tmp bucket.
-            let mut object_entries = fs::read_dir(&object_dir).await.expect("object dir should list");
-            let mut object_files = Vec::new();
-            while let Some(entry) = object_entries
-                .next_entry()
-                .await
-                .expect("object dir entry should be readable")
-            {
-                object_files.push(entry.file_name().to_string_lossy().to_string());
-            }
-            assert_eq!(
-                object_files,
-                vec![STORAGE_FORMAT_FILE.to_string()],
-                "{mode:?}: the object directory must hold only its committed xl.meta"
-            );
-
-            // A retried rewrite (un-injected) publishes cleanly: the path is safely
-            // retryable after the crash.
-            crash_inject::disarm(CrashPoint::MetaWriteAfterTmpBeforeRename, &meta_key);
-            disk.write_metadata("", bucket, object, test_file_info(object, v2, Some(Uuid::new_v4()), None))
-                .await
-                .expect("a retried metadata rewrite must succeed after the crash");
-            disk.read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
-                .await
-                .expect("the retried version must be readable");
-        }
-
-        #[tokio::test]
-        async fn meta_write_crash_before_rename_keeps_old_version_strict() {
-            run(DurabilityMode::Strict).await;
-        }
-
-        #[tokio::test]
-        async fn meta_write_crash_before_rename_keeps_old_version_relaxed() {
-            run(DurabilityMode::Relaxed).await;
+            run_scenario(DurabilityMode::Relaxed, Some(RenameDataCrashPoint::AfterMetaCommit), false).await;
        }
    }

@@ -10472,149 +10376,6 @@ mod test {
        );
    }

-    #[tokio::test(flavor = "current_thread")]
-    async fn test_delete_version_missing_inline_data_dir_does_not_warn() {
-        use tempfile::tempdir;
-
-        let dir = tempdir().expect("temp dir should be created");
-        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
-        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
-
-        let bucket = "bucket";
-        let object = "dir/inline-object";
-        let version_id = Uuid::parse_str("31313131-1111-2222-3333-444444444444").expect("version id should parse");
-        let data_dir = Uuid::parse_str("32323232-1111-2222-3333-444444444444").expect("data dir should parse");
-        let rollback_dir = Uuid::parse_str("33333333-1111-2222-3333-444444444444").expect("rollback dir should parse");
-
-        ensure_test_volume(&disk, bucket).await;
-
-        let object_dir = dir.path().join(bucket).join(object);
-        fs::create_dir_all(&object_dir).await.expect("object dir should be created");
-        let fi = test_file_info(object, version_id, Some(data_dir), Some(Bytes::from_static(b"inline")));
-        fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(fi.clone()))
-            .await
-            .expect("inline metadata should be written");
-        assert!(!object_dir.join(data_dir.to_string()).exists());
-
-        let logs = CapturedLogs::default();
-        let subscriber = tracing_subscriber::fmt()
-            .with_writer(logs.clone())
-            .with_ansi(false)
-            .without_time()
-            .finish();
-        let _guard = tracing::subscriber::set_default(subscriber);
-
-        disk.delete_version(
-            bucket,
-            object,
-            fi,
-            false,
-            DeleteOptions {
-                old_data_dir: Some(rollback_dir),
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("missing inline data dir should not fail deletion");
-
-        assert!(!object_dir.join(STORAGE_FORMAT_FILE).exists());
-        assert!(
-            object_dir
-                .join(rollback_dir.to_string())
-                .join(STORAGE_FORMAT_FILE_BACKUP)
-                .exists()
-        );
-        assert!(!logs.contents().contains("reliable_rename failed"));
-    }
-
-    #[tokio::test(flavor = "current_thread")]
-    async fn test_delete_versions_missing_inline_data_dir_does_not_warn() {
-        use tempfile::tempdir;
-
-        let dir = tempdir().expect("temp dir should be created");
-        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
-        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
-
-        let bucket = "bucket";
-        let object = "dir/inline-object-batch";
-        let version_id = Uuid::parse_str("34343434-1111-2222-3333-444444444444").expect("version id should parse");
-        let data_dir = Uuid::parse_str("35353535-1111-2222-3333-444444444444").expect("data dir should parse");
-        let rollback_dir = Uuid::parse_str("36363636-1111-2222-3333-444444444444").expect("rollback dir should parse");
-
-        ensure_test_volume(&disk, bucket).await;
-
-        let object_dir = dir.path().join(bucket).join(object);
-        fs::create_dir_all(&object_dir).await.expect("object dir should be created");
-        let fi = test_file_info(object, version_id, Some(data_dir), Some(Bytes::from_static(b"inline")));
-        fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(fi.clone()))
-            .await
-            .expect("inline metadata should be written");
-        assert!(!object_dir.join(data_dir.to_string()).exists());
-
-        let logs = CapturedLogs::default();
-        let subscriber = tracing_subscriber::fmt()
-            .with_writer(logs.clone())
-            .with_ansi(false)
-            .without_time()
-            .finish();
-        let _guard = tracing::subscriber::set_default(subscriber);
-
-        disk.delete_versions_internal(
-            bucket,
-            object,
-            &[fi],
-            &DeleteOptions {
-                old_data_dir: Some(rollback_dir),
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("missing inline data dir should not fail batch deletion");
-
-        assert!(!object_dir.join(STORAGE_FORMAT_FILE).exists());
-        assert!(
-            object_dir
-                .join(rollback_dir.to_string())
-                .join(STORAGE_FORMAT_FILE_BACKUP)
-                .exists()
-        );
-        assert!(!logs.contents().contains("reliable_rename failed"));
-    }
-
-    #[tokio::test(flavor = "current_thread")]
-    async fn test_ignore_missing_source_helper_still_warns_on_real_error() {
-        use tempfile::tempdir;
-
-        let dir = tempdir().expect("temp dir should be created");
-        let src = dir.path().join("src");
-        let dst = dir.path().join("dst");
-        fs::create_dir_all(&src).await.expect("source dir should be created");
-        fs::write(src.join("part.1"), b"live-data")
-            .await
-            .expect("source data should be written");
-        fs::create_dir_all(&dst).await.expect("destination dir should be created");
-        fs::write(dst.join("sentinel"), b"conflict")
-            .await
-            .expect("destination conflict should be written");
-
-        let logs = CapturedLogs::default();
-        let subscriber = tracing_subscriber::fmt()
-            .with_writer(logs.clone())
-            .with_ansi(false)
-            .without_time()
-            .finish();
-        let _guard = tracing::subscriber::set_default(subscriber);
-
-        let err = rename_all_ignore_missing_source(&src, &dst, dir.path())
-            .await
-            .expect_err("a non-empty rollback destination must reject rename");
-
-        assert_ne!(err, DiskError::FileNotFound);
-        assert!(src.exists());
-        assert!(dst.join("sentinel").exists());
-        assert!(logs.contents().contains("reliable_rename failed"));
-    }
-
    #[tokio::test]
    async fn test_delete_version_rollback_restores_staged_data_dir() {
        use tempfile::tempdir;
@@ -26,7 +26,7 @@ use std::{
 };
 use tokio::fs;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
-use tracing::warn;
+use tracing::{debug, warn};

 /// Check path length according to OS limits.
 pub fn check_path_length(path_name: &str) -> Result<()> {
@@ -527,7 +527,7 @@ async fn reliable_rename_inner(
    src_file_path: impl AsRef<Path>,
    dst_file_path: impl AsRef<Path>,
    base_dir: impl AsRef<Path>,
-    warn_on_missing_source: bool,
+    warn_on_failure: bool,
 ) -> io::Result<()> {
    if let Some(parent) = dst_file_path.as_ref().parent()
        && !file_exists(parent)
@@ -542,8 +542,32 @@ async fn reliable_rename_inner(
                i += 1;
                continue;
            }
-            if warn_on_missing_source || e.kind() != io::ErrorKind::NotFound {
-                warn_reliable_rename_failure(src_file_path.as_ref(), dst_file_path.as_ref(), base_dir.as_ref(), &e);
+            if warn_on_failure {
+                // NotFound is an expected outcome for speculative renames (staging
+                // a dataDir that inline objects never had, trashing an already-
+                // removed tmp path, restoring an absent rollback backup): those
+                // callers treat FileNotFound as acceptable, and on per-request
+                // delete paths a WARN becomes a log storm (one line per disk per
+                // DELETE). Keep it at debug so quorum-masked cases (e.g. a tmp
+                // file lost before a commit rename) stay diagnosable; genuine
+                // failures keep the WARN. The error is returned either way.
+                if e.kind() == io::ErrorKind::NotFound {
+                    debug!(
+                        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
+                        src_file_path.as_ref(),
+                        dst_file_path.as_ref(),
+                        base_dir.as_ref(),
+                        e
+                    );
+                } else {
+                    warn!(
+                        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
+                        src_file_path.as_ref(),
+                        dst_file_path.as_ref(),
+                        base_dir.as_ref(),
+                        e
+                    );
+                }
            }
            return Err(e);
        }
@@ -554,13 +578,6 @@ async fn reliable_rename_inner(
    Ok(())
 }

-fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
-    warn!(
-        "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
-        src_file_path, dst_file_path, base_dir, err
-    );
-}
-
 /// Whether a failed `rename` in [`reliable_rename_inner`] should be retried.
 ///
 /// Only the first failure is retried, and `NotFound` is never retried: the
@@ -738,22 +755,24 @@ mod tests {
    }

    #[tokio::test]
-    async fn rename_all_ignore_missing_source_returns_ok_without_warn() {
+    async fn rename_all_ignore_missing_source_returns_ok() {
        let temp_dir = tempdir().expect("create temp dir");
        let src = temp_dir.path().join("missing");
        let dst = temp_dir.path().join("dst");

-        let (logs, _guard) = warn_capture();
        rename_all_ignore_missing_source(&src, &dst, temp_dir.path())
            .await
            .expect("missing cleanup source must be ignored");

        assert!(!dst.exists());
-        assert!(!logs.contents().contains("reliable_rename failed"));
    }

    #[tokio::test]
-    async fn rename_all_missing_source_still_warns() {
+    async fn rename_all_missing_source_skips_rename_warn() {
+        // Callers like delete_version's rollback staging accept FileNotFound
+        // (inline objects have no dataDir), so a missing source must not emit
+        // the per-disk "reliable_rename failed" WARN — under delete storms it
+        // was pure log noise. The error itself is still returned.
        let temp_dir = tempdir().expect("create temp dir");
        let src = temp_dir.path().join("missing");
        let dst = temp_dir.path().join("dst");
@@ -766,8 +785,8 @@ mod tests {
        assert!(matches!(err, DiskError::FileNotFound));
        let captured = logs.contents();
        assert!(
-            captured.contains("reliable_rename failed"),
-            "ordinary missing-source failures must keep the WARN, got: {captured}"
+            !captured.contains("reliable_rename failed"),
+            "NotFound must not emit the reliable_rename WARN, got: {captured}"
        );
    }

@@ -1091,7 +1091,6 @@ where
        let mut completed = 0usize;
        let mut failed = 0usize;
        let mut first_shard_recorded = false;
-        let mut hedged = false;
        // Scope the reader borrow (held by `reader_iter`/`sets`) so it is released
        // before the retirement pass mutates `self.readers` below.
        {
@@ -1116,53 +1115,7 @@ where
                ));
            }

-            // Hedge the straggler. Once every healthy shard has had `hedge_delay`
-            // to return, stop waiting out the full per-shard `read_timeout` for a
-            // slow shard: cancel the still-pending reads and let the stripe-aligned
-            // parity substitution below cover their slots — but only when enough
-            // unengaged parity remains to reconstruct them, otherwise keep waiting.
-            // A slow shard here otherwise stalls the whole stripe (and the GET's
-            // first byte) until the 10s read timeout fires: the large-object GET
-            // first-byte tail seen in backlog#1156.
-            let hedge_delay = shard_read_hedge_delay(read_timeout);
-            let hedge_sleep = hedge_delay.map(tokio::time::sleep);
-            tokio::pin!(hedge_sleep);
-            loop {
-                let item = match hedge_sleep.as_mut().as_pin_mut() {
-                    Some(sleep) if !hedged => {
-                        tokio::select! {
-                            biased;
-                            item = sets.next() => item,
-                            _ = sleep => {
-                                hedged = true;
-                                // Stop waiting for the straggler(s) only once the stripe
-                                // can be finished without them: either every data shard
-                                // is already in (no reconstruction needed), or there are
-                                // at least `data_shards + 1` shards — decode quorum plus
-                                // a reconstruction-verification source, so a missing data
-                                // shard is reconstructed *and* verified, never settled at
-                                // exactly `data_shards` (which would silently skip
-                                // verification, erasure.rs). In the read-all default every
-                                // slot is engaged, so `sets` already holds data + parity
-                                // and this stops one slow shard from stalling the stripe
-                                // (and the GET's first byte) for the full read timeout
-                                // (backlog#1156). Otherwise keep waiting — the straggler
-                                // may be the verification source, or the only route to
-                                // quorum on a degraded stripe.
-                                let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
-                                let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
-                                if !data_missing || succeeded > data_shards {
-                                    break;
-                                }
-                                continue;
-                            }
-                        }
-                    }
-                    _ => sets.next().await,
-                };
-                let Some((i, _read_cost, result, _should_retire)) = item else {
-                    break;
-                };
+            while let Some((i, _read_cost, result, _should_retire)) = sets.next().await {
                completed += 1;
                if !first_shard_recorded {
                    if let Some(path) = metrics_path {
@@ -1181,46 +1134,14 @@ where
                        failed += 1;
                    }
                }
-                // Once the hedge has fired, stop as soon as the stripe can finish
-                // (all data present, or the decode+verification quorum) rather than
-                // draining every straggler — a shard that resolves just after the
-                // hedge must not pull the first byte back out to the read timeout.
-                // Gated on `hedged` so a healthy stripe still reads every shard
-                // (the read-all default invariant, backlog#923 gate off).
-                if hedged {
-                    let succeeded = shards.iter().filter(|shard| shard.is_some()).count();
-                    let data_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
-                    if !data_missing || succeeded > data_shards {
-                        break;
-                    }
-                }
            }
        }

-        // Retire any shard the hedge left in flight: dropping `sets` above
-        // cancelled its read, so the reader is now at an indeterminate stream
-        // position and can never be block-aligned again — the same reason a
-        // mid-block read error retires a reader. Its slot stays missing and is
-        // covered by the stripe-aligned parity substitution below.
-        if hedged {
-            for i in 0..num_readers {
-                if participating[i] && shards[i].is_none() && errs[i].is_none() {
-                    errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read hedged after a slow shard")));
-                    retire_readers.push(i);
-                }
-            }
-        }
-
-        // A data shard may have died or been hedged this stripe. The unengaged
+        // A data shard may have died during this stripe's reads. The unengaged
        // parity readers are still unopened, so they can be aligned to *this*
        // stripe and read now: engage them one at a time until the stripe has
        // `data_shards + 1` successful shards (decode quorum plus one
-        // reconstruction-verification source) or parity runs out. In the read-all
-        // default (`RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE` off) every parity
-        // slot is already engaged, so `!self.engaged[idx]` finds nothing and this
-        // loop is a no-op — the hedged data shard is reconstructed from the parity
-        // already read above. It only substitutes deferred parity in the
-        // data-shards-only path (backlog#923).
+        // reconstruction-verification source) or parity runs out.
        loop {
            let success_now = shards.iter().filter(|shard| shard.is_some()).count();
            let data_shard_missing = shards.iter().take(data_shards).any(|shard| shard.is_none());
@@ -3711,152 +3632,6 @@ mod tests {
        assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
    }

-    /// Lockstep GET regression for the large-object first-byte tail (backlog#1156).
-    /// A single slow data shard must not stall the stripe for the full per-shard
-    /// `read_timeout`: once the hedge delay elapses and the stripe already holds a
-    /// decode-plus-verification quorum, the lockstep read retires the straggler and
-    /// returns in ~hedge_delay even though `read_timeout` is 60s. Before the hedge
-    /// the lockstep path waited for every launched read, so this completed in
-    /// ~read_timeout (the 10s stall observed on large-object GETs) instead of ~100ms.
-    #[tokio::test]
-    async fn test_lockstep_hedges_slow_shard_instead_of_waiting_read_timeout() {
-        const NUM_SHARDS: usize = 1;
-        const BLOCK_SIZE: usize = 64;
-        const DATA_SHARDS: usize = 2;
-        // Two parity so data_shards + 1 shards are available without the slow one.
-        const PARITY_SHARDS: usize = 2;
-        const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
-
-        let hash_algo = HashAlgorithm::None;
-        let readers = vec![
-            // Data shard 0 never resolves; without the hedge the stripe waits out
-            // the full 60s read timeout for it.
-            Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
-            Some(BitrotReader::new(
-                TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
-                SHARD_SIZE,
-                hash_algo.clone(),
-                false,
-            )),
-            Some(BitrotReader::new(
-                TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
-                SHARD_SIZE,
-                hash_algo.clone(),
-                false,
-            )),
-            Some(BitrotReader::new(
-                TestShardReader::Ready(Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS])),
-                SHARD_SIZE,
-                hash_algo,
-                false,
-            )),
-        ];
-
-        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-        // Reconstruction verification selects the lockstep read path.
-        let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
-            readers,
-            erasure,
-            0,
-            NUM_SHARDS * BLOCK_SIZE,
-            None,
-            vec![ShardReadCost::Unknown; 4],
-            Duration::from_secs(60),
-            true,
-        );
-        assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
-
-        let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
-            .await
-            .expect("lockstep read must hedge the slow shard instead of waiting out the 60s read timeout");
-
-        // The slow data shard was hedged: retired with a TimedOut error; the stripe
-        // reached data_shards + 1 shards from the healthy data + parity.
-        assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
-        assert!(parallel_reader.readers[0].is_none());
-        assert!(bufs[0].is_none());
-        assert!(bufs[1].is_some());
-        assert!(bufs[2].is_some());
-        assert!(bufs[3].is_some());
-        assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
-    }
-
-    /// Lockstep verification-quorum regression (backlog#1156). When a data shard is
-    /// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus
-    /// a reconstruction-verification source), never at exactly `data_shards` — that
-    /// would silently skip reconstruction verification (erasure.rs verifies only
-    /// when `available_shards > data_shards`) and could stream a corrupt
-    /// reconstruction. Here `data1` and `parity2` are instant, `data0` is dead-slow,
-    /// and the verification-source `parity3` returns *after* the hedge delay. The
-    /// read must wait for `parity3` (reaching `data_shards + 1`) rather than settle
-    /// at `data_shards` and abandon it.
-    #[tokio::test]
-    async fn test_lockstep_hedge_waits_for_verification_quorum() {
-        const NUM_SHARDS: usize = 1;
-        const BLOCK_SIZE: usize = 64;
-        const DATA_SHARDS: usize = 2;
-        const PARITY_SHARDS: usize = 2;
-        const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
-
-        let hash_algo = HashAlgorithm::None;
-        // parity3 resolves 200ms in — after the 100ms hedge, but well under the
-        // 500ms test bound and the 60s read timeout.
-        let parity3_ready_at = TokioInstant::now() + Duration::from_millis(200);
-        let readers = vec![
-            Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo.clone(), false)),
-            Some(BitrotReader::new(
-                TestShardReader::Ready(Cursor::new(vec![1_u8; SHARD_SIZE * NUM_SHARDS])),
-                SHARD_SIZE,
-                hash_algo.clone(),
-                false,
-            )),
-            Some(BitrotReader::new(
-                TestShardReader::Ready(Cursor::new(vec![2_u8; SHARD_SIZE * NUM_SHARDS])),
-                SHARD_SIZE,
-                hash_algo.clone(),
-                false,
-            )),
-            Some(BitrotReader::new(
-                TestShardReader::ReadyAt {
-                    cursor: Cursor::new(vec![3_u8; SHARD_SIZE * NUM_SHARDS]),
-                    ready_at: parity3_ready_at,
-                    sleep: None,
-                },
-                SHARD_SIZE,
-                hash_algo,
-                false,
-            )),
-        ];
-
-        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-        let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
-            readers,
-            erasure,
-            0,
-            NUM_SHARDS * BLOCK_SIZE,
-            None,
-            vec![ShardReadCost::Unknown; 4],
-            Duration::from_secs(60),
-            true,
-        );
-        assert!(parallel_reader.verify_reconstruction, "test must exercise the lockstep read path");
-
-        let (bufs, errs) = tokio::time::timeout(Duration::from_millis(500), parallel_reader.read())
-            .await
-            .expect("hedge must reach data_shards + 1, not stall to the 60s read timeout");
-
-        // data0 stays missing/retired; the verification-source parity3 was NOT
-        // abandoned at the hedge — the read waited for it to reach data_shards + 1.
-        assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
-        assert!(parallel_reader.readers[0].is_none());
-        assert!(bufs[0].is_none());
-        assert!(bufs[1].is_some());
-        assert!(bufs[2].is_some());
-        assert!(bufs[3].is_some(), "verification-source parity must be used, not settled away");
-        assert!(parallel_reader.readers[3].is_some(), "verification-source parity must not be retired");
-        assert_eq!(DATA_SHARDS + 1, bufs.iter().filter(|buf| buf.is_some()).count());
-    }
-
    #[tokio::test]
    async fn test_parallel_reader_drains_completed_shards_after_quorum() {
        const NUM_SHARDS: usize = 1;
@@ -36,7 +36,6 @@ mod cache_value;
 mod cluster;
 mod config;
 mod core;
-mod crash_inject;
 mod data_movement;
 mod data_usage;
 mod diagnostics;
@@ -429,19 +429,4 @@ mod tests {
        assert!(!opts.user_metadata.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()));
        assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
    }
-
-    #[test]
-    fn build_transition_put_options_requests_no_checksum_and_content_md5() {
-        // Regression for rustfs/rustfs#4811: transition uploads must leave the
-        // additional-checksum modes unset and rely on Content-MD5. If `checksum`
-        // were (incorrectly) reported as set, the >128 MiB multipart put path
-        // would call `ChecksumNone.hasher()` and fail with "unsupported checksum
-        // type". Objects <=128 MiB take the single-part path and only worked by
-        // silently dropping the checksum, so pin both invariants here.
-        let opts = build_transition_put_options("COLD".to_string(), HashMap::new());
-
-        assert!(!opts.checksum.is_set(), "transition put must not request an additional checksum");
-        assert!(!opts.auto_checksum.is_set(), "transition put must not preset auto_checksum");
-        assert!(opts.send_content_md5, "transition put must send Content-MD5");
-    }
 }
@@ -1275,10 +1275,6 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
            continue;
        }

-        if files[idx].data.is_none() && disks[idx].is_none() {
-            continue;
-        }
-
        let inline_data = files[idx].data.clone();
        let disk = disks[idx].clone();
        let data_dir = files[idx].data_dir.unwrap_or_default();
@@ -3518,13 +3518,6 @@ fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::Checks
        rustfs_rio::ChecksumType::CRC32 => Some(part.checksum_crc32.clone()),
        rustfs_rio::ChecksumType::CRC32C => Some(part.checksum_crc32c.clone()),
        rustfs_rio::ChecksumType::CRC64_NVME => Some(part.checksum_crc64nvme.clone()),
-        // XXHash3/64/128 and SHA-512 (AWS 2026-04): s3s CompletePart has no typed
-        // field to carry a client-supplied per-part value in the
-        // CompleteMultipartUpload request, so accept the type with no double-check
-        // value (the part was already verified server-side at UploadPart). This
-        // mirrors the missing-value path of the five typed algorithms. Reject only
-        // genuinely unset/invalid types.
-        base if base.is_set() => Some(None),
        _ => None,
    }
 }
@@ -6725,23 +6718,6 @@ mod tests {
            ..Default::default()
        };
        assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
-
-        // AWS 2026-04 additional algorithms have no CompletePart field, so they are
-        // accepted with no double-check value (verified at UploadPart) — Some(None),
-        // NOT the outer None that would fail the multipart completion (#1261).
-        for ct in [
-            rustfs_rio::ChecksumType::XXHASH3,
-            rustfs_rio::ChecksumType::XXHASH64,
-            rustfs_rio::ChecksumType::XXHASH128,
-            rustfs_rio::ChecksumType::SHA512,
-            rustfs_rio::ChecksumType::MD5,
-        ] {
-            assert_eq!(complete_part_checksum(&CompletePart::default(), ct), Some(None), "{ct:?}");
-        }
-
-        // Genuinely unset / invalid types must still be rejected (outer None).
-        assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::NONE), None);
-        assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::INVALID), None);
    }

    fn direct_memory_test_metadata(size: i64) -> (ObjectInfo, FileInfo, ObjectOptions) {
@@ -21,10 +21,10 @@
 //! unchanged; method bodies are moved verbatim and runtime behavior is the same.

 use super::super::*;
-use crate::crash_inject::{self, CrashPoint};
 use std::future::Future;
 use std::time::Duration;
 use tokio::task::JoinSet;
+use tracing::Instrument as _;

 fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err: DiskError) -> Error {
    if err == DiskError::FileNotFound {
@@ -354,6 +354,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
        let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
        let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));

+        let ec_write_span = tracing::info_span!(
+            "rustfs.ec.write",
+            event = "rustfs_ec_write",
+            component = "ecstore",
+            "rustfs.ec.data_shards" = fi.erasure.data_blocks,
+            "rustfs.ec.parity_shards" = fi.erasure.parity_blocks,
+            "rustfs.ec.write_quorum" = write_quorum,
+            "rustfs.ec.block_bytes" = fi.erasure.block_size,
+            "rustfs.distribution.targets" = shuffle_disks.len(),
+            "rustfs.distribution.transport" = "erasure"
+        );
        let result: Result<PartInfo> = async {
            let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
            let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
@@ -437,19 +448,29 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
            rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
            let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);

-            let (reader, w_size) = match write_path {
-                SmallWritePath::SingleBlockNonInline => {
-                    Arc::new(erasure)
-                        .encode_single_block_non_inline(stream, &mut writers, write_quorum)
-                        .await?
+            let (reader, w_size) = async {
+                match write_path {
+                    SmallWritePath::SingleBlockNonInline => {
+                        Arc::new(erasure)
+                            .encode_single_block_non_inline(stream, &mut writers, write_quorum)
+                            .await
+                    }
+                    SmallWritePath::PipelineBatchedLarge => {
+                        Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await
+                    }
+                    SmallWritePath::Inline | SmallWritePath::Pipeline => {
+                        Arc::new(erasure).encode(stream, &mut writers, write_quorum).await
+                    }
                }
-                SmallWritePath::PipelineBatchedLarge => {
-                    Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?
-                }
-                SmallWritePath::Inline | SmallWritePath::Pipeline => {
-                    Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
-                }
-            };
+            }
+            .instrument(tracing::info_span!(
+                "rustfs.ec.distribute",
+                event = "rustfs_ec_distribute",
+                component = "ecstore",
+                "rustfs.distribution.targets" = shuffle_disks.len(),
+                "rustfs.distribution.transport" = "erasure"
+            ))
+            .await?;

            if let Some(stage_start) = encode_stage_start {
                rustfs_io_metrics::record_put_object_stage_duration(
@@ -577,6 +598,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {

            Ok(ret)
        }
+        .instrument(ec_write_span)
        .await;

        if result.is_err()
@@ -1421,15 +1443,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {

        let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);

-        // Crash-consistency injection: hard power loss after the upload is fully
-        // staged and locked but before the authoritative rename_data commit. No
-        // disk has moved the staged data, so a crash here must leave any prior
-        // committed version byte-for-byte intact (rustfs/backlog#864) and the
-        // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
-        if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, object) {
-            return Err(StorageError::Unexpected);
-        }
-
        // The trailing `_` drops the rename_data old-size backfill
        // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
        // `get_object_info` lookup, so the backfill has no consumer here yet.
@@ -1444,16 +1457,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
        )
        .await?;

-        // Crash-consistency injection: hard power loss after the authoritative
-        // rename_data commit succeeded but before the stale part.N.meta cleanup.
-        // The new version is durably committed and visible, so a crash here must
-        // leave the object readable as the new version; the un-reclaimed staging
-        // parts are swept by a retried completion or upload GC (rustfs/backlog#946).
-        // Compiles to a no-op outside `#[cfg(test)]`.
-        if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object) {
-            return Err(StorageError::Unexpected);
-        }
-
        // backlog#946: reclaim the stale per-part metadata (and any superfluous
        // part.N data files no longer in the completed set) only *after* the
        // authoritative rename_data commit above has succeeded. If rename_data
@@ -1532,8 +1535,37 @@ mod tests {
    use crate::storage_api_contracts::namespace::NamespaceLocking as _;
    use rustfs_lock::{LockClient, client::local::LocalClient};
    use serial_test::serial;
+    use std::sync::atomic::{AtomicU8, Ordering};
    use tempfile::TempDir;
    use tokio::sync::RwLock;
+    use tracing_subscriber::{Layer, Registry, layer::SubscriberExt};
+
+    const EC_WRITE_SPAN_SEEN: u8 = 1;
+    const EC_DISTRIBUTE_SPAN_SEEN: u8 = 2;
+
+    #[derive(Clone)]
+    struct EcWriteSpanLayer {
+        seen: Arc<AtomicU8>,
+    }
+
+    impl<S> Layer<S> for EcWriteSpanLayer
+    where
+        S: tracing::Subscriber,
+    {
+        fn on_new_span(
+            &self,
+            attrs: &tracing::span::Attributes<'_>,
+            _: &tracing::Id,
+            _: tracing_subscriber::layer::Context<'_, S>,
+        ) {
+            let flag = match attrs.metadata().name() {
+                "rustfs.ec.write" => EC_WRITE_SPAN_SEEN,
+                "rustfs.ec.distribute" => EC_DISTRIBUTE_SPAN_SEEN,
+                _ => return,
+            };
+            self.seen.fetch_or(flag, Ordering::Relaxed);
+        }
+    }

    async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
        let mut leftovers = Vec::new();
@@ -1762,6 +1794,9 @@ mod tests {

    #[tokio::test]
    async fn put_object_part_failure_cleans_tmp_workspace_inline() {
+        let seen = Arc::new(AtomicU8::new(0));
+        let dispatch = tracing::Dispatch::new(Registry::default().with(EcWriteSpanLayer { seen: Arc::clone(&seen) }));
+        let _guard = tracing::dispatcher::set_default(&dispatch);
        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
        let bucket = "multipart-tmp-clean-bucket";
        let object = "object";
@@ -1785,6 +1820,12 @@ mod tests {
            .await
            .expect_err("short multipart stream should fail");

+        assert_eq!(
+            seen.load(Ordering::Relaxed),
+            EC_WRITE_SPAN_SEEN | EC_DISTRIBUTE_SPAN_SEEN,
+            "multipart write must create EC write and distribution spans"
+        );
+
        let leftovers = non_trash_tmp_entries(&temp_dirs).await;
        assert!(
            leftovers.is_empty(),
@@ -2145,222 +2186,4 @@ mod tests {
        let paged: Vec<&str> = first.iter().chain(second.iter()).map(|u| u.upload_id.as_str()).collect();
        assert_eq!(paged, vec!["u0", "u1", "u2", "u3"]);
    }
-
-    /// Crash-consistency for the two `complete_multipart_upload` commit windows.
-    ///
-    /// rustfs/backlog#864: a fault that interrupts a completion must never mutate
-    /// an already-committed object; the object must read back as the whole old
-    /// version or the whole new version, never a torn mix, and the upload must
-    /// stay recoverable.
-    ///
-    /// [`CrashPoint::MultipartBeforeCommitRename`] fires after the upload is fully
-    /// staged and locked but before the authoritative `rename_data` commit: a
-    /// prior committed version survives byte-for-byte and the upload is retryable.
-    /// [`CrashPoint::MultipartAfterCommitBeforePartsCleanup`] fires after the
-    /// commit lands but before the stale `part.N.meta` cleanup: the new version is
-    /// readable and the un-reclaimed staging is harmless and reclaimable.
-    mod crash_consistency {
-        use super::*;
-        use crate::crash_inject::{self, CrashPoint};
-        use crate::storage_api_contracts::object::ObjectIO as _;
-        use http::HeaderMap;
-        use tokio::io::AsyncReadExt as _;
-
-        /// 1 MiB keeps every object off the 128 KiB inline fast path, so the
-        /// commit moves real erasure shards through `rename_data`.
-        const PART_SIZE: usize = 1 << 20;
-
-        fn payload(fill: u8) -> Vec<u8> {
-            vec![fill; PART_SIZE]
-        }
-
-        async fn make_bucket_on_all(disks: &[DiskStore], bucket: &str) {
-            for disk in disks {
-                disk.make_volume(bucket).await.expect("bucket volume should be created");
-            }
-        }
-
-        /// Stage a single-part multipart upload without completing it. A single
-        /// part is the last part, so the 5 MiB minimum-part-size gate does not
-        /// apply. Returns the upload id and the `CompletePart` list.
-        async fn stage_upload(
-            set_disks: &Arc<SetDisks>,
-            bucket: &str,
-            object: &str,
-            content: &[u8],
-        ) -> (String, Vec<CompletePart>) {
-            let upload = set_disks
-                .new_multipart_upload(bucket, object, &ObjectOptions::default())
-                .await
-                .expect("multipart upload should be created");
-            let mut reader = PutObjReader::new(
-                HashReader::from_stream(
-                    Cursor::new(content.to_vec()),
-                    content.len() as i64,
-                    content.len() as i64,
-                    None,
-                    None,
-                    false,
-                )
-                .expect("hash reader should be constructed"),
-            );
-            let part = set_disks
-                .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("uploading the part should succeed");
-            (
-                upload.upload_id,
-                vec![CompletePart {
-                    part_num: part.part_num,
-                    etag: part.etag,
-                    ..Default::default()
-                }],
-            )
-        }
-
-        async fn complete(
-            set_disks: &Arc<SetDisks>,
-            bucket: &str,
-            object: &str,
-            upload_id: &str,
-            parts: Vec<CompletePart>,
-        ) -> Result<ObjectInfo> {
-            set_disks
-                .clone()
-                .complete_multipart_upload(bucket, object, upload_id, parts, &ObjectOptions::default())
-                .await
-        }
-
-        /// Read the whole committed object back; returns `(body, etag)`. The full
-        /// body read also proves the served length matches (never a torn/short body).
-        async fn read_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> (Vec<u8>, Option<String>) {
-            let mut reader = set_disks
-                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
-                .await
-                .expect("object reader should open");
-            let etag = reader.object_info.etag.clone();
-            let mut body = Vec::new();
-            reader
-                .stream
-                .read_to_end(&mut body)
-                .await
-                .expect("object should stream fully");
-            (body, etag)
-        }
-
-        async fn upload_is_listed(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, upload_id: &str) -> bool {
-            let page = set_disks
-                .list_multipart_uploads(bucket, object, None, None, None, 1000)
-                .await
-                .expect("listing multipart uploads should succeed");
-            page.uploads.iter().any(|u| u.upload_id == upload_id)
-        }
-
-        #[tokio::test]
-        #[serial]
-        async fn pre_commit_crash_keeps_committed_old_version() {
-            let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-            let bucket = "multipart-crash-pre";
-            let object = "crash-pre-object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-
-            // Commit an OLD version through the multipart path.
-            let old = payload(0xA1);
-            let (u_old, parts_old) = stage_upload(&set_disks, bucket, object, &old).await;
-            complete(&set_disks, bucket, object, &u_old, parts_old)
-                .await
-                .expect("the old version should commit");
-            let (old_body, old_etag) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(old_body, old, "the old version must round-trip before the crash");
-
-            // Stage a replacement upload with NEW content, then crash before commit.
-            let new = payload(0xB2);
-            let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
-            crash_inject::arm(CrashPoint::MultipartBeforeCommitRename, object);
-            let crashed = complete(&set_disks, bucket, object, &u_new, parts_new.clone()).await;
-            assert!(
-                matches!(crashed, Err(StorageError::Unexpected)),
-                "the armed pre-commit crash point must be the failure that surfaced, got {crashed:?}"
-            );
-
-            // rustfs/backlog#864: the committed old version is untouched, byte-for-byte.
-            let (after_body, after_etag) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(after_body, old, "the interrupted completion must not mutate the committed old version");
-            assert_eq!(after_etag, old_etag, "the old version's ETag must be unchanged");
-
-            // The staged upload survived intact (part.N.meta present on quorum), so
-            // a retried completion commits the new version cleanly (retryable).
-            assert!(
-                total_part1_meta(&temp_dirs).await > 0,
-                "the un-committed upload must keep its staging parts for retry"
-            );
-            crash_inject::disarm(CrashPoint::MultipartBeforeCommitRename, object);
-            complete(&set_disks, bucket, object, &u_new, parts_new)
-                .await
-                .expect("a retried completion must succeed");
-            let (retry_body, retry_etag) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(retry_body, new, "the retried completion must publish the whole new version");
-            assert_ne!(retry_etag, old_etag, "the new version must carry a distinct ETag");
-        }
-
-        #[tokio::test]
-        #[serial]
-        async fn post_commit_crash_publishes_new_version_and_leaves_reclaimable_upload() {
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-            let bucket = "multipart-crash-post";
-            let object = "crash-post-object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-
-            // Stage an upload and crash AFTER the commit rename but BEFORE cleanup.
-            let new = payload(0xC3);
-            let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
-            let parts_retry = parts_new.clone();
-            crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
-            let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
-            assert!(
-                matches!(crashed, Err(StorageError::Unexpected)),
-                "the armed post-commit crash point must be the failure that surfaced, got {crashed:?}"
-            );
-            crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
-
-            // The commit landed: the new version reads back whole and correct.
-            let (body, _etag) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(body, new, "a post-commit crash must leave the whole new version readable");
-
-            // The completion never reached its cleanup, so the upload's staging
-            // directory is still listable — leftover that must be reclaimable.
-            assert!(
-                upload_is_listed(&set_disks, bucket, object, &u_new).await,
-                "the post-commit crash must leave the upload listable for reclamation"
-            );
-
-            // A retried CompleteMultipartUpload is answered deterministically: the
-            // commit rename already consumed the upload's metadata, so the retry
-            // resolves to InvalidUploadID (NoSuchUpload to the S3 client, the
-            // standard answer for a completed-then-retried upload) — never a torn
-            // state, and the committed object is untouched by the retry.
-            let retried = complete(&set_disks, bucket, object, &u_new, parts_retry).await;
-            assert!(
-                matches!(retried, Err(StorageError::InvalidUploadID(..))),
-                "a retried complete after the commit landed must resolve to InvalidUploadID, got {retried:?}"
-            );
-            let (body_after_retry, _) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(body_after_retry, new, "the failed retry must not disturb the committed object");
-
-            // Reclaim the leftover exactly as the production tail does: delete_all
-            // on the upload path (abort_multipart_upload cannot — the upload's
-            // metadata is gone, so it too answers InvalidUploadID).
-            let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
-            set_disks
-                .delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
-                .await
-                .expect("sweeping the leftover upload staging should succeed");
-            assert!(
-                !upload_is_listed(&set_disks, bucket, object, &u_new).await,
-                "reclaiming the upload must drop it from the listing"
-            );
-            let (body_after, _) = read_object(&set_disks, bucket, object).await;
-            assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
-        }
-    }
 }
@@ -23,6 +23,7 @@ use super::super::*;

 use crate::disk::OldCurrentSize;
 use crate::object_api::GetObjectBodySource;
+use tracing::Instrument as _;

 /// Length of the full plaintext body when — and only when — this read's output
 /// is exactly the object's complete plaintext, so the app-layer body cache may
@@ -740,6 +741,17 @@ impl SetDisks {

        let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());

+        let ec_write_span = tracing::info_span!(
+            "rustfs.ec.write",
+            event = "rustfs_ec_write",
+            component = "ecstore",
+            "rustfs.ec.data_shards" = data_drives,
+            "rustfs.ec.parity_shards" = parity_drives,
+            "rustfs.ec.write_quorum" = write_quorum,
+            "rustfs.ec.block_bytes" = fi.erasure.block_size,
+            "rustfs.distribution.targets" = shuffle_disks.len(),
+            "rustfs.distribution.transport" = "erasure"
+        );
        let result: Result<(ObjectInfo, Option<OldCurrentSize>)> = async {
            let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);

@@ -789,7 +801,15 @@ impl SetDisks {
                    }
                })
                .collect();
-            let writer_results = join_all(writer_futs).await;
+            let writer_results = join_all(writer_futs)
+                .instrument(tracing::info_span!(
+                    "rustfs.ec.distribute",
+                    event = "rustfs_ec_distribute",
+                    component = "ecstore",
+                    "rustfs.distribution.targets" = shuffle_disks.len(),
+                    "rustfs.distribution.transport" = "erasure"
+                ))
+                .await;
            let mut writers = Vec::with_capacity(writer_results.len());
            let mut errors = Vec::with_capacity(writer_results.len());
            for (w, e) in writer_results {
@@ -1139,6 +1159,7 @@ impl SetDisks {
                old_current_size,
            ))
        }
+        .instrument(ec_write_span)
        .await;

        if issue3031_diag_enabled()
@@ -4095,27 +4095,6 @@ mod tests {
        assert_eq!(&out[..n], [b"aaaa", b"bbbb", b"cccc", b"dddd"][fallback_index]);
    }

-    #[tokio::test]
-    async fn bitrot_reader_setup_skips_deferred_slots_without_a_source() {
-        let setup = setup_inline_bitrot_readers_with_env(
-            vec![Some(b"aaaa"), Some(b"bbbb"), None, Some(b"dddd")],
-            2,
-            2,
-            BitrotReaderSetupMode::ReadQuorum,
-            true,
-        )
-        .await;
-
-        assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum));
-        assert_eq!(setup.available_shards(), 2);
-        assert_eq!(setup.deferred_shards(), 1);
-        assert!(setup.readers[2].is_none(), "a slot without inline data or a disk has no usable source");
-        assert!(matches!(setup.errors[2], Some(DiskError::DiskNotFound)));
-        assert!(setup.deferred_stripe_handles[2].is_none());
-        assert!(setup.readers[3].is_some(), "an inline shard remains available as a deferred fallback");
-        assert!(setup.deferred_stripe_handles[3].is_some());
-    }
-
    #[tokio::test]
    async fn bitrot_reader_setup_preference_uses_data_blocks_first_without_env() {
        let setup = setup_inline_bitrot_readers_with_preference(
@@ -280,20 +280,14 @@ async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {

 async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
    let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
-    // `ObjectInfo.size` is the on-disk size. For SSE objects that is the
-    // DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
-    // deliberately larger than the logical object size. The size a client sees
-    // (and what MinIO records via `x-*-internal-actual-size`) comes from
-    // `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
-    // `size` field would never equal the plaintext length for encrypted objects.
-    let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
+    let object_size = object_info.size;
    let kms_key_b64 = minio_static_kms_key_b64();

    let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
        .await
        .expect("fixture must restore with the configured KMS key");

-    assert_eq!(decrypted_size, expected_size);
+    assert_eq!(object_size, expected_size);
    assert_eq!(plaintext.len(), expected_size as usize);
    assert_eq!(sha256_hex(&plaintext), expected_sha256);
 }
@@ -1,209 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Structured version-graph marshal -> load roundtrip property (backlog#1151
-//! sec-10).
-//!
-//! xl.meta roundtrip coverage was a fixed example test plus byte-level no-panic
-//! fuzz properties; nothing generated STRUCTURED version graphs, so a lossy
-//! encoding bug in the version header, ordering, delete-marker, or
-//! dual-internal-metadata-key handling would only be caught if the fixed
-//! example happened to hit it. These properties generate multi-version graphs
-//! (objects + delete markers, distinct and colliding mod_times, user metadata,
-//! and the AGENTS.md dual `x-rustfs-internal-*` / `x-minio-internal-*` keys)
-//! and assert `FileMeta::marshal_msg` -> `FileMeta::load` preserves them
-//! exactly: version count, order, headers, and fully decoded per-version
-//! content.
-
-use proptest::collection::vec;
-use proptest::prelude::*;
-use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMeta, FileMetaVersion, MetaDeleteMarker, MetaObject, VersionType};
-use rustfs_utils::http::insert_bytes;
-use std::collections::HashMap;
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-/// Internal metadata suffixes to write under BOTH internal prefixes (the
-/// cross-cutting dual-key invariant). Realistic suffixes used by production
-/// code paths.
-const SYS_SUFFIXES: &[&str] = &["replication-status", "transition-status", "tier-name"];
-
-#[derive(Debug, Clone)]
-struct GenVersion {
-    is_delete_marker: bool,
-    version_id: u128,
-    /// Seconds offset added to the base timestamp. Small range on purpose so
-    /// some versions share a mod_time and the tie-break ordering is exercised.
-    mod_offset: i64,
-    size: i64,
-    user_meta: Vec<(String, String)>,
-    sys_suffix_idx: usize,
-    sys_value: Vec<u8>,
-}
-
-fn gen_version_strategy() -> impl Strategy<Value = GenVersion> {
-    (
-        proptest::bool::weighted(0.3),
-        1u128..u128::MAX,
-        0i64..8,
-        1i64..(1 << 30),
-        vec(("[a-z]{1,8}", "[a-zA-Z0-9 ]{0,16}"), 0..3),
-        0usize..SYS_SUFFIXES.len(),
-        vec(any::<u8>(), 1..24),
-    )
-        .prop_map(
-            |(is_delete_marker, version_id, mod_offset, size, user_meta, sys_suffix_idx, sys_value)| GenVersion {
-                is_delete_marker,
-                version_id,
-                mod_offset,
-                size,
-                user_meta,
-                sys_suffix_idx,
-                sys_value,
-            },
-        )
-}
-
-const BASE_TS: i64 = 1_700_000_000;
-
-fn build_version(g: &GenVersion) -> FileMetaVersion {
-    let mod_time = OffsetDateTime::from_unix_timestamp(BASE_TS + g.mod_offset).expect("valid timestamp");
-    let mut meta_sys = HashMap::new();
-    // Dual internal metadata keys: insert_bytes writes BOTH the
-    // x-rustfs-internal-* and x-minio-internal-* forms.
-    insert_bytes(&mut meta_sys, SYS_SUFFIXES[g.sys_suffix_idx], g.sys_value.clone());
-
-    if g.is_delete_marker {
-        FileMetaVersion {
-            version_type: VersionType::Delete,
-            delete_marker: Some(MetaDeleteMarker {
-                version_id: Some(Uuid::from_u128(g.version_id)),
-                mod_time: Some(mod_time),
-                meta_sys,
-            }),
-            write_version: 0,
-            ..Default::default()
-        }
-    } else {
-        FileMetaVersion {
-            version_type: VersionType::Object,
-            object: Some(MetaObject {
-                version_id: Some(Uuid::from_u128(g.version_id)),
-                data_dir: Some(Uuid::from_u128(g.version_id.wrapping_add(1).max(1))),
-                erasure_algorithm: ErasureAlgo::ReedSolomon,
-                erasure_m: 2,
-                erasure_n: 2,
-                erasure_block_size: 1 << 20,
-                erasure_index: 1,
-                erasure_dist: vec![1, 2, 3, 4],
-                bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
-                part_numbers: vec![1],
-                part_etags: vec![],
-                part_sizes: vec![g.size as usize],
-                part_actual_sizes: vec![g.size],
-                part_indices: vec![],
-                size: g.size,
-                mod_time: Some(mod_time),
-                meta_sys,
-                meta_user: g.user_meta.iter().cloned().collect(),
-            }),
-            write_version: 0,
-            ..Default::default()
-        }
-    }
-}
-
-proptest! {
-    /// A generated version graph survives marshal -> load exactly: count,
-    /// order, headers, and fully decoded version content (objects, delete
-    /// markers, user metadata, and both internal metadata keys).
-    #[test]
-    fn version_graph_marshal_load_roundtrip(gens in vec(gen_version_strategy(), 1..8)) {
-        let mut fm = FileMeta::new();
-        for g in &gens {
-            fm.add_version_filemata(build_version(g)).expect("add generated version");
-        }
-
-        let encoded = fm.marshal_msg().expect("marshal generated version graph");
-        let loaded = FileMeta::load(&encoded).expect("load marshaled version graph");
-
-        // Count and metadata version survive.
-        prop_assert_eq!(loaded.versions.len(), fm.versions.len(), "version count must survive the roundtrip");
-        prop_assert_eq!(loaded.meta_ver, fm.meta_ver, "meta_ver must survive the roundtrip");
-
-        // Version ORDER survives exactly: add_version_filemata sorted the
-        // in-memory graph (mod_time desc with documented tie-breaks, delete
-        // markers interleaved); load must reproduce that same sequence, not
-        // re-derive a different one.
-        let pre: Vec<_> = fm.versions.iter().map(|v| (v.header.version_id, v.header.version_type.clone())).collect();
-        let post: Vec<_> = loaded.versions.iter().map(|v| (v.header.version_id, v.header.version_type.clone())).collect();
-        prop_assert_eq!(pre, post, "version (id, type) sequence must survive the roundtrip");
-
-        for (orig, back) in fm.versions.iter().zip(loaded.versions.iter()) {
-            // Full header equality (mod_time, flags, signature, ec fields).
-            prop_assert_eq!(&orig.header, &back.header, "version header must survive the roundtrip");
-
-            // Fully decoded version content equality.
-            let orig_v = orig.parse_version_meta().expect("decode original version");
-            let back_v = back.parse_version_meta().expect("decode roundtripped version");
-            prop_assert_eq!(&orig_v, &back_v, "decoded version content must survive the roundtrip");
-
-            // Delete-marker semantics preserved.
-            if back.header.version_type == VersionType::Delete {
-                prop_assert!(back_v.delete_marker.is_some(), "delete marker payload must survive");
-                prop_assert!(back_v.object.is_none(), "a delete marker must not grow an object payload");
-            } else {
-                prop_assert!(back_v.object.is_some(), "object payload must survive");
-            }
-
-            // Dual internal metadata keys: BOTH prefixed forms survive with the
-            // same value (losing either breaks MinIO interop; losing both loses
-            // replication/tier state).
-            let meta_sys = match (&back_v.object, &back_v.delete_marker) {
-                (Some(o), _) => &o.meta_sys,
-                (_, Some(d)) => &d.meta_sys,
-                _ => unreachable!("version decoded above"),
-            };
-            let orig_sys = match (&orig_v.object, &orig_v.delete_marker) {
-                (Some(o), _) => &o.meta_sys,
-                (_, Some(d)) => &d.meta_sys,
-                _ => unreachable!("version decoded above"),
-            };
-            for (k, v) in orig_sys {
-                prop_assert_eq!(
-                    meta_sys.get(k).map(|b| b.as_slice()),
-                    Some(v.as_slice()),
-                    "internal metadata key {} must survive the roundtrip with an identical value",
-                    k
-                );
-            }
-        }
-    }
-
-    /// Idempotence: a second marshal of the loaded graph is byte-identical.
-    /// Catches encoders that mutate or reorder state on the way out (a lossy
-    /// first pass shows up as a divergent second pass).
-    #[test]
-    fn version_graph_marshal_is_idempotent(gens in vec(gen_version_strategy(), 1..6)) {
-        let mut fm = FileMeta::new();
-        for g in &gens {
-            fm.add_version_filemata(build_version(g)).expect("add generated version");
-        }
-
-        let first = fm.marshal_msg().expect("first marshal");
-        let loaded = FileMeta::load(&first).expect("load first marshal");
-        let second = loaded.marshal_msg().expect("second marshal");
-        prop_assert_eq!(first, second, "marshal(load(marshal(x))) must be byte-identical to marshal(x)");
-    }
-}
@@ -48,7 +48,6 @@ base64 = { workspace = true }

 [dev-dependencies]
 serde_json = { workspace = true }
-rustfs-test-utils = { workspace = true }
 serial_test = { workspace = true }
 tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
@@ -99,54 +99,6 @@ impl Error {
    pub fn transient_skip(message: impl Into<String>) -> Self {
        Error::TransientSkip { message: message.into() }
    }
-
-    /// Whether a heal operation can be retried without changing its inputs.
-    pub(crate) fn is_recoverable_heal(&self) -> bool {
-        match self {
-            Error::TaskCancelled => false,
-            Error::TaskTimeout | Error::TransientSkip { .. } => true,
-            Error::Storage(err) => {
-                err.is_quorum_error()
-                    || matches!(err, EcstoreError::SlowDown | EcstoreError::OperationCanceled | EcstoreError::Lock(_))
-                    || is_recoverable_heal_error_message(&err.to_string())
-            }
-            Error::Disk(err) => {
-                matches!(
-                    err,
-                    DiskError::ErasureReadQuorum
-                        | DiskError::ErasureWriteQuorum
-                        | DiskError::Timeout
-                        | DiskError::SourceStalled
-                        | DiskError::FaultyRemoteDisk
-                        | DiskError::FaultyDisk
-                ) || is_recoverable_heal_error_message(&err.to_string())
-            }
-            Error::TaskExecutionFailed { message } | Error::IO(message) | Error::Other(message) => {
-                is_recoverable_heal_error_message(message)
-            }
-            Error::Io(err) => is_recoverable_heal_error_message(&err.to_string()),
-            _ => false,
-        }
-    }
-}
-
-fn is_recoverable_heal_error_message(error: &str) -> bool {
-    let error = error.to_ascii_lowercase();
-    [
-        "failed to acquire read lock",
-        "lock acquisition failed",
-        "lock acquisition timeout",
-        "remote lock rpc timed out",
-        "deadline has elapsed",
-        "timed out",
-        "transport error",
-        "network error",
-        "connection refused",
-        "operation canceled",
-        "quorum not reached",
-    ]
-    .iter()
-    .any(|pattern| error.contains(pattern))
 }

 impl From<Error> for std::io::Error {
@@ -34,7 +34,7 @@ use tokio::{
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};

-use super::{DiskError, HealDiskExt as _, local_disk_map_read};
+use super::{DiskError, EcstoreError, HealDiskExt as _, local_disk_map_read};

 const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
 const LOG_COMPONENT_HEAL: &str = "heal";
@@ -565,12 +565,9 @@ fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(Hea
    if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
        return None;
    }
-    if task.has_batch_failure() {
-        return None;
-    }

    let error = err.to_string();
-    if !err.is_recoverable_heal() {
+    if !is_recoverable_heal_error(err, &error) {
        return None;
    }

@@ -585,6 +582,54 @@ fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
    delay.min(MAX_RECOVERABLE_HEAL_RETRY_DELAY)
 }

+fn is_recoverable_heal_error(err: &Error, error: &str) -> bool {
+    match err {
+        Error::TaskCancelled => false,
+        Error::TaskTimeout | Error::TransientSkip { .. } => true,
+        Error::Storage(err) => is_recoverable_storage_heal_error(err) || is_recoverable_heal_error_message(error),
+        Error::Disk(err) => is_recoverable_disk_heal_error(err) || is_recoverable_heal_error_message(error),
+        Error::TaskExecutionFailed { .. } | Error::Io(_) | Error::IO(_) | Error::Other(_) => {
+            is_recoverable_heal_error_message(error)
+        }
+        _ => false,
+    }
+}
+
+fn is_recoverable_storage_heal_error(err: &EcstoreError) -> bool {
+    err.is_quorum_error() || matches!(err, EcstoreError::SlowDown | EcstoreError::OperationCanceled | EcstoreError::Lock(_))
+}
+
+fn is_recoverable_disk_heal_error(err: &DiskError) -> bool {
+    matches!(
+        err,
+        DiskError::ErasureReadQuorum
+            | DiskError::ErasureWriteQuorum
+            | DiskError::Timeout
+            | DiskError::SourceStalled
+            | DiskError::FaultyRemoteDisk
+            | DiskError::FaultyDisk
+    )
+}
+
+fn is_recoverable_heal_error_message(error: &str) -> bool {
+    let error = error.to_ascii_lowercase();
+    [
+        "failed to acquire read lock",
+        "lock acquisition failed",
+        "lock acquisition timeout",
+        "remote lock rpc timed out",
+        "deadline has elapsed",
+        "timed out",
+        "transport error",
+        "network error",
+        "connection refused",
+        "operation canceled",
+        "quorum not reached",
+    ]
+    .iter()
+    .any(|pattern| error.contains(pattern))
+}
+
 /// Heal config
 #[derive(Debug, Clone)]
 pub struct HealConfig {
@@ -2872,9 +2917,8 @@ fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String,
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::heal::EcstoreError;
    use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
-    use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
+    use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
    use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
    use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
    use rustfs_madmin::heal_commands::HealResultItem;
@@ -3634,24 +3678,6 @@ mod tests {
        assert!(retry_request_for_result(&task, &result).is_none());
    }

-    #[tokio::test]
-    async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() {
-        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-        let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
-        let result = Err(task
-            .record_batch_failure(BatchHealFailure {
-                scope: "bucket:bucket".to_string(),
-                failed: 1,
-                retryable: 1,
-                permanent: 0,
-                first_object: "object".to_string(),
-                first_error: "Lock acquisition timeout".to_string(),
-            })
-            .await);
-
-        assert!(retry_request_for_result(&task, &result).is_none());
-    }
-
    #[test]
    fn test_heal_type_matches_path_normalizes_prefix_trailing_slash() {
        let heal_type = HealType::Prefix {
@@ -25,10 +25,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use serde::{Deserialize, Serialize};
 use std::{
    future::Future,
-    sync::{
-        Arc,
-        atomic::{AtomicBool, Ordering},
-    },
+    sync::Arc,
    time::{Duration, Instant, SystemTime},
 };
 use tokio::sync::RwLock;
@@ -44,8 +41,6 @@ const EVENT_HEAL_TASK_STATE: &str = "heal_task_state";
 const EVENT_HEAL_OBJECT_STAGE: &str = "heal_object_stage";
 const EVENT_HEAL_OBJECT_MISSING: &str = "heal_object_missing";
 const EVENT_HEAL_OBJECT_RESULT: &str = "heal_object_result";
-const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3;
-const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5;
 const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage";
 const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result";
 const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage";
@@ -186,26 +181,6 @@ pub enum HealTaskStatus {
    Timeout,
 }

-#[derive(Debug)]
-pub(crate) struct BatchHealFailure {
-    pub(crate) scope: String,
-    pub(crate) failed: u64,
-    pub(crate) retryable: u64,
-    pub(crate) permanent: u64,
-    pub(crate) first_object: String,
-    pub(crate) first_error: String,
-}
-
-impl std::fmt::Display for BatchHealFailure {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "Heal batch failed for {}: {} failed ({} retryable, {} permanent); first failure at {}: {}",
-            self.scope, self.failed, self.retryable, self.permanent, self.first_object, self.first_error
-        )
-    }
-}
-
 /// Heal request
 #[derive(Debug, Clone)]
 pub struct HealRequest {
@@ -306,8 +281,6 @@ pub struct HealTask {
    pub progress: Arc<RwLock<HealProgress>>,
    /// Result items collected from storage heal calls.
    pub result_items: Arc<RwLock<Vec<HealResultItem>>>,
-    batch_failure: Arc<RwLock<Option<BatchHealFailure>>>,
-    batch_failure_recorded: Arc<AtomicBool>,
    /// Created time
    pub created_at: SystemTime,
    /// Queue admission time
@@ -337,8 +310,6 @@ impl HealTask {
            status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
            progress: Arc::new(RwLock::new(HealProgress::new())),
            result_items: Arc::new(RwLock::new(Vec::new())),
-            batch_failure: Arc::new(RwLock::new(None)),
-            batch_failure_recorded: Arc::new(AtomicBool::new(false)),
            created_at: request.created_at,
            enqueued_at: request.enqueued_at,
            started_at: Arc::new(RwLock::new(None)),
@@ -377,21 +348,6 @@ impl HealTask {
        }
    }

-    pub(crate) fn has_batch_failure(&self) -> bool {
-        self.batch_failure_recorded.load(Ordering::Acquire)
-    }
-
-    pub(crate) async fn record_batch_failure(&self, failure: BatchHealFailure) -> Error {
-        self.batch_failure_recorded.store(true, Ordering::Release);
-        let message = failure.to_string();
-        *self.batch_failure.write().await = Some(failure);
-        Error::TaskExecutionFailed { message }
-    }
-
-    async fn take_batch_failure(&self) -> Option<BatchHealFailure> {
-        self.batch_failure.write().await.take()
-    }
-
    pub fn metric_set_label(&self) -> String {
        match &self.heal_type {
            HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
@@ -521,15 +477,6 @@ impl HealTask {
        }
    }

-    fn bucket_object_retry_delay(&self, retry_attempt: u32) -> Duration {
-        let base = Duration::from_secs(2_u64.saturating_pow(retry_attempt.clamp(1, MAX_BUCKET_OBJECT_HEAL_RETRIES)));
-        let jitter_seed = self
-            .id
-            .bytes()
-            .fold(0_u64, |acc, byte| acc.wrapping_mul(31).wrapping_add(u64::from(byte)));
-        Duration::from_millis(jitter_seed % 500).saturating_add(base)
-    }
-
    fn should_return_typed_heal_error(err: &Error) -> bool {
        matches!(err, Error::Storage(_) | Error::Disk(_))
    }
@@ -1316,61 +1263,9 @@ impl HealTask {
        );

        let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
-        let mut failed = 0_u64;
-        let mut retryable = 0_u64;
-        let mut permanent = 0_u64;
-        let mut first_object = None;
-        let mut first_error = None;
        for bucket_info in bucket_infos {
            self.check_control_flags().await?;
-            let mut retry_attempt = 0_u32;
-            loop {
-                match self.heal_bucket(&bucket_info.name).await {
-                    Ok(()) => break,
-                    Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
-                    Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
-                    Err(err) => {
-                        if let Some(failure) = self.take_batch_failure().await {
-                            failed = failed.saturating_add(failure.failed);
-                            retryable = retryable.saturating_add(failure.retryable);
-                            permanent = permanent.saturating_add(failure.permanent);
-                            first_object.get_or_insert(failure.first_object);
-                            first_error.get_or_insert(failure.first_error);
-                            break;
-                        }
-                        if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
-                            retry_attempt = retry_attempt.saturating_add(1);
-                            self.await_with_control(async {
-                                tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
-                                Ok(())
-                            })
-                            .await?;
-                            continue;
-                        }
-                        failed = failed.saturating_add(1);
-                        if err.is_recoverable_heal() {
-                            retryable = retryable.saturating_add(1);
-                        } else {
-                            permanent = permanent.saturating_add(1);
-                        }
-                        first_object.get_or_insert(bucket_info.name.clone());
-                        first_error.get_or_insert_with(|| err.to_string());
-                        break;
-                    }
-                }
-            }
-        }
-
-        if failed > 0 {
-            let failure = BatchHealFailure {
-                scope: "cluster".to_string(),
-                failed,
-                retryable,
-                permanent,
-                first_object: first_object.unwrap_or_default(),
-                first_error: first_error.unwrap_or_default(),
-            };
-            return Err(self.record_batch_failure(failure).await);
+            self.heal_bucket(&bucket_info.name).await?;
        }

        Ok(())
@@ -1397,12 +1292,7 @@ impl HealTask {
        let mut scanned = 0u64;
        let mut healed = 0u64;
        let mut failed = 0u64;
-        let mut retryable_failed = 0u64;
-        let mut permanent_failed = 0u64;
        let mut bytes = 0u64;
-        let mut first_failed_object = None;
-        let mut first_error = None;
-        let mut failure_samples_logged = 0_u64;

        let heal_opts = HealOpts {
            recursive: false,
@@ -1425,44 +1315,34 @@ impl HealTask {
                )
                .await?;

-            let mut pending = objects;
-            let mut retry_attempt = 0_u32;
-            while !pending.is_empty() {
-                if retry_attempt > 0 {
-                    self.await_with_control(async {
-                        tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
-                        Ok(())
-                    })
-                    .await?;
+            for item in objects {
+                self.check_control_flags().await?;
+                let object = item.name.as_str();
+                scanned += 1;
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.set_current_object(Some(format!("{bucket}/{object}")));
+                    progress.update_progress(scanned, healed, failed, bytes);
                }
-                let mut retry = Vec::with_capacity(pending.len());
-                for item in pending {
-                    self.check_control_flags().await?;
-                    let object = item.name.as_str();
-                    if retry_attempt == 0 {
-                        scanned = scanned.saturating_add(1);
-                    }
-                    {
-                        let mut progress = self.progress.write().await;
-                        progress.set_current_object(Some(format!("{bucket}/{object}")));
-                        progress.update_progress(scanned, healed, failed, bytes);
-                    }

-                    let error = match self
-                        .await_with_control(
-                            self.storage
-                                .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
-                        )
-                        .await
-                    {
-                        Ok((result, None)) => {
-                            healed = healed.saturating_add(1);
-                            bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
-                            self.record_result_item(result).await;
-                            None
-                        }
-                        Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
-                            healed = healed.saturating_add(1);
+                // Heal each enumerated version directly. There is no latest-only
+                // existence gate: genuine absence surfaces through heal_object's
+                // not-found result and is handled below.
+                match self
+                    .await_with_control(
+                        self.storage
+                            .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
+                    )
+                    .await
+                {
+                    Ok((result, None)) => {
+                        healed += 1;
+                        bytes = bytes.saturating_add(result.object_size as u64);
+                        self.record_result_item(result).await;
+                    }
+                    Ok((_, Some(err))) => {
+                        if is_missing_object_dir_heal_result(object, &err) {
+                            healed += 1;
                            debug!(
                                target: "rustfs::heal::task",
                                event = EVENT_HEAL_BUCKET_RESULT,
@@ -1470,16 +1350,12 @@ impl HealTask {
                                subsystem = LOG_SUBSYSTEM_TASK,
                                task_id = %self.id,
                                bucket,
-                                object,
+                                object = %object,
                                result = "object_dir_not_found_skipped",
                                "Heal bucket object-dir candidate skipped after not-found result"
                            );
-                            None
+                            continue;
                        }
-                        Ok((_, Some(err))) | Err(err) => Some(err),
-                    };
-
-                    if let Some(err) = error {
                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
                            warn!(
                                target: "rustfs::heal::task",
@@ -1488,59 +1364,63 @@ impl HealTask {
                                subsystem = LOG_SUBSYSTEM_TASK,
                                task_id = %self.id,
                                bucket,
-                                object,
+                                object = %object,
                                result = "transient_skip",
                                error = %err,
-                                "Heal bucket object repair skipped due to transient metadata error"
+                                "Heal bucket object repair skipped due to transient error"
                            );
-                        } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
-                            debug!(
+                        } else {
+                            failed += 1;
+                            warn!(
                                target: "rustfs::heal::task",
                                event = EVENT_HEAL_BUCKET_RESULT,
                                component = LOG_COMPONENT_HEAL,
                                subsystem = LOG_SUBSYSTEM_TASK,
                                task_id = %self.id,
                                bucket,
-                                object,
-                                retry_attempt = retry_attempt.saturating_add(1),
+                                object = %object,
+                                result = "object_failed",
                                error = %err,
-                                result = "object_retry_scheduled",
-                                "Heal bucket object retry scheduled"
+                                "Heal bucket object repair failed"
                            );
-                            retry.push(item);
-                        } else {
-                            failed = failed.saturating_add(1);
-                            if err.is_recoverable_heal() {
-                                retryable_failed = retryable_failed.saturating_add(1);
-                            } else {
-                                permanent_failed = permanent_failed.saturating_add(1);
-                            }
-                            first_failed_object.get_or_insert_with(|| object.to_string());
-                            first_error.get_or_insert_with(|| err.to_string());
-                            if failure_samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
-                                failure_samples_logged = failure_samples_logged.saturating_add(1);
-                                warn!(
-                                    target: "rustfs::heal::task",
-                                    event = EVENT_HEAL_BUCKET_RESULT,
-                                    component = LOG_COMPONENT_HEAL,
-                                    subsystem = LOG_SUBSYSTEM_TASK,
-                                    task_id = %self.id,
-                                    bucket,
-                                    object,
-                                    retry_attempt,
-                                    error = %err,
-                                    result = "object_failed",
-                                    "Heal bucket object repair failed"
-                                );
-                            }
                        }
                    }
+                    Err(err) => {
+                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                            warn!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object = %object,
+                                result = "transient_skip",
+                                error = %err,
+                                "Heal bucket object repair skipped due to transient error"
+                            );
+                        } else {
+                            failed += 1;
+                            warn!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object = %object,
+                                result = "object_failed",
+                                error = %err,
+                                "Heal bucket object repair failed"
+                            );
+                        }
+                    }
+                }

+                {
                    let mut progress = self.progress.write().await;
                    progress.update_progress(scanned, healed, failed, bytes);
                }
-                pending = retry;
-                retry_attempt = retry_attempt.saturating_add(1);
            }

            if !is_truncated {
@@ -1555,15 +1435,9 @@ impl HealTask {
        }

        if failed > 0 {
-            let failure = BatchHealFailure {
-                scope: format!("bucket:{bucket}"),
-                failed,
-                retryable: retryable_failed,
-                permanent: permanent_failed,
-                first_object: first_failed_object.unwrap_or_default(),
-                first_error: first_error.unwrap_or_default(),
-            };
-            return Err(self.record_batch_failure(failure).await);
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Failed to heal {failed} object(s) in bucket {bucket}"),
+            });
        }

        debug!(
@@ -2349,7 +2223,7 @@ mod tests {
    use super::*;
    use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo};
    use rustfs_madmin::heal_commands::HealResultItem;
-    use std::collections::{HashMap, VecDeque};
+    use std::collections::HashMap;
    use std::sync::Mutex;

    use super::super::storage_api::status::BucketInfo;
@@ -2364,15 +2238,11 @@ mod tests {
        object_exists: Mutex<Option<bool>>,
        object_exists_by_name: Mutex<HashMap<String, MockObjectExists>>,
        heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
-        heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
        deleted_objects: Mutex<Vec<String>>,
        format_no_heal_required: Mutex<bool>,
        listed_prefixes: Mutex<Vec<String>>,
        truncate_without_token: Mutex<bool>,
        include_object_dir_candidate: Mutex<bool>,
-        listed_buckets: Mutex<Option<Vec<String>>>,
-        bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
-        bucket_heal_calls: Mutex<Vec<String>>,
    }

    /// Build a latest, non-delete-marker heal list item with no version id.
@@ -2387,8 +2257,6 @@ mod tests {
    enum MockHealObjectOutcome {
        OkWithOtherError(&'static str),
        ErrOther(&'static str),
-        RetryableReadQuorum,
-        PermanentOther(&'static str),
    }

    #[derive(Clone, Copy)]
@@ -2458,19 +2326,10 @@ mod tests {
        }

        async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
-            let buckets = self
-                .listed_buckets
-                .lock()
-                .unwrap()
-                .clone()
-                .unwrap_or_else(|| vec!["bucket-a".to_string()]);
-            Ok(buckets
-                .into_iter()
-                .map(|name| BucketInfo {
-                    name,
-                    ..Default::default()
-                })
-                .collect())
+            Ok(vec![BucketInfo {
+                name: "bucket-a".to_string(),
+                ..Default::default()
+            }])
        }

        async fn object_exists(&self, _bucket: &str, object: &str) -> Result<bool> {
@@ -2505,37 +2364,12 @@ mod tests {
                .unwrap()
                .push(version_id.map(ToString::to_string));
            self.object_heal_opts.lock().unwrap().push(*opts);
-            if let Some(outcome) = self
-                .heal_object_outcomes
-                .lock()
-                .unwrap()
-                .get_mut(object)
-                .and_then(VecDeque::pop_front)
-            {
-                return match outcome {
-                    MockHealObjectOutcome::RetryableReadQuorum => Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
-                        bucket.to_string(),
-                        object.to_string(),
-                    ))),
-                    MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
-                    MockHealObjectOutcome::OkWithOtherError(message) => {
-                        Ok((HealResultItem::default(), Some(Error::other(message))))
-                    }
-                    MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
-                };
-            }
            if let Some(outcome) = self.heal_object_outcome.lock().unwrap().take() {
                return match outcome {
                    MockHealObjectOutcome::OkWithOtherError(message) => {
                        Ok((HealResultItem::default(), Some(Error::other(message))))
                    }
-                    MockHealObjectOutcome::ErrOther(message) | MockHealObjectOutcome::PermanentOther(message) => {
-                        Err(Error::other(message))
-                    }
-                    MockHealObjectOutcome::RetryableReadQuorum => Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
-                        bucket.to_string(),
-                        object.to_string(),
-                    ))),
+                    MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
                };
            }
            if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
@@ -2559,18 +2393,8 @@ mod tests {
            ))
        }

-        async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
-            self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
+        async fn heal_bucket(&self, _bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
            self.bucket_heal_opts.lock().unwrap().push(*opts);
-            if let Some(message) = self
-                .bucket_heal_errors
-                .lock()
-                .unwrap()
-                .get_mut(bucket)
-                .and_then(VecDeque::pop_front)
-            {
-                return Err(Error::other(message));
-            }
            Ok(HealResultItem::default())
        }

@@ -2746,152 +2570,6 @@ mod tests {
        assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
    }

-    #[tokio::test(start_paused = true)]
-    async fn test_recursive_bucket_heal_retries_only_retryable_objects() {
-        let storage = Arc::new(MockStorage::default());
-        storage
-            .heal_object_outcomes
-            .lock()
-            .unwrap()
-            .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
-        let request = HealRequest::new(
-            HealType::Bucket {
-                bucket: "bucket-a".to_string(),
-            },
-            HealOptions {
-                recursive: true,
-                timeout: None,
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        let task = HealTask::from_request(request, storage.clone());
-
-        task.heal_bucket("bucket-a")
-            .await
-            .expect("retryable object failure should be retried within the listing page");
-
-        assert_eq!(
-            storage.heal_object_calls.lock().unwrap().as_slice(),
-            ["object-a".to_string(), "object-b".to_string(), "object-a".to_string()]
-        );
-        let progress = task.get_progress().await;
-        assert_eq!(progress.objects_scanned, 2);
-        assert_eq!(progress.objects_healed, 2);
-        assert_eq!(progress.objects_failed, 0);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failures() {
-        let storage = Arc::new(MockStorage::default());
-        storage.heal_object_outcomes.lock().unwrap().insert(
-            "object-a".to_string(),
-            VecDeque::from([
-                MockHealObjectOutcome::RetryableReadQuorum,
-                MockHealObjectOutcome::RetryableReadQuorum,
-                MockHealObjectOutcome::RetryableReadQuorum,
-                MockHealObjectOutcome::RetryableReadQuorum,
-            ]),
-        );
-        storage.heal_object_outcomes.lock().unwrap().insert(
-            "object-b".to_string(),
-            VecDeque::from([MockHealObjectOutcome::PermanentOther("invalid metadata")]),
-        );
-        let request = HealRequest::new(
-            HealType::Bucket {
-                bucket: "bucket-a".to_string(),
-            },
-            HealOptions {
-                recursive: true,
-                timeout: None,
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        let task = HealTask::from_request(request, storage.clone());
-
-        let err = task
-            .heal_bucket("bucket-a")
-            .await
-            .expect_err("exhausted retryable and permanent failures must fail the task");
-
-        assert!(matches!(err, Error::TaskExecutionFailed { .. }));
-        let failure = task
-            .take_batch_failure()
-            .await
-            .expect("batch failure details should be retained on the task");
-        assert_eq!(failure.failed, 2);
-        assert_eq!(failure.retryable, 1);
-        assert_eq!(failure.permanent, 1);
-        assert_eq!(failure.first_object, "object-b");
-        let calls = storage.heal_object_calls.lock().unwrap();
-        assert_eq!(calls.iter().filter(|object| object.as_str() == "object-a").count(), 4);
-        assert_eq!(calls.iter().filter(|object| object.as_str() == "object-b").count(), 1);
-    }
-
-    #[tokio::test]
-    async fn test_cluster_heal_continues_after_bucket_failure() {
-        let storage = Arc::new(MockStorage {
-            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
-            ..Default::default()
-        });
-        let request = HealRequest::new(
-            HealType::Cluster,
-            HealOptions {
-                recursive: true,
-                timeout: None,
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        let task = HealTask::from_request(request, storage.clone());
-
-        let err = task.execute().await.expect_err("cluster task must report the failed bucket");
-
-        assert!(matches!(err, Error::TaskExecutionFailed { .. }));
-        let failure = task
-            .take_batch_failure()
-            .await
-            .expect("cluster failure details should be retained on the task");
-        assert_eq!(failure.failed, 1);
-        assert_eq!(
-            storage.bucket_heal_calls.lock().unwrap().as_slice(),
-            ["bucket-a".to_string(), "bucket-b".to_string()]
-        );
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn test_cluster_heal_retries_only_recoverable_bucket() {
-        let storage = Arc::new(MockStorage {
-            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-            bucket_heal_errors: Mutex::new(HashMap::from([(
-                "bucket-a".to_string(),
-                VecDeque::from(["lock acquisition timeout"]),
-            )])),
-            ..Default::default()
-        });
-        let request = HealRequest::new(
-            HealType::Cluster,
-            HealOptions {
-                recursive: true,
-                timeout: None,
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        let task = HealTask::from_request(request, storage.clone());
-
-        task.execute()
-            .await
-            .expect("cluster task should retry a recoverable bucket failure");
-
-        assert_eq!(
-            storage.bucket_heal_calls.lock().unwrap().as_slice(),
-            ["bucket-a".to_string(), "bucket-a".to_string(), "bucket-b".to_string()]
-        );
-    }
-
    #[tokio::test]
    async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
        let storage = Arc::new(MockStorage::default());
@@ -18,7 +18,7 @@
 //!
 //! These drive the REAL `ECStoreHealStorage` (not a mock) against a real 4-disk
 //! `ECStore`, mirroring `heal_integration_test.rs`. Every test is `#[serial]`
-//! and re-runs the full `heal_env` init (which sets the process-global
+//! and re-runs the full `setup_test_env` init (which sets the process-global
 //! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs
 //! in its own process so the OnceCell never collides.

@@ -34,15 +34,20 @@ use rustfs_heal::heal::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::Arc,
+    sync::{Arc, Once},
    time::Duration,
 };
+use tokio::fs;
+use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};
+use storage_api::integration::{
+    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
+    ObjectOperations as _, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
+};

 /// 256 KiB + change: large enough to be stored as non-inline erasure shards
 /// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -57,16 +62,78 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
        .collect()
 }

-/// Build a real 4-disk `ECStore` + `ECStoreHealStorage` via the shared
-/// rustfs-test-utils environment (backlog#1153 infra-1). Mirrors
-/// `heal_integration_test::heal_env`.
-async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    let env = rustfs_test_utils::TestECStoreEnv::builder()
-        .prefix("rustfs_heal_b5_test")
-        .build()
-        .await;
-    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
-    (env.disk_paths, env.ecstore, heal_storage)
+static INIT: Once = Once::new();
+
+fn init_tracing() {
+    INIT.call_once(|| {
+        let _ = tracing_subscriber::fmt()
+            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+            .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
+            .with_thread_names(true)
+            .try_init();
+    });
+}
+
+/// Build a real 4-disk `ECStore` + `ECStoreHealStorage`. Mirrors
+/// `heal_integration_test::setup_test_env`.
+async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    init_tracing();
+
+    let test_base_dir = format!("/tmp/rustfs_heal_b5_test_{}", uuid::Uuid::new_v4());
+    let temp_dir = PathBuf::from(&test_base_dir);
+    if temp_dir.exists() {
+        fs::remove_dir_all(&temp_dir).await.ok();
+    }
+    fs::create_dir_all(&temp_dir).await.unwrap();
+
+    let disk_paths = vec![
+        temp_dir.join("disk1"),
+        temp_dir.join("disk2"),
+        temp_dir.join("disk3"),
+        temp_dir.join("disk4"),
+    ];
+    for disk_path in &disk_paths {
+        fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: 4,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
+
+    init_local_disks(endpoint_pools.clone()).await.unwrap();
+
+    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();
+
+    let buckets_list = ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+
+    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+    (disk_paths, ecstore, heal_storage)
 }

 /// Create a bucket with S3 versioning ENABLED at creation time. Without this the
@@ -241,7 +308,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_enumeration_includes_all_versions_and_delete_marker_real_fixture() {
-        let (_disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-enum-versions";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -278,7 +345,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_old_nonlatest_version_after_disk_wipe() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-old-version-heal";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -330,7 +397,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_delete_marker_latest_enumerated_and_healed() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-dm-latest-heal";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -396,7 +463,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_version_id_normalization_null_and_unversioned_real_fixture() {
-        let (_disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-unversioned-normalize";
        create_unversioned_bucket(&ecstore, bucket).await;

@@ -432,7 +499,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_unversioned_bucket_e2e() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-unversioned-e2e";
        create_unversioned_bucket(&ecstore, bucket).await;

@@ -517,7 +584,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_resume_across_page_boundary_e2e() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
        let bucket = "b5-resume-e2e";
        create_versioned_bucket(&ecstore, bucket).await;

@@ -29,13 +29,18 @@ use rustfs_heal::heal::storage::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::Arc,
+    sync::{Arc, Once},
 };
+use tokio::fs;
+use tokio_util::sync::CancellationToken;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _};
+use storage_api::integration::{
+    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
+    PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
+};

 /// 256 KiB + change: large enough to be stored as non-inline erasure shards, so
 /// deleting the `xl.meta` file does NOT delete the data (the `part.*` shards live
@@ -52,6 +57,16 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
        .collect()
 }

+static INIT: Once = Once::new();
+
+fn init_tracing() {
+    INIT.call_once(|| {
+        let _ = tracing_subscriber::fmt()
+            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+            .try_init();
+    });
+}
+
 /// Disable the dangling-delete grace window so the destructive path is genuinely
 /// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
 /// be dangling-deleted here. With grace at its 1h default the delete path would be
@@ -63,16 +78,60 @@ fn disable_dangling_grace() {
    }
 }

-/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
-/// shared rustfs-test-utils environment (backlog#1153 infra-1).
-async fn heal_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    let env = rustfs_test_utils::TestECStoreEnv::builder()
-        .prefix("rustfs_heal_b920_test")
-        .disk_count(n_disks)
-        .build()
-        .await;
-    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
-    (env.disk_paths, env.ecstore, heal_storage)
+/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage`.
+async fn setup_test_env_n(n_disks: usize) -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    init_tracing();
+
+    let test_base_dir = format!("/tmp/rustfs_heal_b920_test_{}", uuid::Uuid::new_v4());
+    let temp_dir = PathBuf::from(&test_base_dir);
+    if temp_dir.exists() {
+        fs::remove_dir_all(&temp_dir).await.ok();
+    }
+    fs::create_dir_all(&temp_dir).await.unwrap();
+
+    let disk_paths: Vec<PathBuf> = (0..n_disks).map(|i| temp_dir.join(format!("disk{}", i + 1))).collect();
+    for disk_path in &disk_paths {
+        fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: n_disks,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
+
+    init_local_disks(endpoint_pools.clone()).await.unwrap();
+
+    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();
+
+    let buckets_list = ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+
+    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+    (disk_paths, ecstore, heal_storage)
 }

 async fn create_versioned_bucket(ecstore: &Arc<ECStore>, bucket: &str) {
@@ -214,7 +273,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn disk_walk_page_enumerates_subquorum_version_omitted_by_list_object_versions() {
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
        let bucket = "b920-enum-gap";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -251,7 +310,7 @@ mod serial_tests {
    #[serial]
    async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
        let bucket = "b920-meta-lost";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -301,7 +360,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
        let bucket = "b920-torn";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -350,7 +409,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(8).await;
        let bucket = "b920-reconstruct";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -404,7 +463,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn disk_walk_multipage_resume_heals_each_version_once() {
-        let (_disk_paths, ecstore, _heal_storage) = heal_env_n(4).await;
+        let (_disk_paths, ecstore, _heal_storage) = setup_test_env_n(4).await;
        let bucket = "b920-multipage";
        create_versioned_bucket(&ecstore, bucket).await;

@@ -464,7 +523,7 @@ mod serial_tests {
    #[serial]
    async fn offline_disk_during_walk_does_not_dangling_delete() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
        let bucket = "b920-offline";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -511,7 +570,7 @@ mod serial_tests {
    #[serial]
    async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
        disable_dangling_grace();
-        let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env_n(4).await;
        let bucket = "b1044-deep-verify";
        let object = "obj.bin";
        create_versioned_bucket(&ecstore, bucket).await;
@@ -22,16 +22,20 @@ use rustfs_heal::heal::{
 use serial_test::serial;
 use std::{
    path::{Path, PathBuf},
-    sync::Arc,
+    sync::{Arc, Once},
    time::Duration,
 };
 use tokio::fs;
+use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

 mod storage_api;

-use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};
+use storage_api::integration::{
+    BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
+    PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
+};

 const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
 const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
@@ -54,16 +58,89 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration
    }
 }

-/// Test helper: build the shared 4-disk temp-dir ECStore environment
-/// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage
-/// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest.
-async fn heal_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
-    let env = rustfs_test_utils::TestECStoreEnv::builder()
-        .prefix("rustfs_heal_heal_test")
-        .build()
-        .await;
-    let heal_storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
-    (env.disk_paths, env.ecstore, heal_storage)
+static INIT: Once = Once::new();
+
+pub fn init_tracing() {
+    INIT.call_once(|| {
+        let _ = tracing_subscriber::fmt()
+            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+            .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
+            .with_thread_names(true)
+            .try_init();
+    });
+}
+
+/// Test helper: Create test environment with ECStore
+async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    init_tracing();
+
+    // create temp dir as 4 disks with unique base dir
+    let test_base_dir = format!("/tmp/rustfs_heal_heal_test_{}", uuid::Uuid::new_v4());
+    let temp_dir = std::path::PathBuf::from(&test_base_dir);
+    if temp_dir.exists() {
+        fs::remove_dir_all(&temp_dir).await.ok();
+    }
+    fs::create_dir_all(&temp_dir).await.unwrap();
+
+    // create 4 disk dirs
+    let disk_paths = vec![
+        temp_dir.join("disk1"),
+        temp_dir.join("disk2"),
+        temp_dir.join("disk3"),
+        temp_dir.join("disk4"),
+    ];
+
+    for disk_path in &disk_paths {
+        fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    // create EndpointServerPools
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        // set correct index
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: 4,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+
+    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
+
+    // format disks (only first time)
+    init_local_disks(endpoint_pools.clone()).await.unwrap();
+
+    // Use port 0 so nextest can run this integration binary in parallel
+    // with other ECStore-backed tests without sharing a fixed peer port.
+    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();
+
+    // init bucket metadata system
+    let buckets_list = ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+    init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+
+    // Create heal storage layer
+    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+
+    (disk_paths, ecstore, heal_storage)
 }

 /// Test helper: Create a test bucket
@@ -92,7 +169,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_object_basic() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;

        // Create test bucket and object
        let bucket_name = "test-heal-object-basic";
@@ -161,7 +238,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_object_reclaims_orphan_data_dir_when_healthy() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;

        let bucket_name = "test-heal-reclaim-orphan";
        let object_name = "healthy-object.bin";
@@ -273,7 +350,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_bucket_basic() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;

        // Create test bucket
        let bucket_name = "test-heal-bucket-basic";
@@ -343,7 +420,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_format_basic() {
-        let (disk_paths, _ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;

        // ─── 1️⃣ delete format.json on one disk ──────────────
        let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
@@ -369,7 +446,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_format_with_data() {
-        let (disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (disk_paths, ecstore, heal_storage) = setup_test_env().await;

        // Create test bucket and object
        let bucket_name = "test-heal-format-with-data";
@@ -458,7 +535,7 @@ mod serial_tests {
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    #[serial]
    async fn test_heal_storage_api_direct() {
-        let (_disk_paths, ecstore, heal_storage) = heal_env().await;
+        let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;

        // Test direct heal storage API calls

@@ -15,13 +15,17 @@ pub(crate) mod endpoint_index {
    pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
 }

-// The temp-disk ECStore bootstrap itself lives in rustfs-test-utils
-// (backlog#1153 infra-1); this surface keeps only what the heal test bodies
-// still touch directly.
 #[allow(unused_imports)]
 pub(crate) mod integration {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
+    pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
+    pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools;
+    pub(crate) use rustfs_ecstore::api::layout::Endpoints;
+    pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints;
    pub(crate) use rustfs_ecstore::api::storage::ECStore;
+    pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
    pub(crate) use rustfs_storage_api::BucketOperations;
+    pub(crate) use rustfs_storage_api::BucketOptions;
    pub(crate) use rustfs_storage_api::MakeBucketOptions;
    pub(crate) use rustfs_storage_api::ObjectIO;
    pub(crate) use rustfs_storage_api::ObjectOperations;
@@ -59,7 +59,6 @@ url = { workspace = true }

 [dev-dependencies]
 pollster.workspace = true
-rustfs-test-utils = { workspace = true }
 serial_test = { workspace = true }
 temp-env = { workspace = true, features = ["async_closure"] }
 tempfile = { workspace = true }
@@ -22,15 +22,6 @@ pub(crate) fn credentials_or_default() -> Credentials {
    credentials().unwrap_or_default()
 }

-/// The signing key for STS session tokens.
-///
-/// GHSA-m77q-r63m-pj89 (intentionally UNFIXED): this returns the root secret
-/// key, so STS JWTs are signed with the shared root secret and anyone holding
-/// it can forge session tokens. The behavior is pinned by
-/// `test_ghsa_m77q_sts_session_token_signed_with_root_secret` in `sys.rs`;
-/// fixing the advisory (a dedicated STS signing key distinct from the root
-/// secret) must update that test red -> green. See
-/// docs/testing/security-regressions.md.
 pub(crate) fn token_signing_key() -> Option<String> {
    credentials().map(|cred| cred.secret_key)
 }
@@ -1952,18 +1952,6 @@ mod tests {
        );
    }

-    /// GHSA-m77q-r63m-pj89 (STS JWTs signed with the shared root secret —
-    /// intentionally UNFIXED). This test characterizes the STS credential
-    /// lifecycle: a session token signed with the active token-signing key
-    /// decodes and authorizes through the STS path. It pins that the SAME key
-    /// both signs and verifies STS tokens.
-    ///
-    /// MUST UPDATE WHEN GHSA-m77q IS FIXED: the fix introduces a dedicated STS
-    /// signing key distinct from the root secret. The narrower pin that captures
-    /// the advisory's exact signature (signing key == root secret) is
-    /// `test_ghsa_m77q_sts_session_token_signed_with_root_secret`; keep both in
-    /// lockstep and flip them red -> green together.
-    /// Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89>
    #[tokio::test]
    async fn test_created_sts_credentials_authorize_with_session_token_claims() {
        ensure_test_global_credentials();
@@ -2065,95 +2053,6 @@ mod tests {
        );
    }

-    /// GHSA-m77q-r63m-pj89 flow-level pin (STS session tokens are signed with the
-    /// shared root secret — intentionally UNFIXED). Anyone holding the root secret
-    /// can forge STS session tokens because RustFS reuses the root secret as the
-    /// STS token-signing key. This test PINS that vulnerable-by-design behavior end
-    /// to end: (1) the token-signing key IS the root secret, (2) an AssumeRole-style
-    /// session token issued with that key decodes with the ROOT secret and NOT with
-    /// any other secret, and (3) it authorizes through the STS path.
-    ///
-    /// MUST UPDATE WHEN GHSA-m77q IS FIXED: the fix introduces a dedicated STS
-    /// signing key distinct from the root secret. That makes assertion (1) and the
-    /// "root secret decodes the token" assertion fail (red). Whoever fixes m77q must
-    /// replace this pin with a regression asserting the NEW behavior — the root
-    /// secret can no longer decode STS session tokens — i.e. red -> green.
-    /// Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89>
-    #[tokio::test]
-    async fn test_ghsa_m77q_sts_session_token_signed_with_root_secret() {
-        ensure_test_global_credentials();
-
-        let root = crate::root_credentials::credentials().expect("global action (root) credentials should be initialized");
-        assert!(!root.secret_key.is_empty(), "root secret must be present for the STS signing-key pin");
-
-        // (1) m77q signature: the STS token-signing key IS the root secret. A fix
-        // that introduces a dedicated STS key makes this fail -> update red->green.
-        assert_eq!(
-            crate::root_credentials::token_signing_key().as_deref(),
-            Some(root.secret_key.as_str()),
-            "GHSA-m77q pin: STS tokens are signed with the root secret. If this fails, \
-             m77q may have been fixed (dedicated STS key) — update this test red->green."
-        );
-
-        // AssumeRole-style issuance: mint a session token with the active signing key.
-        // Use the mock store's registered STS parent (group `testgroup` -> readwrite)
-        // so authorization resolves through the STS path; the m77q pin is about the
-        // signing key, not the parent identity.
-        let parent_user = "sts-fallback-test-parent";
-        let signing_key = crate::root_credentials::token_signing_key().expect("STS token-signing key should be present");
-        let mut claims = HashMap::new();
-        claims.insert("parent".to_string(), Value::String(parent_user.to_string()));
-        claims.insert(
-            "exp".to_string(),
-            Value::Number(serde_json::Number::from(
-                (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(),
-            )),
-        );
-        let mut cred = get_new_credentials_with_metadata(&claims, &signing_key).expect("STS credentials should be created");
-        cred.parent_user = parent_user.to_string();
-
-        // (2) The session token decodes with the ROOT secret specifically — the crux
-        // of m77q — and rejects any non-root secret (HMAC verification fails).
-        let decoded = get_claims_from_token_with_secret(&cred.session_token, &root.secret_key)
-            .expect("GHSA-m77q pin: STS session token must decode with the ROOT secret");
-        assert_eq!(decoded.get("parent").and_then(Value::as_str), Some(parent_user));
-        assert!(
-            get_claims_from_token_with_secret(&cred.session_token, "not-the-root-secret").is_err(),
-            "GHSA-m77q pin: STS session token must NOT decode with a non-root secret"
-        );
-
-        // (3) The root-secret-signed STS credentials authorize end to end.
-        let store = StsTestMockStore::new(false);
-        let cache_manager = IamCache::new(store).await.unwrap();
-        let iam_sys = IamSys::new(cache_manager);
-        iam_sys
-            .set_temp_user(&cred.access_key, &cred, None)
-            .await
-            .expect("STS credentials should be persisted in the temp-user cache");
-
-        let groups: Option<Vec<String>> = None;
-        let args = Args {
-            account: &cred.access_key,
-            groups: &groups,
-            action: Action::S3Action(S3Action::ListBucketAction),
-            bucket: "mybucket",
-            conditions: &HashMap::new(),
-            is_owner: false,
-            object: "",
-            claims: &decoded,
-            deny_only: false,
-        };
-        let prepared = iam_sys.prepare_auth(&args).await;
-        assert!(
-            matches!(prepared.mode, PreparedIamMode::Sts { .. }),
-            "root-secret-signed STS credentials must use the STS authorization path"
-        );
-        assert!(
-            iam_sys.eval_prepared(&prepared, &args).await,
-            "root-secret-signed STS credentials should authorize through the parent group policy (m77q)"
-        );
-    }
-
    /// Regression test: temp credentials without groups in args still receive group-attached
    /// policies via the parent user (groups fallback). Without the fallback, policy_db_get
    /// would get None for groups and the user would have no group policies, so the action
@@ -22,7 +22,9 @@

 #[allow(unused_imports)]
 pub(crate) mod fixture {
-    pub(crate) use rustfs_ecstore::api::layout::SetupType;
+    pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
+    pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType};
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};

    // `update_erasure_type` is a write-side global facade entry. Its use is
    // restricted to reviewed storage_api boundaries; this test-only module is
@@ -31,16 +31,55 @@

 mod ecstore_test_compat;

-use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
+use ecstore_test_compat::fixture::{
+    ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, SetupType, init_local_disks, update_erasure_type,
+};
 use rustfs_iam::cache::Cache;
 use rustfs_iam::store::object::ObjectStore;
 use rustfs_iam::store::{GroupInfo, Store};
 use serial_test::serial;
 use std::collections::HashMap;
+use std::sync::Arc;
+use tokio_util::sync::CancellationToken;

 const TEST_GROUP: &str = "seq-restart-group";
 const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];

+async fn build_local_ecstore(temp_dir: &std::path::Path) -> Arc<ECStore> {
+    let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.join(format!("disk{i}"))).collect();
+    for disk_path in &disk_paths {
+        tokio::fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: 4,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+    let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
+
+    init_local_disks(endpoint_pools.clone()).await.unwrap();
+
+    // Port 0 keeps this integration binary parallel-safe alongside other
+    // ECStore-backed tests.
+    let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
+    ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap()
+}
+
 /// Restores single-node erasure mode even when an assertion panics, so a
 /// failing run cannot poison later `#[serial]` tests in this process.
 struct ErasureModeGuard;
@@ -58,15 +97,7 @@ async fn load_all_bypasses_namespace_lock_quorum() {
    // must be shortened before the first locked operation of this process.
    temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
        let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_no_lock_test_").unwrap();
-        // Shared temp-disk ECStore env (rustfs-test-utils, backlog#1153 infra-1).
-        // base_dir keeps cleanup ownership with this TempDir; the historical
-        // bootstrap never initialized the bucket-metadata system, so opt out.
-        let ecstore = rustfs_test_utils::TestECStoreEnv::builder()
-            .base_dir(temp_dir.path())
-            .init_bucket_metadata(false)
-            .build()
-            .await
-            .ecstore;
+        let ecstore = build_local_ecstore(temp_dir.path()).await;
        let store = ObjectStore::new(ecstore);

        // Seed IAM data while namespace locks still work (single-node mode).
@@ -38,7 +38,6 @@ url.workspace = true
 uuid = { workspace = true, features = ["v4", "serde"] }

 [dev-dependencies]
-proptest = "1"
 serial_test.workspace = true
 temp-env.workspace = true
 tokio = { workspace = true, features = ["macros", "rt"] }
@@ -965,7 +965,7 @@ pub async fn abort_incomplete_multipart_upload_due(
        .min_by_key(|(due, _)| due.unix_timestamp_nanos())
 }

-#[derive(Debug, Default)]
+#[derive(Default)]
 pub struct ObjectOpts {
    pub name: String,
    pub user_tags: String,
@@ -3456,576 +3456,4 @@ mod tests {
        // Without ExpiredObjectAllVersions, should use normal DeleteAction
        assert_eq!(event.action, IlmAction::DeleteAction);
    }
-
-    /// backlog#1148 ilm-8: once a restored copy's expiry passes, the evaluator
-    /// must emit `DeleteRestoredAction` for the current version (and the
-    /// `...Version` variant for a noncurrent one) so the scanner removes only
-    /// the local restored copy. The restore branch fires independently of any
-    /// configured rules.
-    #[tokio::test]
-    async fn eval_inner_emits_delete_restored_action_when_restore_expired() {
-        let lc = BucketLifecycleConfiguration {
-            expiry_updated_at: None,
-            rules: vec![],
-        };
-        let now = datetime!(2025-06-01 12:00:00 UTC);
-
-        let current = ObjectOpts {
-            name: "docs/report.bin".to_string(),
-            mod_time: Some(datetime!(2025-05-01 00:00:00 UTC)),
-            is_latest: true,
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            restore_expires: Some(now - Duration::hours(1)),
-            ..Default::default()
-        };
-        let event = lc.eval_inner(&current, now, 0).await;
-        assert_eq!(event.action, IlmAction::DeleteRestoredAction);
-
-        let noncurrent = ObjectOpts {
-            name: "docs/report.bin".to_string(),
-            mod_time: Some(datetime!(2025-05-01 00:00:00 UTC)),
-            is_latest: false,
-            version_id: Some(Uuid::from_u128(9)),
-            successor_mod_time: Some(datetime!(2025-05-02 00:00:00 UTC)),
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            restore_expires: Some(now - Duration::hours(1)),
-            ..Default::default()
-        };
-        let event = lc.eval_inner(&noncurrent, now, 0).await;
-        assert_eq!(event.action, IlmAction::DeleteRestoredVersionAction);
-    }
-
-    /// A restored copy whose expiry is still in the future must not be
-    /// scheduled for cleanup.
-    #[tokio::test]
-    async fn eval_inner_keeps_unexpired_restore_copy() {
-        let lc = BucketLifecycleConfiguration {
-            expiry_updated_at: None,
-            rules: vec![],
-        };
-        let now = datetime!(2025-06-01 12:00:00 UTC);
-
-        let current = ObjectOpts {
-            name: "docs/report.bin".to_string(),
-            mod_time: Some(datetime!(2025-05-01 00:00:00 UTC)),
-            is_latest: true,
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            restore_expires: Some(now + Duration::hours(1)),
-            ..Default::default()
-        };
-        let event = lc.eval_inner(&current, now, 0).await;
-        assert_eq!(event.action, IlmAction::NoneAction);
-    }
-
-    /// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
-    /// follow-up to backlog#1030 / rustfs#4455).
-    ///
-    /// backlog#1030 found that the old hand-written winner comparator was not a
-    /// strict weak ordering and could panic — proof that enumerated cases do
-    /// not cover the space of colliding events. These properties pin, over
-    /// randomized rule sets and object states:
-    ///
-    /// * `eval_inner` never panics and is deterministic for a fixed input;
-    /// * the winning event matches an independently recomputed candidate set:
-    ///   earliest `due` wins, ties break toward delete-class actions (the
-    ///   `min_by_key` selection that replaced the rustfs#4455 comparator);
-    /// * `expected_expiry_time` is monotonically non-decreasing in `days` and
-    ///   always lands on the processing boundary, both at production defaults
-    ///   and under an explicit `RUSTFS_ILM_PROCESS_TIME`.
-    ///
-    /// Case counts are tuned so the whole module runs in seconds inside the
-    /// default CI test job.
-    mod proptests {
-        use super::*;
-        use proptest::prelude::*;
-        use s3s::dto::{NoncurrentVersionExpiration, Tag};
-        use serial_test::serial;
-
-        const DAY_SECS: i64 = 86400;
-
-        /// Fixed positive anchor so generated instants stay in a realistic
-        /// range; strategies explore offsets around it.
-        fn base() -> OffsetDateTime {
-            datetime!(2025-01-01 00:00:00 UTC)
-        }
-
-        fn ts(offset_secs: i64) -> OffsetDateTime {
-            base() + Duration::seconds(offset_secs)
-        }
-
-        /// Pin the day length and processing boundary to production defaults
-        /// for the duration of `f`. `temp_env` also serializes environment
-        /// access against the other env-sensitive tests in this file.
-        fn with_production_time_env<T>(f: impl FnOnce() -> T) -> T {
-            temp_env::with_vars(
-                [
-                    (ENV_ILM_DEBUG_DAY_SECS, None::<&str>),
-                    (ENV_ILM_PROCESS_TIME, None),
-                    (ENV_ILM_PROCESS_TIME_DEPRECATED, None),
-                ],
-                f,
-            )
-        }
-
-        fn arb_opt_time(window_secs: i64) -> impl Strategy<Value = Option<OffsetDateTime>> {
-            prop_oneof![
-                2 => Just(None),
-                1 => Just(Some(OffsetDateTime::UNIX_EPOCH)),
-                5 => (-window_secs..window_secs).prop_map(|s| Some(ts(s))),
-            ]
-        }
-
-        fn arb_object_name() -> impl Strategy<Value = String> {
-            prop_oneof![
-                1 => Just(String::new()),
-                3 => Just("docs/report.txt".to_string()),
-                2 => Just("logs/app.log".to_string()),
-                1 => "[a-z]{1,12}",
-            ]
-        }
-
-        fn arb_user_tags() -> impl Strategy<Value = String> {
-            prop_oneof![
-                Just(String::new()),
-                Just("env=prod".to_string()),
-                Just("env=prod&team=storage".to_string()),
-                Just("team=storage".to_string()),
-            ]
-        }
-
-        fn arb_transition_status() -> impl Strategy<Value = String> {
-            prop_oneof![
-                Just(String::new()),
-                Just(TRANSITION_PENDING.to_string()),
-                Just(TRANSITION_COMPLETE.to_string()),
-            ]
-        }
-
-        prop_compose! {
-            fn arb_object_identity()(
-                name in arb_object_name(),
-                user_tags in arb_user_tags(),
-                mod_time in arb_opt_time(4 * DAY_SECS),
-                size in prop_oneof![Just(0usize), (1usize..1_048_576), Just(usize::MAX >> 1)],
-                version_id in prop_oneof![Just(None), Just(Some(Uuid::nil())), Just(Some(Uuid::from_u128(7)))],
-                is_latest in any::<bool>(),
-                delete_marker in any::<bool>(),
-            ) -> (String, String, Option<OffsetDateTime>, usize, Option<Uuid>, bool, bool) {
-                (name, user_tags, mod_time, size, version_id, is_latest, delete_marker)
-            }
-        }
-
-        prop_compose! {
-            /// Random object state across the version-chain dimensions: chain
-            /// position (`is_latest` / `num_versions` / `successor_mod_time`),
-            /// delete markers, suspension, and transition status.
-            fn arb_object_opts()(
-                (name, user_tags, mod_time, size, version_id, is_latest, delete_marker) in arb_object_identity(),
-                num_versions in 0usize..4,
-                successor_mod_time in arb_opt_time(4 * DAY_SECS),
-                transition_status in arb_transition_status(),
-                restore_ongoing in any::<bool>(),
-                restore_expires in arb_opt_time(2 * DAY_SECS),
-                versioned in any::<bool>(),
-                version_suspended in any::<bool>(),
-            ) -> ObjectOpts {
-                ObjectOpts {
-                    name,
-                    user_tags,
-                    mod_time,
-                    size,
-                    version_id,
-                    is_latest,
-                    delete_marker,
-                    num_versions,
-                    successor_mod_time,
-                    transition_status,
-                    restore_ongoing,
-                    restore_expires,
-                    versioned,
-                    version_suspended,
-                    ..Default::default()
-                }
-            }
-        }
-
-        fn arb_prefix() -> impl Strategy<Value = Option<String>> {
-            prop_oneof![
-                3 => Just(None),
-                1 => Just(Some(String::new())),
-                2 => Just(Some("docs/".to_string())),
-                1 => Just(Some("logs/".to_string())),
-            ]
-        }
-
-        fn arb_filter() -> impl Strategy<Value = Option<LifecycleRuleFilter>> {
-            prop_oneof![
-                4 => Just(None),
-                1 => Just(Some(LifecycleRuleFilter {
-                    prefix: Some("docs/".to_string()),
-                    ..Default::default()
-                })),
-                1 => Just(Some(LifecycleRuleFilter {
-                    tag: Some(Tag {
-                        key: Some("env".to_string()),
-                        value: Some("prod".to_string()),
-                    }),
-                    ..Default::default()
-                })),
-                1 => (0i64..2048).prop_map(|gt| Some(LifecycleRuleFilter {
-                    object_size_greater_than: Some(gt),
-                    ..Default::default()
-                })),
-            ]
-        }
-
-        prop_compose! {
-            fn arb_expiration()(
-                date_off in prop::option::of(-2 * DAY_SECS..2 * DAY_SECS),
-                days in prop::option::of(0i32..4),
-                delete_marker_flag in prop::option::of(any::<bool>()),
-                all_versions in prop::option::of(any::<bool>()),
-            ) -> LifecycleExpiration {
-                LifecycleExpiration {
-                    date: date_off.map(|s| ts(s).into()),
-                    days,
-                    expired_object_delete_marker: delete_marker_flag,
-                    expired_object_all_versions: all_versions,
-                }
-            }
-        }
-
-        prop_compose! {
-            fn arb_transition()(
-                date_off in prop::option::of(-2 * DAY_SECS..2 * DAY_SECS),
-                days in prop::option::of(0i32..4),
-                storage_class in prop_oneof![
-                    Just(None),
-                    Just(Some(TransitionStorageClass::from_static(""))),
-                    Just(Some(TransitionStorageClass::from_static("COLD"))),
-                ],
-            ) -> Transition {
-                Transition {
-                    date: date_off.map(|s| ts(s).into()),
-                    days,
-                    storage_class,
-                }
-            }
-        }
-
-        prop_compose! {
-            /// Fully random rule across every dimension the evaluator reads:
-            /// Days/Date expiration and transition, prefix, tag/size filters,
-            /// noncurrent actions, delete-marker actions, AllVersions.
-            fn arb_rule()(
-                enabled in prop::bool::weighted(0.8),
-                id in prop::option::of("[a-z]{1,6}"),
-                prefix in arb_prefix(),
-                filter in arb_filter(),
-                expiration in prop::option::of(arb_expiration()),
-                transitions in prop::option::of(prop::collection::vec(arb_transition(), 0..2)),
-                nc_exp in prop::option::of((prop::option::of(0i32..4), prop::option::of(0i32..3))),
-                nc_tr in prop::option::of((prop::option::of(0i32..4), any::<bool>())),
-                dme_days in prop::option::of(0i32..3),
-                abort_days in prop::option::of(0i32..3),
-            ) -> LifecycleRule {
-                LifecycleRule {
-                    status: if enabled {
-                        ExpirationStatus::from_static(ExpirationStatus::ENABLED)
-                    } else {
-                        ExpirationStatus::from_static(ExpirationStatus::DISABLED)
-                    },
-                    id,
-                    prefix,
-                    filter,
-                    expiration,
-                    transitions,
-                    noncurrent_version_expiration: nc_exp.map(|(noncurrent_days, newer_noncurrent_versions)| {
-                        NoncurrentVersionExpiration {
-                            noncurrent_days,
-                            newer_noncurrent_versions,
-                        }
-                    }),
-                    noncurrent_version_transitions: nc_tr.map(|(noncurrent_days, has_sc)| {
-                        vec![NoncurrentVersionTransition {
-                            noncurrent_days,
-                            newer_noncurrent_versions: None,
-                            storage_class: has_sc.then(|| TransitionStorageClass::from_static("COLD")),
-                        }]
-                    }),
-                    del_marker_expiration: dme_days.map(|days| s3s::dto::DelMarkerExpiration { days: Some(days) }),
-                    abort_incomplete_multipart_upload: abort_days.map(|days| {
-                        s3s::dto::AbortIncompleteMultipartUpload {
-                            days_after_initiation: Some(days),
-                        }
-                    }),
-                }
-            }
-        }
-
-        proptest! {
-            #![proptest_config(ProptestConfig::with_cases(96))]
-
-            /// `eval_inner` must never panic on any rule/object/now
-            /// combination, and must be deterministic: the same input
-            /// evaluated twice yields an identical event.
-            #[test]
-            #[serial]
-            fn eval_inner_never_panics_and_is_deterministic(
-                rules in prop::collection::vec(arb_rule(), 0..4),
-                obj in arb_object_opts(),
-                now_off in (-4 * DAY_SECS)..(8 * DAY_SECS),
-                newer_noncurrent in 0usize..4,
-            ) {
-                let lc = BucketLifecycleConfiguration {
-                    expiry_updated_at: None,
-                    rules,
-                };
-                let now = ts(now_off);
-                let (first, second) = block_on_with_env(&[], || async {
-                    let first = lc.eval_inner(&obj, now, newer_noncurrent).await;
-                    let second = lc.eval_inner(&obj, now, newer_noncurrent).await;
-                    (first, second)
-                });
-                prop_assert_eq!(first.action, second.action);
-                prop_assert_eq!(&first.rule_id, &second.rule_id);
-                prop_assert_eq!(first.due, second.due);
-                prop_assert_eq!(&first.storage_class, &second.storage_class);
-            }
-        }
-
-        /// One oracle candidate: `(due unix timestamp, action rank)` with rank
-        /// 0 for delete-class actions and 1 otherwise.
-        type Candidate = (i64, u8);
-
-        /// Independent re-statement of the delete-class rank. Deliberately NOT
-        /// `ilm_action_priority_rank`: the winner assertion classifies the
-        /// actual event with this copy, so a regression in the production rank
-        /// function cannot silently re-rank both sides of the comparison.
-        fn oracle_rank(action: &IlmAction) -> u8 {
-            match action {
-                IlmAction::DeleteAllVersionsAction
-                | IlmAction::DelMarkerDeleteAllVersionsAction
-                | IlmAction::DeleteAction
-                | IlmAction::DeleteVersionAction => 0,
-                _ => 1,
-            }
-        }
-
-        /// Independently recompute the eligible event set `eval_inner` should
-        /// consider for a live current version under `selection`-shaped rules
-        /// (expiration and first-transition only, no filters): expiration
-        /// fires when `now >= due`, transition when `now > due` and the object
-        /// has not already transitioned. Selection semantics under test:
-        /// earliest due wins, ties prefer delete-class.
-        fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
-            let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
-            let mut candidates = Vec::new();
-            for rule in &lc.rules {
-                if let Some(expiration) = &rule.expiration {
-                    // Mirrors the eval branches: a Date-based expiry needs a
-                    // non-zero date and fires when `now >= date`; a Days-based
-                    // expiry fires when `now >= expected_expiry_time`.
-                    if let Some(date) = &expiration.date {
-                        let due = OffsetDateTime::from(date.clone());
-                        if due.unix_timestamp() != 0 && now.unix_timestamp() >= due.unix_timestamp() {
-                            candidates.push((due.unix_timestamp(), 0));
-                        }
-                    } else if let Some(days) = expiration.days {
-                        let due = expected_expiry_time(mod_time, days);
-                        if now.unix_timestamp() >= due.unix_timestamp() {
-                            candidates.push((due.unix_timestamp(), 0));
-                        }
-                    }
-                }
-                if obj.transition_status != TRANSITION_COMPLETE
-                    && let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
-                    && transition.storage_class.as_ref().is_some_and(|sc| !sc.as_str().is_empty())
-                {
-                    let due = if let Some(date) = &transition.date {
-                        Some(OffsetDateTime::from(date.clone()))
-                    } else {
-                        transition.days.map(|days| expected_expiry_time(mod_time, days))
-                    };
-                    if let Some(due) = due
-                        && now.unix_timestamp() > due.unix_timestamp()
-                    {
-                        candidates.push((due.unix_timestamp(), 1));
-                    }
-                }
-            }
-            candidates
-        }
-
-        prop_compose! {
-            /// Rules restricted to the shapes the selection oracle models:
-            /// enabled, unfiltered, with exactly one of Date/Days per action so
-            /// the Date-over-Days precedence inside one action never applies.
-            fn arb_selection_rule()(
-                kind in 0u8..3,
-                exp_by_date in any::<bool>(),
-                exp_date_off in (-2 * DAY_SECS)..(2 * DAY_SECS),
-                exp_days in 0i32..4,
-                all_versions in any::<bool>(),
-                tr_by_date in any::<bool>(),
-                tr_date_off in (-2 * DAY_SECS)..(2 * DAY_SECS),
-                tr_days in 0i32..4,
-                id in "[a-z]{1,6}",
-            ) -> LifecycleRule {
-                let expiration = (kind != 1).then(|| LifecycleExpiration {
-                    date: exp_by_date.then(|| ts(exp_date_off).into()),
-                    days: (!exp_by_date).then_some(exp_days),
-                    expired_object_delete_marker: None,
-                    expired_object_all_versions: all_versions.then_some(true),
-                });
-                let transitions = (kind != 0).then(|| {
-                    vec![Transition {
-                        date: tr_by_date.then(|| ts(tr_date_off).into()),
-                        days: (!tr_by_date).then_some(tr_days),
-                        storage_class: Some(TransitionStorageClass::from_static("COLD")),
-                    }]
-                });
-                LifecycleRule {
-                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                    id: Some(id),
-                    prefix: None,
-                    filter: None,
-                    expiration,
-                    transitions,
-                    noncurrent_version_expiration: None,
-                    noncurrent_version_transitions: None,
-                    del_marker_expiration: None,
-                    abort_incomplete_multipart_upload: None,
-                }
-            }
-        }
-
-        proptest! {
-            #![proptest_config(ProptestConfig::with_cases(128))]
-
-            /// Differential test of winner selection (the rustfs#4455 fix):
-            /// for a live current version under randomized expiration and
-            /// transition rules, `eval_inner`'s winner must carry the
-            /// minimum `(due, rank)` of the independently recomputed
-            /// candidate set — earliest due wins, ties prefer delete-class —
-            /// and must be `NoneAction` exactly when that set is empty.
-            #[test]
-            #[serial]
-            fn eval_inner_winner_matches_selection_oracle(
-                rules in prop::collection::vec(arb_selection_rule(), 0..5),
-                mod_off in 0i64..(2 * DAY_SECS),
-                now_off in (-DAY_SECS)..(6 * DAY_SECS),
-                already_transitioned in any::<bool>(),
-            ) {
-                let lc = BucketLifecycleConfiguration {
-                    expiry_updated_at: None,
-                    rules,
-                };
-                let obj = ObjectOpts {
-                    name: "docs/report.txt".to_string(),
-                    mod_time: Some(ts(mod_off)),
-                    is_latest: true,
-                    transition_status: if already_transitioned {
-                        TRANSITION_COMPLETE.to_string()
-                    } else {
-                        String::new()
-                    },
-                    ..Default::default()
-                };
-                let now = ts(now_off);
-
-                // Oracle and evaluator must observe the same (pinned) time env.
-                let (event, expected) = with_production_time_env(|| {
-                    let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
-                    let rt = tokio::runtime::Builder::new_current_thread()
-                        .enable_all()
-                        .build()
-                        .expect("build runtime");
-                    let event = rt.block_on(lc.eval_inner(&obj, now, 0));
-                    (event, expected)
-                });
-
-                match expected {
-                    None => prop_assert_eq!(event.action, IlmAction::NoneAction),
-                    Some((due_ts, rank)) => {
-                        prop_assert_ne!(event.action, IlmAction::NoneAction);
-                        let actual_due = event.due.expect("winning event carries a due").unix_timestamp();
-                        prop_assert_eq!(actual_due, due_ts);
-                        prop_assert_eq!(oracle_rank(&event.action), rank);
-                    }
-                }
-            }
-        }
-
-        proptest! {
-            #![proptest_config(ProptestConfig::with_cases(256))]
-
-            /// At production defaults `expected_expiry_time` is monotonically
-            /// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
-            /// any post-1970 deadline).
-            #[test]
-            #[serial]
-            fn expected_expiry_time_is_monotonic_in_days(
-                mod_off in 0i64..(3650 * DAY_SECS),
-                d1 in 0i32..2000,
-                d2 in 0i32..2000,
-            ) {
-                let (lo, hi) = if d1 <= d2 { (d1, d2) } else { (d2, d1) };
-                let mod_time = ts(mod_off);
-                let (e_lo, e_hi) = with_production_time_env(|| {
-                    (expected_expiry_time(mod_time, lo), expected_expiry_time(mod_time, hi))
-                });
-                prop_assert!(
-                    e_lo <= e_hi,
-                    "expected_expiry_time not monotonic: days={} -> {:?}, days={} -> {:?}",
-                    lo, e_lo, hi, e_hi
-                );
-            }
-
-            /// At production defaults every Days-based deadline is rounded up
-            /// to the next whole-day boundary: the result is day-aligned, not
-            /// before `mod_time + days`, and less than one boundary beyond it.
-            #[test]
-            #[serial]
-            fn expected_expiry_time_lands_on_default_day_boundary(
-                mod_off in 0i64..(3650 * DAY_SECS),
-                days in 1i32..2000,
-            ) {
-                let mod_time = ts(mod_off);
-                let result = with_production_time_env(|| expected_expiry_time(mod_time, days));
-                let boundary_nanos = i128::from(DAY_SECS) * 1_000_000_000;
-                prop_assert_eq!(result.unix_timestamp_nanos().rem_euclid(boundary_nanos), 0);
-                let raw = mod_time + Duration::seconds(i64::from(days) * DAY_SECS);
-                prop_assert!(result >= raw);
-                prop_assert!(result - raw < Duration::seconds(DAY_SECS));
-            }
-
-            /// With an explicit `RUSTFS_ILM_PROCESS_TIME`, deadlines round up
-            /// to that boundary instead: aligned to it, never early, and less
-            /// than one boundary late.
-            #[test]
-            #[serial]
-            fn expected_expiry_time_lands_on_explicit_process_boundary(
-                mod_off in 0i64..(365 * DAY_SECS),
-                days in 1i32..400,
-                boundary in 1u32..7200,
-            ) {
-                let mod_time = ts(mod_off);
-                let boundary_str = boundary.to_string();
-                let result = temp_env::with_vars(
-                    [
-                        (ENV_ILM_DEBUG_DAY_SECS, None::<&str>),
-                        (ENV_ILM_PROCESS_TIME, Some(boundary_str.as_str())),
-                        (ENV_ILM_PROCESS_TIME_DEPRECATED, None),
-                    ],
-                    || expected_expiry_time(mod_time, days),
-                );
-                let boundary_nanos = i128::from(boundary) * 1_000_000_000;
-                prop_assert_eq!(result.unix_timestamp_nanos().rem_euclid(boundary_nanos), 0);
-                let raw = mod_time + Duration::seconds(i64::from(days) * DAY_SECS);
-                prop_assert!(result >= raw);
-                prop_assert!(result - raw < Duration::seconds(i64::from(boundary)));
-            }
-        }
-    }
 }
@@ -16,12 +16,11 @@ use crate::{GroupAddRemove, GroupDesc, SRSvcAccCreate, UserInfo};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::{BTreeMap, HashMap};
-use std::fmt;
 use time::OffsetDateTime;

 pub const SITE_REPL_API_VERSION: &str = "1";

-#[derive(Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct PeerSite {
    #[serde(default)]
    pub name: String,
@@ -31,21 +30,6 @@ pub struct PeerSite {
    pub access_key: String,
    #[serde(rename = "secretKey", default)]
    pub secret_key: String,
-    #[serde(rename = "skipTlsVerify", default)]
-    pub skip_tls_verify: bool,
-    #[serde(rename = "caCertPem", default)]
-    pub ca_cert_pem: String,
-}
-
-impl fmt::Debug for PeerSite {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PeerSite")
-            .field("name", &self.name)
-            .field("endpoint", &self.endpoint)
-            .field("skip_tls_verify", &self.skip_tls_verify)
-            .field("has_custom_ca", &!self.ca_cert_pem.is_empty())
-            .finish()
-    }
 }

 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -122,7 +106,7 @@ pub enum SyncStatus {
    Unknown,
 }

-#[derive(Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct PeerInfo {
    #[serde(default)]
    pub endpoint: String,
@@ -138,26 +122,10 @@ pub struct PeerInfo {
    pub replicate_ilm_expiry: bool,
    #[serde(rename = "objectNamingMode", default, skip_serializing_if = "String::is_empty")]
    pub object_naming_mode: String,
-    #[serde(rename = "skipTlsVerify", default)]
-    pub skip_tls_verify: bool,
-    #[serde(rename = "caCertPem", default)]
-    pub ca_cert_pem: String,
    #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")]
    pub api_version: Option<String>,
 }

-impl fmt::Debug for PeerInfo {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PeerInfo")
-            .field("endpoint", &self.endpoint)
-            .field("name", &self.name)
-            .field("deployment_id", &self.deployment_id)
-            .field("skip_tls_verify", &self.skip_tls_verify)
-            .field("has_custom_ca", &!self.ca_cert_pem.is_empty())
-            .finish()
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct SRPolicyMapping {
    #[serde(rename = "userOrGroup", default)]
@@ -1199,107 +1167,3 @@ pub struct SiteNetPerfResult {
    #[serde(rename = "nodeResults", default, skip_serializing_if = "Vec::is_empty")]
    pub node_results: Vec<SiteNetPerfNodeResult>,
 }
-
-#[cfg(test)]
-mod tests {
-    use super::{PeerInfo, PeerSite};
-    use serde_json::{Value, json};
-
-    const TEST_CA_CERT: &str = "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----";
-
-    #[test]
-    fn peer_tls_fields_default_when_missing_from_legacy_json() {
-        let site: PeerSite = serde_json::from_value(json!({
-            "name": "site-a",
-            "endpoints": "https://site-a.example.com"
-        }))
-        .expect("legacy PeerSite JSON should deserialize");
-        let peer: PeerInfo = serde_json::from_value(json!({
-            "endpoint": "https://site-a.example.com",
-            "name": "site-a",
-            "deploymentID": "deployment-a"
-        }))
-        .expect("legacy PeerInfo JSON should deserialize");
-
-        assert!(!site.skip_tls_verify);
-        assert_eq!(site.ca_cert_pem, "");
-        assert!(!peer.skip_tls_verify);
-        assert_eq!(peer.ca_cert_pem, "");
-    }
-
-    #[test]
-    fn peer_tls_fields_round_trip_with_exact_json_names() {
-        let site_json = json!({
-            "name": "site-a",
-            "endpoints": "https://site-a.example.com",
-            "accessKey": "access-key",
-            "secretKey": "secret-key",
-            "skipTlsVerify": true,
-            "caCertPem": TEST_CA_CERT
-        });
-        let peer_json = json!({
-            "endpoint": "https://site-a.example.com",
-            "name": "site-a",
-            "deploymentID": "deployment-a",
-            "sync": "unknown",
-            "defaultbandwidth": {
-                "bandwidthLimitPerBucket": 0,
-                "set": false
-            },
-            "replicate-ilm-expiry": false,
-            "objectNamingMode": "path",
-            "skipTlsVerify": true,
-            "caCertPem": TEST_CA_CERT
-        });
-
-        let site: PeerSite = serde_json::from_value(site_json.clone()).expect("PeerSite JSON should deserialize");
-        let peer: PeerInfo = serde_json::from_value(peer_json.clone()).expect("PeerInfo JSON should deserialize");
-
-        assert_eq!(serde_json::to_value(site).expect("PeerSite should serialize"), site_json);
-        assert_eq!(serde_json::to_value(peer).expect("PeerInfo should serialize"), peer_json);
-    }
-
-    #[test]
-    fn peer_tls_false_and_empty_ca_are_still_serialized() {
-        let site = serde_json::to_value(PeerSite::default()).expect("PeerSite should serialize");
-        let peer = serde_json::to_value(PeerInfo::default()).expect("PeerInfo should serialize");
-
-        for value in [site, peer] {
-            let object = value.as_object().expect("peer JSON should be an object");
-            assert_eq!(object.get("skipTlsVerify"), Some(&Value::Bool(false)));
-            assert_eq!(object.get("caCertPem"), Some(&Value::String(String::new())));
-        }
-    }
-
-    #[test]
-    fn peer_debug_output_redacts_secrets_and_ca_contents() {
-        let site = PeerSite {
-            name: "site-a".to_owned(),
-            endpoint: "https://site-a.example.com".to_owned(),
-            access_key: "sensitive-access-key".to_owned(),
-            secret_key: "sensitive-secret-key".to_owned(),
-            skip_tls_verify: true,
-            ca_cert_pem: TEST_CA_CERT.to_owned(),
-        };
-        let peer = PeerInfo {
-            endpoint: "https://site-a.example.com".to_owned(),
-            name: "site-a".to_owned(),
-            deployment_id: "deployment-a".to_owned(),
-            skip_tls_verify: false,
-            ca_cert_pem: TEST_CA_CERT.to_owned(),
-            ..PeerInfo::default()
-        };
-
-        let site_debug = format!("{site:?}");
-        let peer_debug = format!("{peer:?}");
-
-        assert!(!site_debug.contains("BEGIN CERTIFICATE"));
-        assert!(!site_debug.contains("sensitive-access-key"));
-        assert!(!site_debug.contains("sensitive-secret-key"));
-        assert!(site_debug.contains("skip_tls_verify: true"));
-        assert!(site_debug.contains("has_custom_ca: true"));
-        assert!(!peer_debug.contains("BEGIN CERTIFICATE"));
-        assert!(peer_debug.contains("skip_tls_verify: false"));
-        assert!(peer_debug.contains("has_custom_ca: true"));
-    }
-}
@@ -80,7 +80,6 @@ pub struct NotificationMetricSnapshot {
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct NotificationTargetMetricSnapshot {
    pub failed_messages: u64,
-    pub failed_store_length: u64,
    pub queue_length: u64,
    pub target_id: String,
    pub target_type: String,
@@ -19,13 +19,12 @@ use rustfs_targets::{
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::{RwLock, Semaphore};
-use tracing::{debug, info, warn};
+use tracing::{debug, info};

 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
-const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";

 #[derive(Clone)]
 pub struct NotifyRuntimeFacade {
@@ -49,18 +48,9 @@ impl NotifyRuntimeFacade {
                    match event {
                        ReplayEvent::Delivered { .. } => metrics.increment_processed(),
                        ReplayEvent::RetryableError { .. } => {}
-                        ReplayEvent::RetryExhausted { detail, key, target } => {
-                            warn!(
-                                event = EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED,
-                                component = LOG_COMPONENT_NOTIFY,
-                                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                                target_id = %target.id(),
-                                replay_key = %key,
-                                error = %detail,
-                                "notify replay retry budget exhausted, entry stays queued and retries"
-                            );
-                        }
-                        ReplayEvent::Dropped { target, .. } | ReplayEvent::PermanentFailure { target, .. } => {
+                        ReplayEvent::Dropped { target, .. }
+                        | ReplayEvent::PermanentFailure { target, .. }
+                        | ReplayEvent::RetryExhausted { target, .. } => {
                            target.record_final_failure();
                            metrics.increment_failed();
                        }
@@ -51,7 +51,6 @@ impl NotifyRuntimeView {
            .into_iter()
            .map(|snapshot| NotificationTargetMetricSnapshot {
                failed_messages: snapshot.failed_messages,
-                failed_store_length: snapshot.failed_store_length,
                queue_length: snapshot.queue_length,
                target_id: snapshot.target_id,
                target_type: snapshot.target_type,
@@ -91,7 +90,6 @@ mod tests {
        active: bool,
        enabled: bool,
        failed_messages: Arc<AtomicU64>,
-        failed_store_length: u64,
        id: TargetID,
        total_messages: Arc<AtomicU64>,
    }
@@ -102,7 +100,6 @@ mod tests {
                active: true,
                enabled: true,
                failed_messages: Arc::new(AtomicU64::new(0)),
-                failed_store_length: 0,
                id: TargetID::new(id.to_string(), name.to_string()),
                total_messages: Arc::new(AtomicU64::new(0)),
            }
@@ -113,11 +110,6 @@ mod tests {
            self
        }

-        fn with_failed_store_length(mut self, failed_store_length: u64) -> Self {
-            self.failed_store_length = failed_store_length;
-            self
-        }
-
        fn with_enabled(mut self, enabled: bool) -> Self {
            self.enabled = enabled;
            self
@@ -176,7 +168,6 @@ mod tests {
        fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
            TargetDeliverySnapshot {
                failed_messages: self.failed_messages.load(Ordering::Relaxed),
-                failed_store_length: self.failed_store_length,
                queue_length: 0,
                total_messages: self.total_messages.load(Ordering::Relaxed),
            }
@@ -215,7 +206,7 @@ mod tests {
        let target_list = Arc::new(RwLock::new(TargetList::new()));
        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));

-        let online = Arc::new(TestTarget::new("primary", "webhook").with_failed_store_length(7));
+        let online = Arc::new(TestTarget::new("primary", "webhook"));
        online.record_successes(3);
        online.record_failures(1);

@@ -248,11 +239,9 @@ mod tests {
        assert_eq!(metric_snapshots.len(), 2);
        assert_eq!(metric_snapshots[0].target_id, "backup:mqtt");
        assert_eq!(metric_snapshots[0].failed_messages, 0);
-        assert_eq!(metric_snapshots[0].failed_store_length, 0);
        assert_eq!(metric_snapshots[0].total_messages, 2);
        assert_eq!(metric_snapshots[1].target_id, "primary:webhook");
        assert_eq!(metric_snapshots[1].failed_messages, 1);
-        assert_eq!(metric_snapshots[1].failed_store_length, 7);
        assert_eq!(metric_snapshots[1].total_messages, 3);

        let health_snapshots = runtime_view.snapshot_target_health().await;
@@ -352,9 +352,7 @@ impl MokaBackend {
        // Moka maintenance is still needed to retire queued removals, but clear
        // must not rely on lazy invalidation alone: under heavy contention an
        // entry can remain physically resident (and still counted) even though
-        // the invalidation fence already hides it from lookups. Sweep both the
-        // entries visible to iteration and a fallback invalidate_all fence so
-        // delayed internal writes cannot strand a counted entry.
+        // the invalidation fence already hides it from lookups.
        for _ in 0..256 {
            self.cache.run_pending_tasks().await;
            let lingering_keys: Vec<_> = self.cache.iter().map(|(key, _)| key.as_ref().clone()).collect();
@@ -362,9 +360,6 @@ impl MokaBackend {
                self.cache.remove(&key).await;
            }

-            self.cache.invalidate_all();
-            self.cache.run_pending_tasks().await;
-
            if self.cache.entry_count() == 0 {
                break;
            }
@@ -940,28 +935,22 @@ mod tests {
        const ITERS: usize = 400;
        const KEYS: usize = 6; // small pool → heavy contention on shared identities

-        let stress_plans: Arc<Vec<_>> = Arc::new(
-            (0..KEYS)
-                .map(|k| versioned_plan(&format!("object-{}", k % 3), &format!("v{k}"), &format!("etag-{k}")))
-                .collect(),
-        );
        let mut handles = Vec::new();
        for t in 0..TASKS {
            let backend = Arc::clone(&backend);
-            let stress_plans = Arc::clone(&stress_plans);
            handles.push(tokio::spawn(async move {
                for i in 0..ITERS {
                    let k = (t + i) % KEYS;
                    let object = format!("object-{}", k % 3);
-                    let plan = &stress_plans[k];
+                    let plan = versioned_plan(&object, &format!("v{k}"), &format!("etag-{k}"));
                    // Deterministic per-(task,iter) mix: mostly fills, plus
                    // lookups and invalidations racing against them.
                    match (t + i) % 4 {
                        0 | 1 => {
-                            let _ = backend.fill_body(plan, Bytes::from_static(b"hello")).await;
+                            let _ = backend.fill_body(&plan, Bytes::from_static(b"hello")).await;
                        }
                        2 => {
-                            let _ = backend.lookup_body(plan).await;
+                            let _ = backend.lookup_body(&plan).await;
                        }
                        _ => {
                            let identity = ObjectDataCacheIdentity::new("bucket", object.as_str());
@@ -1001,17 +990,7 @@ mod tests {
        );

        let _ = backend.clear().await;
-        for plan in stress_plans.iter().chain(std::iter::once(&clean)) {
-            assert!(
-                matches!(backend.lookup_body(plan).await, ObjectDataCacheLookup::Miss),
-                "clear() must make every exercised key unobservable"
-            );
-        }
-        assert!(
-            backend.cache.iter().next().is_none(),
-            "clear() must expose no entries through iteration (approximate entry count: {})",
-            backend.entry_count()
-        );
+        assert_eq!(backend.entry_count(), 0, "clear() must drop every entry");
    }

    /// A memory-constrained container (low snapshot) must reject an entire
@@ -115,6 +115,7 @@ libc = { workspace = true }


 [dev-dependencies]
+metrics-util = { version = "0.20", features = ["debugging"] }
 tokio = { workspace = true, features = ["full"] }
 tempfile = { workspace = true }
 temp-env = { workspace = true }
@@ -121,7 +121,7 @@ All configuration is read from environment variables at startup.
 | `RUSTFS_OBS_LOGS_EXPORT_ENABLED`      | `true`    | Toggle OTLP log export                                     |
 | `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `false`   | Toggle profiling export                                    |
 | `RUSTFS_OBS_USE_STDOUT`               | `false`   | Mirror all signals to stdout alongside OTLP                |
-| `RUSTFS_OBS_SAMPLE_RATIO`             | `0.1`     | Trace sampling ratio `0.0`–`1.0`                           |
+| `RUSTFS_OBS_SAMPLE_RATIO`             | `1.0`     | Trace sampling ratio `0.0`–`1.0`                           |
 | `RUSTFS_OBS_METER_INTERVAL`           | `15`      | Metrics export interval (seconds)                          |

 ### Service identity
@@ -131,12 +131,14 @@ All configuration is read from environment variables at startup.
 | `RUSTFS_OBS_SERVICE_NAME`    | `rustfs`          | OTel `service.name`                                     |
 | `RUSTFS_OBS_SERVICE_VERSION` | _(crate version)_ | OTel `service.version`                                  |
 | `RUSTFS_OBS_ENVIRONMENT`     | `development`     | Deployment environment (`production`, `development`, …) |
+| `RUSTFS_OBS_INSTANCE_ID`     | _(empty)_         | Stable OTel `service.instance.id`; omitted when unset    |
+| `RUSTFS_OBS_CLUSTER_ID`      | _(empty)_         | Stable RustFS deployment identity; omitted when unset    |

 ### Local logging

 | Variable                        | Default      | Description                                                                                                                                                                                                                                                                                                                                                           |
 |---------------------------------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `RUSTFS_OBS_LOGGER_LEVEL`       | `info`       | Log level; `RUST_LOG` syntax supported                                                                                                                                                                                                                                                                                                                                |
+| `RUSTFS_OBS_LOGGER_LEVEL`       | `error`      | Log level; `RUST_LOG` syntax supported                                                                                                                                                                                                                                                                                                                               |
 | `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false`      | When file logging is active, also mirror to stdout                                                                                                                                                                                                                                                                                                                    |
 | `RUSTFS_OBS_LOG_DIRECTORY`      | _(empty)_    | **Directory for rolling log files. When empty, logs go to stdout only**                                                                                                                                                                                                                                                                                               |
 | `RUSTFS_OBS_LOG_FILENAME`       | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `<timestamp>-<counter>.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.<timestamp>-<counter>` (e.g., `rustfs.log.20231027103001.123456-0`). |
@@ -167,6 +169,22 @@ All configuration is read from environment variables at startup.

 ---

+## Local all-in-one
+
+After starting Grafana/Tempo/Loki/Prometheus from `/Users/tang/code/otel-all-in-one`, configure RustFS to export all three OTLP signals directly:
+
+```bash
+export RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
+export RUSTFS_OBS_SERVICE_NAME=rustfs
+export RUSTFS_OBS_ENVIRONMENT=local
+export RUSTFS_OBS_SAMPLE_RATIO=1
+export RUSTFS_OBS_LOGGER_LEVEL=info
+```
+
+Do not also configure promtail to collect the same RustFS logs: OTLP logs can already be correlated from Tempo to Loki. When `RUSTFS_OBS_ENDPOINT` is unset, RustFS retains its existing local logging fallback.
+
+---
+
 ## Cleaner & Rotation Metrics

 The log rotation and cleanup pipeline emits these metrics (via the `metrics` facade):
@@ -28,14 +28,14 @@ use rustfs_config::observability::{
    DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE,
    DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
    DEFAULT_OBS_LOG_PARALLEL_COMPRESS, DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL,
-    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_ENDPOINT, ENV_OBS_ENDPOINT_HEADERS,
-    ENV_OBS_ENDPOINT_LOGS_HEADERS, ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_METRICS_HEADERS,
-    ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TRACES_HEADERS,
-    ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS,
-    ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_COMPRESSION_ALGORITHM,
-    ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT,
-    ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_FILES,
-    ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
+    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_CLUSTER_ID, ENV_OBS_ENDPOINT,
+    ENV_OBS_ENDPOINT_HEADERS, ENV_OBS_ENDPOINT_LOGS_HEADERS, ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS,
+    ENV_OBS_ENDPOINT_METRICS_HEADERS, ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TIMEOUT_MILLIS,
+    ENV_OBS_ENDPOINT_TRACES_HEADERS, ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, ENV_OBS_ENVIRONMENT, ENV_OBS_INSTANCE_ID,
+    ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
+    ENV_OBS_LOG_COMPRESSION_ALGORITHM, ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN,
+    ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL,
+    ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
    ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_PARALLEL_COMPRESS, ENV_OBS_LOG_PARALLEL_WORKERS, ENV_OBS_LOG_ROTATION_TIME,
    ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOG_ZSTD_COMPRESSION_LEVEL, ENV_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, ENV_OBS_LOG_ZSTD_WORKERS,
    ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT,
@@ -162,6 +162,10 @@ pub struct OtelConfig {
    pub service_version: Option<String>,
    /// Deployment environment tag, e.g. `production` or `development`.
    pub environment: Option<String>,
+    /// Stable identity for this process instance. Empty values are omitted.
+    pub instance_id: Option<String>,
+    /// Stable RustFS deployment identity. Empty values are omitted.
+    pub cluster_id: Option<String>,

    // ── Local logging ─────────────────────────────────────────────────────────
    /// Minimum log level directive (default: `error`).
@@ -307,6 +311,8 @@ impl OtelConfig {
            service_name: Some(get_env_str(ENV_OBS_SERVICE_NAME, APP_NAME)),
            service_version: Some(get_env_str(ENV_OBS_SERVICE_VERSION, SERVICE_VERSION)),
            environment: Some(get_env_str(ENV_OBS_ENVIRONMENT, ENVIRONMENT)),
+            instance_id: get_env_opt_str(ENV_OBS_INSTANCE_ID),
+            cluster_id: get_env_opt_str(ENV_OBS_CLUSTER_ID),
            // Local logging
            logger_level: Some(get_env_str(ENV_OBS_LOGGER_LEVEL, DEFAULT_LOG_LEVEL)),
            log_stdout_enabled: get_env_opt_bool(ENV_OBS_LOG_STDOUT_ENABLED),
@@ -64,6 +64,8 @@ mod error;
 mod global;
 mod logging;
 pub mod metrics;
+pub mod semconv;
+mod task;
 mod telemetry;

 pub use cleaner::*;
@@ -78,6 +80,8 @@ pub use metrics::{
    MetricsRuntimeShutdownHandle, MetricsRuntimeStatusSnapshot, MetricsRuntimeWorkerMutation, init_metrics_runtime,
    metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
 };
+pub use semconv::{ErrorClass, Operation, ResultClass, Stage, StreamDirection, observe_operation, stage_span, stream_span};
+pub use task::spawn_traced;
 pub use telemetry::{OtelGuard, Recorder};

 // Dial9 Tokio runtime telemetry
@@ -33,8 +33,6 @@ pub struct AuditTargetStats {
    pub target_id: String,
    /// Number of messages that failed to send
    pub failed_messages: u64,
-    /// Number of messages held in the failed-events store
-    pub failed_store_length: u64,
    /// Number of unsent messages in queue
    pub queue_length: u64,
    /// Total number of messages sent
@@ -50,7 +48,7 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
        return Vec::new();
    }

-    let mut metrics = Vec::with_capacity(stats.len() * 4);
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
    for stat in stats {
        let target_id_label: Cow<'static, str> = Cow::Owned(stat.target_id.clone());

@@ -58,10 +56,6 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
            PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
                .with_label("target_id", target_id_label.clone()),
        );
-        metrics.push(
-            PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_MD, stat.failed_store_length as f64)
-                .with_label("target_id", target_id_label.clone()),
-        );
        metrics.push(
            PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
                .with_label("target_id", target_id_label.clone()),
@@ -86,14 +80,12 @@ mod tests {
            AuditTargetStats {
                target_id: "target-1".to_string(),
                failed_messages: 5,
-                failed_store_length: 3,
                queue_length: 10,
                total_messages: 1000,
            },
            AuditTargetStats {
                target_id: "target-2".to_string(),
                failed_messages: 2,
-                failed_store_length: 1,
                queue_length: 5,
                total_messages: 500,
            },
@@ -101,19 +93,12 @@ mod tests {

        let metrics = collect_audit_metrics(&stats);

-        assert_eq!(metrics.len(), 8); // 2 targets * 4 metrics each
+        assert_eq!(metrics.len(), 6); // 2 targets * 3 metrics each

        let failed = metrics
            .iter()
            .find(|m| m.value == 5.0 && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1"));
        assert!(failed.is_some());
-
-        let failed_store = metrics.iter().find(|m| {
-            m.value == 3.0
-                && m.name == AUDIT_FAILED_STORE_LENGTH_MD.get_full_metric_name()
-                && m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1")
-        });
-        assert!(failed_store.is_some());
    }

    #[test]
@@ -126,7 +111,6 @@ mod tests {
    #[test]
    fn audit_target_totals_are_exported_as_gauges() {
        assert_eq!(AUDIT_FAILED_MESSAGES_MD.metric_type, MetricType::Gauge);
-        assert_eq!(AUDIT_FAILED_STORE_LENGTH_MD.metric_type, MetricType::Gauge);
        assert_eq!(AUDIT_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
        assert_eq!(AUDIT_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
    }
@@ -16,15 +16,14 @@

 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::notification_target::{
-    NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
-    NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID, TARGET_TYPE,
+    NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD,
+    TARGET_ID, TARGET_TYPE,
 };
 use std::borrow::Cow;

 #[derive(Debug, Clone, Default)]
 pub struct NotificationTargetStats {
    pub failed_messages: u64,
-    pub failed_store_length: u64,
    pub queue_length: u64,
    pub target_id: String,
    pub target_type: String,
@@ -36,7 +35,7 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
        return Vec::new();
    }

-    let mut metrics = Vec::with_capacity(stats.len() * 4);
+    let mut metrics = Vec::with_capacity(stats.len() * 3);
    for stat in stats {
        let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
        let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone());
@@ -46,11 +45,6 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
                .with_label(TARGET_ID, target_id.clone())
                .with_label(TARGET_TYPE, target_type.clone()),
        );
-        metrics.push(
-            PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, stat.failed_store_length as f64)
-                .with_label(TARGET_ID, target_id.clone())
-                .with_label(TARGET_TYPE, target_type.clone()),
-        );
        metrics.push(
            PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
                .with_label(TARGET_ID, target_id.clone())
@@ -75,7 +69,6 @@ mod tests {
    fn test_collect_notification_target_metrics() {
        let stats = vec![NotificationTargetStats {
            failed_messages: 2,
-            failed_store_length: 3,
            queue_length: 4,
            target_id: "primary:webhook".to_string(),
            target_type: "webhook".to_string(),
@@ -84,15 +77,7 @@ mod tests {

        let metrics = collect_notification_target_metrics(&stats);

-        assert_eq!(metrics.len(), 4);
-        assert!(metrics.iter().any(|metric| {
-            metric.value == 3.0
-                && metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name()
-                && metric
-                    .labels
-                    .iter()
-                    .any(|(key, value)| *key == TARGET_ID && value == "primary:webhook")
-        }));
+        assert_eq!(metrics.len(), 3);
        assert!(metrics.iter().any(|metric| {
            metric.value == 42.0
                && metric
@@ -109,7 +94,6 @@ mod tests {
    #[test]
    fn notification_target_totals_are_exported_as_gauges() {
        assert_eq!(NOTIFICATION_TARGET_FAILED_MESSAGES_MD.metric_type, MetricType::Gauge);
-        assert_eq!(NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.metric_type, MetricType::Gauge);
        assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
        assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
    }
@@ -1222,7 +1222,6 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                            .into_iter()
                            .map(|snapshot| AuditTargetStats {
                                failed_messages: snapshot.failed_messages,
-                                failed_store_length: snapshot.failed_store_length,
                                queue_length: snapshot.queue_length,
                                target_id: snapshot.target_id,
                                total_messages: snapshot.total_messages,
@@ -1275,7 +1274,6 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                            .into_iter()
                            .map(|snapshot| NotificationTargetStats {
                                failed_messages: snapshot.failed_messages,
-                                failed_store_length: snapshot.failed_store_length,
                                queue_length: snapshot.queue_length,
                                target_id: snapshot.target_id,
                                target_type: snapshot.target_type,
@@ -33,15 +33,6 @@ pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
    )
 });

-pub static AUDIT_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
-    new_gauge_md(
-        MetricName::AuditFailedStoreLength,
-        "Number of audit messages held in the failed-events store for target",
-        &[TARGET_ID],
-        subsystems::AUDIT,
-    )
-});
-
 pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::AuditTargetQueueLength,
@@ -175,7 +175,6 @@ pub enum MetricName {

    // Audit metrics
    AuditFailedMessages,
-    AuditFailedStoreLength,
    AuditTargetQueueLength,
    AuditTotalMessages,

@@ -222,7 +221,6 @@ pub enum MetricName {
    NotificationEventsSentTotal,
    NotificationEventsSkippedTotal,
    NotificationTargetFailedMessages,
-    NotificationTargetFailedStoreLength,
    NotificationTargetQueueLength,
    NotificationTargetTotalMessages,

@@ -583,7 +581,6 @@ impl MetricName {
            Self::ApiTrafficRecvBytes => "traffic_received_bytes".to_string(),

            Self::AuditFailedMessages => "failed_messages".to_string(),
-            Self::AuditFailedStoreLength => "failed_store_length".to_string(),
            Self::AuditTargetQueueLength => "target_queue_length".to_string(),
            Self::AuditTotalMessages => "total_messages".to_string(),

@@ -630,7 +627,6 @@ impl MetricName {
            Self::NotificationEventsSentTotal => "events_sent_total".to_string(),
            Self::NotificationEventsSkippedTotal => "events_skipped_total".to_string(),
            Self::NotificationTargetFailedMessages => "failed_messages".to_string(),
-            Self::NotificationTargetFailedStoreLength => "failed_store_length".to_string(),
            Self::NotificationTargetQueueLength => "target_queue_length".to_string(),
            Self::NotificationTargetTotalMessages => "total_messages".to_string(),

@@ -31,15 +31,6 @@ pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> =
    )
 });

-pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
-    new_gauge_md(
-        MetricName::NotificationTargetFailedStoreLength,
-        "Number of notification messages held in the failed-events store for target",
-        &NOTIFICATION_TARGET_LABELS,
-        subsystems::NOTIFICATION,
-    )
-});
-
 pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::NotificationTargetQueueLength,
@@ -0,0 +1,305 @@
+//! Stable, low-cardinality observability vocabulary for RustFS request paths.
+//!
+//! This module intentionally accepts no bucket, object, request, URL, or
+//! credential values. Those values are either unbounded or sensitive and must
+//! not become telemetry dimensions.
+
+use metrics::{Gauge, counter, gauge, histogram};
+use opentelemetry::trace::Status;
+use rustfs_utils::trace_attributes;
+use std::future::Future;
+use std::time::Instant;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+const METRIC_REQUESTS_TOTAL: &str = "rustfs_s3_requests_total";
+const METRIC_REQUEST_FAILURES_TOTAL: &str = "rustfs_s3_request_failures_total";
+const METRIC_REQUEST_DURATION_SECONDS: &str = "rustfs_s3_request_duration_seconds";
+const METRIC_REQUESTS_IN_FLIGHT: &str = "rustfs_s3_requests_in_flight";
+
+struct InFlightRequest {
+    gauge: Gauge,
+}
+
+impl Drop for InFlightRequest {
+    fn drop(&mut self) {
+        self.gauge.decrement(1.0);
+    }
+}
+
+/// A stable S3 operation name suitable for telemetry dimensions.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum Operation {
+    CreateBucket,
+    DeleteBucket,
+    HeadBucket,
+    ListBuckets,
+    ListObjects,
+    ListObjectsV2,
+    PutObject,
+    GetObject,
+    HeadObject,
+    DeleteObject,
+    CreateMultipartUpload,
+    UploadPart,
+    UploadPartCopy,
+    CompleteMultipartUpload,
+    AbortMultipartUpload,
+}
+
+impl Operation {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::CreateBucket => "create_bucket",
+            Self::DeleteBucket => "delete_bucket",
+            Self::HeadBucket => "head_bucket",
+            Self::ListBuckets => "list_buckets",
+            Self::ListObjects => "list_objects",
+            Self::ListObjectsV2 => "list_objects_v2",
+            Self::PutObject => "put_object",
+            Self::GetObject => "get_object",
+            Self::HeadObject => "head_object",
+            Self::DeleteObject => "delete_object",
+            Self::CreateMultipartUpload => "create_multipart_upload",
+            Self::UploadPart => "upload_part",
+            Self::UploadPartCopy => "upload_part_copy",
+            Self::CompleteMultipartUpload => "complete_multipart_upload",
+            Self::AbortMultipartUpload => "abort_multipart_upload",
+        }
+    }
+}
+
+/// A bounded internal operation stage. Do not create spans per shard or block.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum Stage {
+    Authorize,
+    Metadata,
+    EcRead,
+    EcWrite,
+    Quorum,
+    Response,
+    ReplicationDispatch,
+    ReplicationRemote,
+}
+
+/// Bounded direction for a stream traversing the object pipeline.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum StreamDirection {
+    Ingress,
+    Copy,
+}
+
+impl StreamDirection {
+    const fn as_str(self) -> &'static str {
+        match self {
+            Self::Ingress => trace_attributes::stream_direction::INGRESS,
+            Self::Copy => trace_attributes::stream_direction::COPY,
+        }
+    }
+}
+
+impl Stage {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Authorize => "authorize",
+            Self::Metadata => "metadata",
+            Self::EcRead => "ec_read",
+            Self::EcWrite => "ec_write",
+            Self::Quorum => "quorum",
+            Self::Response => "response",
+            Self::ReplicationDispatch => "replication_dispatch",
+            Self::ReplicationRemote => "replication_remote",
+        }
+    }
+}
+
+/// Bounded result classification shared by metrics and spans.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ResultClass {
+    Success,
+    Error,
+}
+
+impl ResultClass {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Success => "success",
+            Self::Error => "error",
+        }
+    }
+}
+
+/// Stable error categories for future fine-grained operation instrumentation.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ErrorClass {
+    Validation,
+    NotFound,
+    Conflict,
+    Timeout,
+    Canceled,
+    ResourceExhausted,
+    Io,
+    Network,
+    Quorum,
+    Consistency,
+    Internal,
+}
+
+impl ErrorClass {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Validation => "validation_error",
+            Self::NotFound => "not_found",
+            Self::Conflict => "conflict",
+            Self::Timeout => "timeout",
+            Self::Canceled => "canceled",
+            Self::ResourceExhausted => "resource_exhausted",
+            Self::Io => "io_error",
+            Self::Network => "network_error",
+            Self::Quorum => "quorum_failed",
+            Self::Consistency => "consistency_violation",
+            Self::Internal => "internal_error",
+        }
+    }
+}
+
+/// Create a child span for a bounded internal stage.
+pub fn stage_span(stage: Stage) -> tracing::Span {
+    tracing::info_span!(
+        "rustfs.stage",
+        event = "rustfs_stage",
+        component = "storage",
+        stage = stage.as_str(),
+        "rustfs.stage" = stage.as_str()
+    )
+}
+
+/// Create a child span for a request stream without exposing payload contents.
+pub fn stream_span(direction: StreamDirection, expected_bytes: i64, buffer_bytes: usize) -> tracing::Span {
+    tracing::info_span!(
+        "rustfs.stream",
+        event = "rustfs_stream",
+        component = "storage",
+        "rustfs.stream.direction" = direction.as_str(),
+        "rustfs.stream.expected_bytes" = expected_bytes,
+        "rustfs.stream.buffer_bytes" = buffer_bytes
+    )
+}
+
+/// Observe a complete S3 operation without exposing request data.
+pub async fn observe_operation<T, E, F>(operation: Operation, future: F) -> Result<T, E>
+where
+    F: Future<Output = Result<T, E>>,
+{
+    let operation_name = operation.as_str();
+    let span = tracing::info_span!(
+        "rustfs.s3.operation",
+        event = "s3_operation",
+        component = "storage",
+        operation = operation_name,
+        "rustfs.operation" = operation_name
+    );
+    let in_flight = gauge!(METRIC_REQUESTS_IN_FLIGHT, "operation" => operation_name);
+    in_flight.increment(1.0);
+    let _in_flight = InFlightRequest { gauge: in_flight };
+    let started_at = Instant::now();
+    let result = future.instrument(span.clone()).await;
+
+    let result_class = if result.is_ok() {
+        ResultClass::Success
+    } else {
+        ResultClass::Error
+    };
+    span.set_status(match result_class {
+        ResultClass::Success => Status::Ok,
+        ResultClass::Error => Status::error(ErrorClass::Internal.as_str()),
+    });
+    counter!(METRIC_REQUESTS_TOTAL, "operation" => operation_name, "result" => result_class.as_str()).increment(1);
+    histogram!(METRIC_REQUEST_DURATION_SECONDS, "operation" => operation_name, "result" => result_class.as_str())
+        .record(started_at.elapsed().as_secs_f64());
+    if result_class == ResultClass::Error {
+        counter!(METRIC_REQUEST_FAILURES_TOTAL, "operation" => operation_name, "error.type" => ErrorClass::Internal.as_str())
+            .increment(1);
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use metrics::with_local_recorder;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+    use std::collections::HashSet;
+    use std::task::Poll;
+
+    fn capture_operation_metrics(result: Result<(), ()>) -> Vec<(String, HashSet<String>)> {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let runtime = tokio::runtime::Runtime::new().expect("create test runtime");
+        with_local_recorder(&recorder, || {
+            let observed = runtime.block_on(observe_operation(Operation::PutObject, async { result }));
+            assert_eq!(observed, result);
+        });
+        snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .map(|(composite, _, _, _)| {
+                (
+                    composite.key().name().to_string(),
+                    composite.key().labels().map(|label| label.key().to_string()).collect(),
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn operation_names_are_stable_and_bounded() {
+        assert_eq!(Operation::PutObject.as_str(), "put_object");
+        assert_eq!(Operation::UploadPartCopy.as_str(), "upload_part_copy");
+        assert_eq!(Operation::CompleteMultipartUpload.as_str(), "complete_multipart_upload");
+        assert_eq!(Stage::ReplicationRemote.as_str(), "replication_remote");
+        assert_eq!(StreamDirection::Ingress.as_str(), trace_attributes::stream_direction::INGRESS);
+        assert_eq!(ErrorClass::Quorum.as_str(), "quorum_failed");
+    }
+
+    #[test]
+    fn operation_metrics_use_only_bounded_safe_labels() {
+        let metrics = capture_operation_metrics(Err(()));
+        let labels_for = |name| {
+            metrics
+                .iter()
+                .filter(|(metric_name, _)| metric_name == name)
+                .map(|(_, labels)| labels)
+                .collect::<Vec<_>>()
+        };
+
+        let operation_result = HashSet::from(["operation".to_string(), "result".to_string()]);
+        let operation_error = HashSet::from(["operation".to_string(), "error.type".to_string()]);
+        let operation_only = HashSet::from(["operation".to_string()]);
+        assert_eq!(labels_for(METRIC_REQUESTS_TOTAL), vec![&operation_result]);
+        assert_eq!(labels_for(METRIC_REQUEST_FAILURES_TOTAL), vec![&operation_error]);
+        assert_eq!(labels_for(METRIC_REQUEST_DURATION_SECONDS), vec![&operation_result]);
+        assert_eq!(labels_for(METRIC_REQUESTS_IN_FLIGHT), vec![&operation_only]);
+    }
+
+    #[test]
+    fn canceled_operation_releases_in_flight_gauge() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let runtime = tokio::runtime::Runtime::new().expect("create test runtime");
+        with_local_recorder(&recorder, || {
+            runtime.block_on(async {
+                let operation = std::pin::pin!(observe_operation(Operation::GetObject, std::future::pending::<Result<(), ()>>()));
+                assert!(matches!(futures_util::poll!(operation), Poll::Pending));
+            });
+        });
+
+        let gauge = snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .find_map(|(composite, _, _, value)| (composite.key().name() == METRIC_REQUESTS_IN_FLIGHT).then_some(value));
+        assert_eq!(gauge, Some(DebugValue::Gauge(0.0.into())));
+    }
+}
@@ -0,0 +1,52 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Task helpers for preserving tracing context across Tokio task boundaries.
+
+use tracing::Instrument;
+
+/// Spawn a Tokio task that inherits the current tracing span.
+///
+/// Tokio does not propagate [`tracing::Span::current`] into spawned tasks.
+/// Use this for work that remains part of the caller's operation. Detached
+/// background side effects should start a new span instead.
+pub fn spawn_traced<F>(future: F) -> tokio::task::JoinHandle<F::Output>
+where
+    F: std::future::Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    tokio::spawn(future.instrument(tracing::Span::current()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::spawn_traced;
+
+    #[tokio::test]
+    async fn spawned_task_inherits_current_span() {
+        tracing::subscriber::set_global_default(tracing_subscriber::Registry::default())
+            .expect("task tracing test should install its subscriber");
+
+        let (parent_id, task) = {
+            let parent = tracing::info_span!("parent");
+            let parent_id = parent.id().expect("enabled parent span should have an id");
+            let task = parent
+                .in_scope(|| spawn_traced(async { tracing::Span::current().id().expect("task should retain its parent span") }));
+            (parent_id, task)
+        };
+
+        let task_span_id = task.await.expect("task should complete");
+        assert_eq!(task_span_id, parent_id);
+    }
+}
@@ -228,6 +228,16 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
    filter
 }

+/// Build the filter used exclusively by the OTLP trace exporter.
+///
+/// Trace collection must not depend on the log sink's severity: production
+/// deployments commonly run with `RUST_LOG=warn` or `error` while still
+/// expecting request spans to be exported. RustFS operation spans are emitted
+/// at `info`, so retain that level regardless of `RUST_LOG`.
+pub(super) fn build_trace_filter() -> EnvFilter {
+    EnvFilter::new("info")
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -344,6 +354,13 @@ mod tests {
        });
    }

+    #[test]
+    fn trace_filter_ignores_rust_log_severity() {
+        temp_env::with_var("RUST_LOG", Some("error"), || {
+            assert_eq!(build_trace_filter().to_string(), "info");
+        });
+    }
+
    #[test]
    fn test_build_env_filter_target_only_rust_log_keeps_target_verbose() {
        // `RUST_LOG=hyper` is a target-only directive and should not be treated
@@ -39,7 +39,7 @@
 use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
-use crate::telemetry::filter::build_env_filter;
+use crate::telemetry::filter::{build_env_filter, build_trace_filter};
 use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
 use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
 use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -241,8 +241,10 @@ pub(super) fn init_observability_http(
    // ── Tracing subscriber registry ───────────────────────────────────────────
    let tracer_layer = tracer_provider
        .as_ref()
-        .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())));
-    let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone()));
+        .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())).with_filter(build_trace_filter()));
+    let metrics_layer = meter_provider
+        .as_ref()
+        .map(|p| MetricsLayer::new(p.clone()).with_filter(build_env_filter(logger_level, None)));

    // Optional stdout mirror (matching init_file_logging_internal logic)
    // This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules.
@@ -253,12 +255,11 @@ pub(super) fn init_observability_http(
    }
    let local_file_fallback_enabled = file_layer_opt.is_some();
    let stdout_mirror_enabled = stdout_guard.is_some();
-    let filter = build_env_filter(logger_level, None);
    tracing_subscriber::registry()
-        .with(filter)
+        .with(build_trace_filter())
        .with(ErrorLayer::default())
-        .with(file_layer_opt)
-        .with(stdout_layer_opt)
+        .with(file_layer_opt.map(|layer| layer.with_filter(build_env_filter(logger_level, None))))
+        .with(stdout_layer_opt.map(|layer| layer.with_filter(build_env_filter(logger_level, None))))
        .with(tracer_layer)
        .with(otel_bridge)
        .with(metrics_layer)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
唐小鸭	f9eddf14b1	feat(obs): enhance observability with tracing spans and metrics integration	2026-07-19 22:03:03 +08:00
唐小鸭	e7e40007ab	Add observability identities and operation spans	2026-07-14 15:22:20 +08:00